/**
* Re-implement browse.
*
* We need to do slightly different things for groups vs saved search groups, hence we
* re-implement browse from Page_Basic.
*
* @param int $action
*/
public function browse($action = NULL)
{
$groupPermission = CRM_Core_Permission::check('edit groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW;
$this->assign('groupPermission', $groupPermission);
$showOrgInfo = FALSE;
// CRM-9936
$reservedPermission = CRM_Core_Permission::check('administer reserved groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW;
$this->assign('reservedPermission', $reservedPermission);
if (CRM_Core_Permission::check('administer Multiple Organizations') && CRM_Core_Permission::isMultisiteEnabled()) {
$showOrgInfo = TRUE;
}
$this->assign('showOrgInfo', $showOrgInfo);
// Refresh smart group cache
if (!empty($_GET['update_smart_groups'])) {
CRM_Contact_BAO_GroupContactCache::loadAll();
} else {
CRM_Contact_BAO_GroupContactCache::fillIfEmpty();
}
$this->search();
}
/**
* This api reloads all the smart groups.
*
* If the org has a large number of smart groups it is recommended that they use the limit clause
* to limit the number of smart groups evaluated on a per job basis.
*
* Might also help to increase the smartGroupCacheTimeout and use the cache.
*
* @param array $params
*
* @return array
* @throws \API_Exception
*/
function civicrm_api3_job_group_rebuild($params)
{
$lock = Civi::lockManager()->acquire('worker.core.GroupRebuild');
if (!$lock->isAcquired()) {
throw new API_Exception('Could not acquire lock, another EmailProcessor process is running');
}
$limit = CRM_Utils_Array::value('limit', $params, 0);
CRM_Contact_BAO_GroupContactCache::loadAll(NULL, $limit);
$lock->release();
return civicrm_api3_create_success();
}
/**
* This api reloads all the smart groups. If the org has a large number of smart groups
* it is recommended that they use the limit clause to limit the number of smart groups
* evaluated on a per job basis. Might also help to increase the smartGroupCacheTimeout
* and use the cache
*/
function civicrm_api3_job_group_rebuild($params)
{
$lock = new CRM_Core_Lock('civimail.job.groupRebuild');
if (!$lock->isAcquired()) {
return civicrm_api3_create_error('Could not acquire lock, another EmailProcessor process is running');
}
$limit = CRM_Utils_Array::value('limit', $params, 0);
CRM_Contact_BAO_GroupContactCache::loadAll(null, $limit);
$lock->release();
return civicrm_api3_create_success();
}
/**
* Collect CiviCRM data into temporary working table.
*/
static function syncCollectCiviCRM($listID)
{
CRM_Mailchimp_Utils::checkDebug('Start-CRM_Mailchimp_Form_Sync syncCollectCiviCRM $listID= ', $listID);
// Nb. these are temporary tables but we don't use TEMPORARY table because they are
// needed over multiple sessions because of queue.
CRM_Core_DAO::executeQuery("DROP TABLE IF EXISTS tmp_mailchimp_push_c;");
$dao = CRM_Core_DAO::executeQuery("CREATE TABLE tmp_mailchimp_push_c (\n contact_id INT(10) UNSIGNED NOT NULL,\n email_id INT(10) UNSIGNED NOT NULL,\n email VARCHAR(200),\n first_name VARCHAR(100),\n last_name VARCHAR(100),\n hash CHAR(32),\n groupings VARCHAR(4096),\n PRIMARY KEY (email_id, email, hash)\n );");
// Cheekily access the database directly to obtain a prepared statement.
$db = $dao->getDatabaseConnection();
$insert = $db->prepare('INSERT INTO tmp_mailchimp_push_c VALUES(?, ?, ?, ?, ?, ?, ?)');
//create table for mailchim civicrm syn errors
$dao = CRM_Core_DAO::executeQuery("CREATE TABLE IF NOT EXISTS mailchimp_civicrm_syn_errors (\n id int(11) NOT NULL AUTO_INCREMENT,\n email VARCHAR(200),\n error VARCHAR(200),\n error_count int(10),\n group_id int(20),\n list_id VARCHAR(20),\n PRIMARY KEY (id)\n );");
// We need to know what groupings we have maps to.
// We only care about CiviCRM groups that are mapped to this MC List:
$mapped_groups = CRM_Mailchimp_Utils::getGroupsToSync(array(), $listID);
// First, get all subscribers from the membership group for this list.
// ... Find CiviCRM group id for the membership group.
// ... And while we're at it, build an SQL-safe array of groupIds for groups mapped to groupings.
// (we use that later)
$membership_group_id = FALSE;
$grouping_group_ids = array('normal' => array(), 'smart' => array());
$default_info = array();
foreach ($mapped_groups as $group_id => $details) {
CRM_Contact_BAO_GroupContactCache::loadAll($group_id);
if (!$details['grouping_id']) {
$membership_group_id = $group_id;
} else {
$grouping_group_ids[$details['civigroup_uses_cache'] ? 'smart' : 'normal'][] = (int) $group_id;
$default_info[$details['grouping_id']][$details['group_id']] = FALSE;
}
}
$grouping_group_ids['smart'] = implode(',', $grouping_group_ids['smart']);
$grouping_group_ids['normal'] = implode(',', $grouping_group_ids['normal']);
if (!$membership_group_id) {
throw new Exception("No CiviCRM group is mapped to determine membership of Mailchimp list {$listID}");
}
// ... Load all subscribers in $groupContact object
if (!($groupContact = CRM_Mailchimp_Utils::getGroupContactObject($membership_group_id))) {
CRM_Mailchimp_Utils::checkDebug('get group contact= ', $groupContact);
throw new Exception("No CiviCRM group is mapped to determine membership of Mailchimp list {$listID}. CiviCRM group {$membership_group_id} failed to load");
}
// Now we iterate through the subscribers, collecting data about the other mapped groups
// This is pretty inefficient :-(
while ($groupContact->fetch()) {
// Find the contact, for the name fields
$contact = new CRM_Contact_BAO_Contact();
$contact->id = $groupContact->contact_id;
$contact->is_deleted = 0;
if (!$contact->find(TRUE)) {
continue;
}
// Find their primary (bulk) email
$email = new CRM_Core_BAO_Email();
$email->contact_id = $groupContact->contact_id;
$email->is_primary = TRUE;
if (!$email->find(TRUE)) {
continue;
}
// If no email, it's like they're not there.
if (!$email->email || $email->on_hold || $contact->is_opt_out || $contact->do_not_email) {
//@todo update stats.
continue;
}
// Find out if they're in any groups that we care about.
// Start off as not in the groups...
$info = $default_info;
// We can do this with two queries, one for normal groups, one for smart groups.
// Normal groups.
if ($grouping_group_ids['normal']) {
$groupContact2 = new CRM_Contact_BAO_GroupContact();
$groupContact2->contact_id = $groupContact->contact_id;
$groupContact2->whereAdd("status = 'Added'");
$groupContact2->whereAdd("group_id IN ({$grouping_group_ids['normal']})");
$groupContact2->find();
while ($groupContact2->fetch()) {
// need MC grouping_id and group_id
$details = $mapped_groups[$groupContact2->group_id];
$info[$details['grouping_id']][$details['group_id']] = TRUE;
}
unset($groupContact2);
}
// Smart groups
if ($grouping_group_ids['smart']) {
$groupContactCache = new CRM_Contact_BAO_GroupContactCache();
$groupContactCache->contact_id = $groupContact->contact_id;
$groupContactCache->whereAdd("group_id IN ({$grouping_group_ids['smart']})");
$groupContactCache->find();
while ($groupContactCache->fetch()) {
// need MC grouping_id and group_id
$details = $mapped_groups[$groupContactCache->group_id];
$info[$details['grouping_id']][$details['group_id']] = TRUE;
}
unset($groupContactCache);
}
// OK we should now have all the info we need.
// Serialize the grouping array for SQL storage - this is the fastest way.
$info = serialize($info);
// we're ready to store this but we need a hash that contains all the info
// for comparison with the hash created from the CiviCRM data (elsewhere).
// email, first name, last name, groupings
$hash = md5($email->email . $contact->first_name . $contact->last_name . $info);
// run insert prepared statement
$db->execute($insert, array($contact->id, $email->id, $email->email, $contact->first_name, $contact->last_name, $hash, $info));
}
// Tidy up.
$db->freePrepared($insert);
// count
$dao = CRM_Core_DAO::executeQuery("SELECT COUNT(*) c FROM tmp_mailchimp_push_c");
$dao->fetch();
CRM_Mailchimp_Utils::checkDebug('End-CRM_Mailchimp_Form_Sync syncCollectCiviCRM $listID= ', $listID);
return $dao->c;
}
/**
* Re-implement browse.
*
* We need to do slightly different things for groups vs saved search groups, hence we
* re-implement browse from Page_Basic.
*
* @param int $action
*/
public function browse($action = NULL)
{
$groupPermission = CRM_Core_Permission::check('edit groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW;
$this->assign('groupPermission', $groupPermission);
$showOrgInfo = FALSE;
// CRM-9936
$reservedPermission = CRM_Core_Permission::check('administer reserved groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW;
$this->assign('reservedPermission', $reservedPermission);
if (CRM_Core_Permission::check('administer Multiple Organizations') && CRM_Core_Permission::isMultisiteEnabled()) {
$showOrgInfo = TRUE;
}
$this->assign('showOrgInfo', $showOrgInfo);
// Refresh smart group cache
if (!empty($_GET['update_smart_groups'])) {
CRM_Contact_BAO_GroupContactCache::loadAll();
} elseif (!CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_group_contact_cache LIMIT 1")) {
CRM_Core_Session::setStatus(ts('Count data for smart groups is not currently calculated. You may click Update Smart Groups to generate it. Be aware this can cause significant server load'));
}
$this->search();
}
/**
* Collect CiviCRM data into temporary working table.
*
* Speed notes.
*
* Various strategies have been tried here to speed things up. Originally we
* used the API with a chained API call, but this was very slow (~10s for
* ~5k contacts), so now we load all the contacts, then all the emails in a
* 2nd API call. This is about 10x faster, taking less than 1s for ~5k
* contacts. Likewise the structuring of the emails on the contact array has
* been tried various ways, and this structure-by-type way has reduced the
* origninal loop time from 7s down to just under 4s.
*
*
* @param string $mode pull|push.
* @return int number of contacts collected.
*/
public function collectCiviCrm($mode)
{
CRM_Mailchimp_Utils::checkDebug('Start-CRM_Mailchimp_Form_Sync syncCollectCiviCRM $this->list_id= ', $this->list_id);
if (!in_array($mode, ['pull', 'push'])) {
throw new InvalidArgumentException(__FUNCTION__ . " expects push/pull but called with '{$mode}'.");
}
// Cheekily access the database directly to obtain a prepared statement.
$dao = static::createTemporaryTableForCiviCRM();
$db = $dao->getDatabaseConnection();
// There used to be a distinction between the handling of 'normal' groups
// and smart groups. But now the API will take care of this but this
// requires the following function to have run.
foreach ($this->interest_group_details as $group_id => $details) {
if ($mode == 'push' || $details['is_mc_update_grouping'] == 1) {
// Either we are collecting for a push from C->M,
// or we're pulling and this group is configured to allow updates.
// Therefore we need to make sure the cache is filled.
CRM_Contact_BAO_GroupContactCache::loadAll($group_id);
}
}
// Use a nice API call to get the information for tmp_mailchimp_push_c.
// The API will take care of smart groups.
$start = microtime(TRUE);
$result = civicrm_api3('Contact', 'get', ['is_deleted' => 0, 'is_opt_out' => 0, 'do_not_email' => 0, 'on_hold' => 0, 'is_deceased' => 0, 'group' => $this->membership_group_id, 'return' => ['first_name', 'last_name', 'group'], 'options' => ['limit' => 0]]);
if ($result['count'] == 0) {
// No-one is in the group according to CiviCRM.
return 0;
}
// Load emails for these contacts.
$emails = civicrm_api3('Email', 'get', ['on_hold' => 0, 'return' => 'contact_id,email,is_bulkmail,is_primary', 'contact_id' => ['IN' => array_keys($result['values'])], 'options' => ['limit' => 0]]);
// Index emails by contact_id.
foreach ($emails['values'] as $email) {
if ($email['is_bulkmail']) {
$result['values'][$email['contact_id']]['bulk_email'] = $email['email'];
} elseif ($email['is_primary']) {
$result['values'][$email['contact_id']]['primary_email'] = $email['email'];
} else {
$result['values'][$email['contact_id']]['other_email'] = $email['email'];
}
}
/**
* We have a contact that has no other deets.
*/
$start = microtime(TRUE);
$collected = 0;
$insert = $db->prepare('INSERT INTO tmp_mailchimp_push_c VALUES(?, ?, ?, ?, ?, ?)');
// Loop contacts:
foreach ($result['values'] as $id => $contact) {
// Which email to use?
$email = isset($contact['bulk_email']) ? $contact['bulk_email'] : (isset($contact['primary_email']) ? $contact['primary_email'] : (isset($contact['other_email']) ? $contact['other_email'] : NULL));
if (!$email) {
// Hmmm.
continue;
}
// Find out the ID's of the groups the $contact belongs to, and
// save in $info.
$info = $this->getComparableInterestsFromCiviCrmGroups($contact['groups'], $mode);
// OK we should now have all the info we need.
// Serialize the grouping array for SQL storage - this is the fastest way.
$info = serialize($info);
// we're ready to store this but we need a hash that contains all the info
// for comparison with the hash created from the CiviCRM data (elsewhere).
// email, first name, last name, groupings
// See note above about why we don't include email in the hash.
// $hash = md5($email . $contact['first_name'] . $contact['last_name'] . $info);
$hash = md5($contact['first_name'] . $contact['last_name'] . $info);
// run insert prepared statement
$db->execute($insert, array($contact['id'], $email, $contact['first_name'], $contact['last_name'], $hash, $info));
$collected++;
}
// Tidy up.
$db->freePrepared($insert);
return $collected;
}
