/** * Re-implement browse. * * We need to do slightly different things for groups vs saved search groups, hence we * re-implement browse from Page_Basic. * * @param int $action */ public function browse($action = NULL) { $groupPermission = CRM_Core_Permission::check('edit groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW; $this->assign('groupPermission', $groupPermission); $showOrgInfo = FALSE; // CRM-9936 $reservedPermission = CRM_Core_Permission::check('administer reserved groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW; $this->assign('reservedPermission', $reservedPermission); if (CRM_Core_Permission::check('administer Multiple Organizations') && CRM_Core_Permission::isMultisiteEnabled()) { $showOrgInfo = TRUE; } $this->assign('showOrgInfo', $showOrgInfo); // Refresh smart group cache if (!empty($_GET['update_smart_groups'])) { CRM_Contact_BAO_GroupContactCache::loadAll(); } else { CRM_Contact_BAO_GroupContactCache::fillIfEmpty(); } $this->search(); }
/** * This api reloads all the smart groups. * * If the org has a large number of smart groups it is recommended that they use the limit clause * to limit the number of smart groups evaluated on a per job basis. * * Might also help to increase the smartGroupCacheTimeout and use the cache. * * @param array $params * * @return array * @throws \API_Exception */ function civicrm_api3_job_group_rebuild($params) { $lock = Civi::lockManager()->acquire('worker.core.GroupRebuild'); if (!$lock->isAcquired()) { throw new API_Exception('Could not acquire lock, another EmailProcessor process is running'); } $limit = CRM_Utils_Array::value('limit', $params, 0); CRM_Contact_BAO_GroupContactCache::loadAll(NULL, $limit); $lock->release(); return civicrm_api3_create_success(); }
/** * This api reloads all the smart groups. If the org has a large number of smart groups * it is recommended that they use the limit clause to limit the number of smart groups * evaluated on a per job basis. Might also help to increase the smartGroupCacheTimeout * and use the cache */ function civicrm_api3_job_group_rebuild($params) { $lock = new CRM_Core_Lock('civimail.job.groupRebuild'); if (!$lock->isAcquired()) { return civicrm_api3_create_error('Could not acquire lock, another EmailProcessor process is running'); } $limit = CRM_Utils_Array::value('limit', $params, 0); CRM_Contact_BAO_GroupContactCache::loadAll(null, $limit); $lock->release(); return civicrm_api3_create_success(); }
/** * Collect CiviCRM data into temporary working table. */ static function syncCollectCiviCRM($listID) { CRM_Mailchimp_Utils::checkDebug('Start-CRM_Mailchimp_Form_Sync syncCollectCiviCRM $listID= ', $listID); // Nb. these are temporary tables but we don't use TEMPORARY table because they are // needed over multiple sessions because of queue. CRM_Core_DAO::executeQuery("DROP TABLE IF EXISTS tmp_mailchimp_push_c;"); $dao = CRM_Core_DAO::executeQuery("CREATE TABLE tmp_mailchimp_push_c (\n contact_id INT(10) UNSIGNED NOT NULL,\n email_id INT(10) UNSIGNED NOT NULL,\n email VARCHAR(200),\n first_name VARCHAR(100),\n last_name VARCHAR(100),\n hash CHAR(32),\n groupings VARCHAR(4096),\n PRIMARY KEY (email_id, email, hash)\n );"); // Cheekily access the database directly to obtain a prepared statement. $db = $dao->getDatabaseConnection(); $insert = $db->prepare('INSERT INTO tmp_mailchimp_push_c VALUES(?, ?, ?, ?, ?, ?, ?)'); //create table for mailchim civicrm syn errors $dao = CRM_Core_DAO::executeQuery("CREATE TABLE IF NOT EXISTS mailchimp_civicrm_syn_errors (\n id int(11) NOT NULL AUTO_INCREMENT,\n email VARCHAR(200),\n error VARCHAR(200),\n error_count int(10),\n group_id int(20),\n list_id VARCHAR(20),\n PRIMARY KEY (id)\n );"); // We need to know what groupings we have maps to. // We only care about CiviCRM groups that are mapped to this MC List: $mapped_groups = CRM_Mailchimp_Utils::getGroupsToSync(array(), $listID); // First, get all subscribers from the membership group for this list. // ... Find CiviCRM group id for the membership group. // ... And while we're at it, build an SQL-safe array of groupIds for groups mapped to groupings. // (we use that later) $membership_group_id = FALSE; $grouping_group_ids = array('normal' => array(), 'smart' => array()); $default_info = array(); foreach ($mapped_groups as $group_id => $details) { CRM_Contact_BAO_GroupContactCache::loadAll($group_id); if (!$details['grouping_id']) { $membership_group_id = $group_id; } else { $grouping_group_ids[$details['civigroup_uses_cache'] ? 'smart' : 'normal'][] = (int) $group_id; $default_info[$details['grouping_id']][$details['group_id']] = FALSE; } } $grouping_group_ids['smart'] = implode(',', $grouping_group_ids['smart']); $grouping_group_ids['normal'] = implode(',', $grouping_group_ids['normal']); if (!$membership_group_id) { throw new Exception("No CiviCRM group is mapped to determine membership of Mailchimp list {$listID}"); } // ... Load all subscribers in $groupContact object if (!($groupContact = CRM_Mailchimp_Utils::getGroupContactObject($membership_group_id))) { CRM_Mailchimp_Utils::checkDebug('get group contact= ', $groupContact); throw new Exception("No CiviCRM group is mapped to determine membership of Mailchimp list {$listID}. CiviCRM group {$membership_group_id} failed to load"); } // Now we iterate through the subscribers, collecting data about the other mapped groups // This is pretty inefficient :-( while ($groupContact->fetch()) { // Find the contact, for the name fields $contact = new CRM_Contact_BAO_Contact(); $contact->id = $groupContact->contact_id; $contact->is_deleted = 0; if (!$contact->find(TRUE)) { continue; } // Find their primary (bulk) email $email = new CRM_Core_BAO_Email(); $email->contact_id = $groupContact->contact_id; $email->is_primary = TRUE; if (!$email->find(TRUE)) { continue; } // If no email, it's like they're not there. if (!$email->email || $email->on_hold || $contact->is_opt_out || $contact->do_not_email) { //@todo update stats. continue; } // Find out if they're in any groups that we care about. // Start off as not in the groups... $info = $default_info; // We can do this with two queries, one for normal groups, one for smart groups. // Normal groups. if ($grouping_group_ids['normal']) { $groupContact2 = new CRM_Contact_BAO_GroupContact(); $groupContact2->contact_id = $groupContact->contact_id; $groupContact2->whereAdd("status = 'Added'"); $groupContact2->whereAdd("group_id IN ({$grouping_group_ids['normal']})"); $groupContact2->find(); while ($groupContact2->fetch()) { // need MC grouping_id and group_id $details = $mapped_groups[$groupContact2->group_id]; $info[$details['grouping_id']][$details['group_id']] = TRUE; } unset($groupContact2); } // Smart groups if ($grouping_group_ids['smart']) { $groupContactCache = new CRM_Contact_BAO_GroupContactCache(); $groupContactCache->contact_id = $groupContact->contact_id; $groupContactCache->whereAdd("group_id IN ({$grouping_group_ids['smart']})"); $groupContactCache->find(); while ($groupContactCache->fetch()) { // need MC grouping_id and group_id $details = $mapped_groups[$groupContactCache->group_id]; $info[$details['grouping_id']][$details['group_id']] = TRUE; } unset($groupContactCache); } // OK we should now have all the info we need. // Serialize the grouping array for SQL storage - this is the fastest way. $info = serialize($info); // we're ready to store this but we need a hash that contains all the info // for comparison with the hash created from the CiviCRM data (elsewhere). // email, first name, last name, groupings $hash = md5($email->email . $contact->first_name . $contact->last_name . $info); // run insert prepared statement $db->execute($insert, array($contact->id, $email->id, $email->email, $contact->first_name, $contact->last_name, $hash, $info)); } // Tidy up. $db->freePrepared($insert); // count $dao = CRM_Core_DAO::executeQuery("SELECT COUNT(*) c FROM tmp_mailchimp_push_c"); $dao->fetch(); CRM_Mailchimp_Utils::checkDebug('End-CRM_Mailchimp_Form_Sync syncCollectCiviCRM $listID= ', $listID); return $dao->c; }
/** * Re-implement browse. * * We need to do slightly different things for groups vs saved search groups, hence we * re-implement browse from Page_Basic. * * @param int $action */ public function browse($action = NULL) { $groupPermission = CRM_Core_Permission::check('edit groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW; $this->assign('groupPermission', $groupPermission); $showOrgInfo = FALSE; // CRM-9936 $reservedPermission = CRM_Core_Permission::check('administer reserved groups') ? CRM_Core_Permission::EDIT : CRM_Core_Permission::VIEW; $this->assign('reservedPermission', $reservedPermission); if (CRM_Core_Permission::check('administer Multiple Organizations') && CRM_Core_Permission::isMultisiteEnabled()) { $showOrgInfo = TRUE; } $this->assign('showOrgInfo', $showOrgInfo); // Refresh smart group cache if (!empty($_GET['update_smart_groups'])) { CRM_Contact_BAO_GroupContactCache::loadAll(); } elseif (!CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_group_contact_cache LIMIT 1")) { CRM_Core_Session::setStatus(ts('Count data for smart groups is not currently calculated. You may click Update Smart Groups to generate it. Be aware this can cause significant server load')); } $this->search(); }
/** * Collect CiviCRM data into temporary working table. * * Speed notes. * * Various strategies have been tried here to speed things up. Originally we * used the API with a chained API call, but this was very slow (~10s for * ~5k contacts), so now we load all the contacts, then all the emails in a * 2nd API call. This is about 10x faster, taking less than 1s for ~5k * contacts. Likewise the structuring of the emails on the contact array has * been tried various ways, and this structure-by-type way has reduced the * origninal loop time from 7s down to just under 4s. * * * @param string $mode pull|push. * @return int number of contacts collected. */ public function collectCiviCrm($mode) { CRM_Mailchimp_Utils::checkDebug('Start-CRM_Mailchimp_Form_Sync syncCollectCiviCRM $this->list_id= ', $this->list_id); if (!in_array($mode, ['pull', 'push'])) { throw new InvalidArgumentException(__FUNCTION__ . " expects push/pull but called with '{$mode}'."); } // Cheekily access the database directly to obtain a prepared statement. $dao = static::createTemporaryTableForCiviCRM(); $db = $dao->getDatabaseConnection(); // There used to be a distinction between the handling of 'normal' groups // and smart groups. But now the API will take care of this but this // requires the following function to have run. foreach ($this->interest_group_details as $group_id => $details) { if ($mode == 'push' || $details['is_mc_update_grouping'] == 1) { // Either we are collecting for a push from C->M, // or we're pulling and this group is configured to allow updates. // Therefore we need to make sure the cache is filled. CRM_Contact_BAO_GroupContactCache::loadAll($group_id); } } // Use a nice API call to get the information for tmp_mailchimp_push_c. // The API will take care of smart groups. $start = microtime(TRUE); $result = civicrm_api3('Contact', 'get', ['is_deleted' => 0, 'is_opt_out' => 0, 'do_not_email' => 0, 'on_hold' => 0, 'is_deceased' => 0, 'group' => $this->membership_group_id, 'return' => ['first_name', 'last_name', 'group'], 'options' => ['limit' => 0]]); if ($result['count'] == 0) { // No-one is in the group according to CiviCRM. return 0; } // Load emails for these contacts. $emails = civicrm_api3('Email', 'get', ['on_hold' => 0, 'return' => 'contact_id,email,is_bulkmail,is_primary', 'contact_id' => ['IN' => array_keys($result['values'])], 'options' => ['limit' => 0]]); // Index emails by contact_id. foreach ($emails['values'] as $email) { if ($email['is_bulkmail']) { $result['values'][$email['contact_id']]['bulk_email'] = $email['email']; } elseif ($email['is_primary']) { $result['values'][$email['contact_id']]['primary_email'] = $email['email']; } else { $result['values'][$email['contact_id']]['other_email'] = $email['email']; } } /** * We have a contact that has no other deets. */ $start = microtime(TRUE); $collected = 0; $insert = $db->prepare('INSERT INTO tmp_mailchimp_push_c VALUES(?, ?, ?, ?, ?, ?)'); // Loop contacts: foreach ($result['values'] as $id => $contact) { // Which email to use? $email = isset($contact['bulk_email']) ? $contact['bulk_email'] : (isset($contact['primary_email']) ? $contact['primary_email'] : (isset($contact['other_email']) ? $contact['other_email'] : NULL)); if (!$email) { // Hmmm. continue; } // Find out the ID's of the groups the $contact belongs to, and // save in $info. $info = $this->getComparableInterestsFromCiviCrmGroups($contact['groups'], $mode); // OK we should now have all the info we need. // Serialize the grouping array for SQL storage - this is the fastest way. $info = serialize($info); // we're ready to store this but we need a hash that contains all the info // for comparison with the hash created from the CiviCRM data (elsewhere). // email, first name, last name, groupings // See note above about why we don't include email in the hash. // $hash = md5($email . $contact['first_name'] . $contact['last_name'] . $info); $hash = md5($contact['first_name'] . $contact['last_name'] . $info); // run insert prepared statement $db->execute($insert, array($contact['id'], $email, $contact['first_name'], $contact['last_name'], $hash, $info)); $collected++; } // Tidy up. $db->freePrepared($insert); return $collected; }