/**
* Build a queue of tasks by dividing dupe pairs in batches.
*/
public static function getRunner()
{
$rgid = CRM_Utils_Request::retrieve('rgid', 'Positive', $this, FALSE, 0);
$gid = CRM_Utils_Request::retrieve('gid', 'Positive', $this, FALSE, 0);
$action = CRM_Utils_Request::retrieve('action', 'String', CRM_Core_DAO::$_nullObject);
$mode = CRM_Utils_Request::retrieve('mode', 'String', CRM_Core_DAO::$_nullObject, FALSE, 'safe');
$contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type');
$cacheKeyString = "merge {$contactType}";
$cacheKeyString .= $rgid ? "_{$rgid}" : '_0';
$cacheKeyString .= $gid ? "_{$gid}" : '_0';
$urlQry = "reset=1&action=update&rgid={$rgid}";
$urlQry = $gid ? $urlQry . "&gid={$gid}" : $urlQry;
if ($mode == 'aggressive' && !CRM_Core_Permission::check('force merge duplicate contacts')) {
CRM_Core_Session::setStatus(ts('You do not have permission to force merge duplicate contact records'), ts('Permission Denied'), 'error');
CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry));
}
// Setup the Queue
$queue = CRM_Queue_Service::singleton()->create(array('name' => $cacheKeyString, 'type' => 'Sql', 'reset' => TRUE));
$where = NULL;
if ($action == CRM_Core_Action::MAP) {
$where = "pn.is_selected = 1";
$isSelected = 1;
} else {
// else merge all (2)
$isSelected = 2;
}
$total = CRM_Core_BAO_PrevNextCache::getCount($cacheKeyString, NULL, $where);
if ($total <= 0) {
// Nothing to do.
CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry));
}
// reset merge stats, so we compute new stats
CRM_Dedupe_Merger::resetMergeStats($cacheKeyString);
for ($i = 1; $i <= ceil($total / self::BATCHLIMIT); $i++) {
$task = new CRM_Queue_Task(array('CRM_Contact_Page_DedupeMerge', 'callBatchMerge'), array($rgid, $gid, $mode, TRUE, self::BATCHLIMIT, $isSelected), "Processed " . $i * self::BATCHLIMIT . " pair of duplicates out of " . $total);
// Add the Task to the Queue
$queue->createItem($task);
}
// Setup the Runner
$urlQry .= "&context=conflicts";
$runner = new CRM_Queue_Runner(array('title' => ts('Merging Duplicates..'), 'queue' => $queue, 'errorMode' => CRM_Queue_Runner::ERROR_ABORT, 'onEndUrl' => CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry, TRUE, NULL, FALSE)));
return $runner;
}
public static function updateMergeStats($cacheKeyString, $result = array())
{
// gather latest stats
$merged = count($result['merged']);
$skipped = count($result['skipped']);
if ($merged <= 0 && $skipped <= 0) {
return;
}
// get previous stats
$previousStats = CRM_Core_BAO_PrevNextCache::retrieve("{$cacheKeyString}_stats");
if (!empty($previousStats)) {
if ($previousStats[0]['merged']) {
$merged = $merged + $previousStats[0]['merged'];
}
if ($previousStats[0]['skipped']) {
$skipped = $skipped + $previousStats[0]['skipped'];
}
}
// delete old stats
CRM_Dedupe_Merger::resetMergeStats($cacheKeyString);
// store the updated stats
$data = array('merged' => $merged, 'skipped' => $skipped);
$data = CRM_Core_DAO::escapeString(serialize($data));
$values = array();
$values[] = " ( 'civicrm_contact', 0, 0, '{$cacheKeyString}_stats', '{$data}' ) ";
CRM_Core_BAO_PrevNextCache::setItem($values);
}
/**
* Browse all rule groups.
*/
public function run()
{
$gid = CRM_Utils_Request::retrieve('gid', 'Positive', $this, FALSE, 0);
$action = CRM_Utils_Request::retrieve('action', 'String', $this, FALSE, 0);
$context = CRM_Utils_Request::retrieve('context', 'String', $this);
$session = CRM_Core_Session::singleton();
$contactIds = $session->get('selectedSearchContactIds');
if ($context == 'search' || !empty($contactIds)) {
$context = 'search';
$this->assign('backURL', $session->readUserContext());
}
if ($action & CRM_Core_Action::RENEW) {
// empty cache
$rgid = CRM_Utils_Request::retrieve('rgid', 'Positive', $this, FALSE, 0);
if ($rgid) {
$contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type');
$cacheKeyString = "merge {$contactType}";
$cacheKeyString .= $rgid ? "_{$rgid}" : '_0';
$cacheKeyString .= $gid ? "_{$gid}" : '_0';
CRM_Core_BAO_PrevNextCache::deleteItem(NULL, $cacheKeyString);
}
$urlQry = "reset=1&action=update&rgid={$rgid}";
if ($gid) {
$urlQry .= "&gid={$gid}";
}
CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry));
} elseif ($action & CRM_Core_Action::MAP) {
// do a batch merge if requested
$rgid = CRM_Utils_Request::retrieve('rgid', 'Positive', $this, FALSE, 0);
$result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', TRUE, 75);
$skippedCount = CRM_Utils_Request::retrieve('skipped', 'Positive', $this, FALSE, 0);
$skippedCount = $skippedCount + count($result['skipped']);
$mergedCount = CRM_Utils_Request::retrieve('merged', 'Positive', $this, FALSE, 0);
$mergedCount = $mergedCount + count($result['merged']);
if (empty($result['merged']) && empty($result['skipped'])) {
$message = '';
if ($mergedCount >= 1) {
$message = ts("%1 pairs of duplicates were merged", array(1 => $mergedCount));
}
if ($skippedCount >= 1) {
$message = $message ? "{$message} and " : '';
$message .= ts("%1 pairs of duplicates were skipped due to conflict", array(1 => $skippedCount));
}
$message .= ts(" during the batch merge process with safe mode.");
CRM_Core_Session::setStatus($message, ts('Merge Complete'), 'success');
$urlQry = "reset=1&action=update&rgid={$rgid}";
if ($gid) {
$urlQry .= "&gid={$gid}";
}
CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry));
} else {
$urlQry = "reset=1&action=map&rgid={$rgid}";
if ($gid) {
$urlQry .= "&gid={$gid}";
}
$urlQry .= "&skipped={$skippedCount}&merged={$mergedCount}";
CRM_Utils_System::jsRedirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry), ts('Batch Merge Task in progress'), ts('The batch merge task is still in progress. This page will be refreshed automatically.'));
}
}
if ($action & CRM_Core_Action::UPDATE || $action & CRM_Core_Action::BROWSE) {
$cid = CRM_Utils_Request::retrieve('cid', 'Positive', $this, FALSE, 0);
$rgid = CRM_Utils_Request::retrieve('rgid', 'Positive', $this, FALSE, 0);
$this->action = CRM_Core_Action::UPDATE;
//calculate the $contactType
if ($rgid) {
$contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type');
}
$sourceParams = 'snippet=4';
if ($gid) {
$sourceParams .= "&gid={$gid}";
}
if ($rgid) {
$sourceParams .= "&rgid={$rgid}";
}
if ($context == 'conflicts') {
$sourceParams .= "&selected=1";
}
$this->assign('sourceUrl', CRM_Utils_System::url('civicrm/ajax/dedupefind', $sourceParams, FALSE, NULL, FALSE));
//reload from cache table
$cacheKeyString = "merge {$contactType}";
$cacheKeyString .= $rgid ? "_{$rgid}" : '_0';
$cacheKeyString .= $gid ? "_{$gid}" : '_0';
$stats = CRM_Dedupe_Merger::getMergeStatsMsg($cacheKeyString);
if ($stats) {
CRM_Core_Session::setStatus($stats);
// reset so we not displaying same message again
CRM_Dedupe_Merger::resetMergeStats($cacheKeyString);
}
$join = "LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND\n pn.entity_id2 = de.contact_id2 )";
$where = "de.id IS NULL";
if ($context == 'conflicts') {
$where .= " AND pn.is_selected = 1";
}
$this->_mainContacts = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where);
if (empty($this->_mainContacts)) {
if ($context == 'conflicts') {
// if the current screen was intended to list only selected contacts, move back to full dupe list
$sourceParams = 'reset=1&action=update';
if ($gid) {
$sourceParams .= "&gid={$gid}";
}
if ($rgid) {
$sourceParams .= "&rgid={$rgid}";
}
CRM_Utils_System::redirect(CRM_Utils_System::url(CRM_Utils_System::currentPath(), $sourceParams));
}
if ($gid) {
$foundDupes = $this->get("dedupe_dupes_{$gid}");
if (!$foundDupes) {
$foundDupes = CRM_Dedupe_Finder::dupesInGroup($rgid, $gid);
}
$this->set("dedupe_dupes_{$gid}", $foundDupes);
} elseif (!empty($contactIds)) {
$foundDupes = $this->get("search_dedupe_dupes_{$gid}");
if (!$foundDupes) {
$foundDupes = CRM_Dedupe_Finder::dupes($rgid, $contactIds);
}
$this->get("search_dedupe_dupes_{$gid}", $foundDupes);
} else {
$foundDupes = $this->get('dedupe_dupes');
if (!$foundDupes) {
$foundDupes = CRM_Dedupe_Finder::dupes($rgid);
}
$this->set('dedupe_dupes', $foundDupes);
}
if (!$foundDupes) {
$ruleGroup = new CRM_Dedupe_BAO_RuleGroup();
$ruleGroup->id = $rgid;
$ruleGroup->find(TRUE);
$session = CRM_Core_Session::singleton();
$session->setStatus(ts('No possible duplicates were found using %1 rule.', array(1 => $ruleGroup->name)), ts('None Found'), 'info');
$url = CRM_Utils_System::url('civicrm/contact/deduperules', 'reset=1');
if ($context == 'search') {
$url = $session->readUserContext();
}
CRM_Utils_System::redirect($url);
} else {
$cids = array();
foreach ($foundDupes as $dupe) {
$cids[$dupe[0]] = 1;
$cids[$dupe[1]] = 1;
}
$cidString = implode(', ', array_keys($cids));
$sql = "SELECT id, display_name FROM civicrm_contact WHERE id IN ({$cidString}) ORDER BY sort_name";
$dao = new CRM_Core_DAO();
$dao->query($sql);
$displayNames = array();
while ($dao->fetch()) {
$displayNames[$dao->id] = $dao->display_name;
}
// FIXME: sort the contacts; $displayName
// is already sort_name-sorted, so use that
// (also, consider sorting by dupe count first)
// lobo - change the sort to by threshold value
// so the more likely dupes are sorted first
$session = CRM_Core_Session::singleton();
$userId = $session->get('userID');
$mainContacts = $permission = array();
foreach ($foundDupes as $dupes) {
$srcID = $dupes[0];
$dstID = $dupes[1];
if ($dstID == $userId) {
$srcID = $dupes[1];
$dstID = $dupes[0];
}
/***
* Eliminate this since it introduces 3 queries PER merge row
* and hence is very expensive
* CRM-8822
* if ( !array_key_exists( $srcID, $permission ) ) {
* $permission[$srcID] = CRM_Contact_BAO_Contact_Permission::allow( $srcID, CRM_Core_Permission::EDIT );
* }
* if ( !array_key_exists( $dstID, $permission ) ) {
* $permission[$dstID] = CRM_Contact_BAO_Contact_Permission::allow( $dstID, CRM_Core_Permission::EDIT );
* }
*
* $canMerge = ( $permission[$dstID] && $permission[$srcID] );
*
*/
// we'll do permission checking during the merge process
$canMerge = TRUE;
$mainContacts[] = $row = array('srcID' => $srcID, 'srcName' => $displayNames[$srcID], 'dstID' => $dstID, 'dstName' => $displayNames[$dstID], 'weight' => $dupes[2], 'canMerge' => $canMerge);
$data = CRM_Core_DAO::escapeString(serialize($row));
$values[] = " ( 'civicrm_contact', {$srcID}, {$dstID}, '{$cacheKeyString}', '{$data}' ) ";
}
if ($cid) {
$this->_cid = $cid;
}
if ($gid) {
$this->_gid = $gid;
}
$this->_rgid = $rgid;
$this->_mainContacts = $mainContacts;
CRM_Core_BAO_PrevNextCache::setItem($values);
$session = CRM_Core_Session::singleton();
if ($this->_cid) {
$session->pushUserContext(CRM_Utils_System::url('civicrm/contact/deduperules', "action=update&rgid={$this->_rgid}&gid={$this->_gid}&cid={$this->_cid}"));
} else {
$session->pushUserContext(CRM_Utils_System::url('civicrm/contact/dedupefind', "reset=1&action=update&rgid={$this->_rgid}"));
}
}
} else {
if ($cid) {
$this->_cid = $cid;
}
if ($gid) {
$this->_gid = $gid;
}
$this->_rgid = $rgid;
}
$this->assign('action', $this->action);
$this->browse();
} else {
$this->action = CRM_Core_Action::UPDATE;
$this->edit($this->action);
$this->assign('action', $this->action);
}
$this->assign('context', $context);
// parent run
return parent::run();
}
/**
* Browse all rule groups.
*/
public function run()
{
$gid = CRM_Utils_Request::retrieve('gid', 'Positive', $this, FALSE, 0);
$action = CRM_Utils_Request::retrieve('action', 'String', $this, FALSE, 0);
$context = CRM_Utils_Request::retrieve('context', 'String', $this);
$limit = CRM_Utils_Request::retrieve('limit', 'Integer', $this);
$rgid = CRM_Utils_Request::retrieve('rgid', 'Positive');
$urlQry = "reset=1&rgid={$rgid}&gid={$gid}&limit={$limit}";
$this->assign('urlQuery', $urlQry);
$session = CRM_Core_Session::singleton();
$contactIds = $session->get('selectedSearchContactIds');
if ($context == 'search' || !empty($contactIds)) {
$context = 'search';
$this->assign('backURL', $session->readUserContext());
}
if ($action & CRM_Core_Action::RENEW) {
// empty cache
if ($rgid) {
CRM_Core_BAO_PrevNextCache::deleteItem(NULL, CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid));
}
CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry . "&action=update"));
} elseif ($action & CRM_Core_Action::MAP) {
// do a batch merge if requested
$result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', TRUE, 75);
$skippedCount = CRM_Utils_Request::retrieve('skipped', 'Positive', $this, FALSE, 0);
$skippedCount = $skippedCount + count($result['skipped']);
$mergedCount = CRM_Utils_Request::retrieve('merged', 'Positive', $this, FALSE, 0);
$mergedCount = $mergedCount + count($result['merged']);
if (empty($result['merged']) && empty($result['skipped'])) {
$message = '';
if ($mergedCount >= 1) {
$message = ts("%1 pairs of duplicates were merged", array(1 => $mergedCount));
}
if ($skippedCount >= 1) {
$message = $message ? "{$message} and " : '';
$message .= ts("%1 pairs of duplicates were skipped due to conflict", array(1 => $skippedCount));
}
$message .= ts(" during the batch merge process with safe mode.");
CRM_Core_Session::setStatus($message, ts('Merge Complete'), 'success');
CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry . "&action=update"));
} else {
$urlQry .= "&action=map&skipped={$skippedCount}&merged={$mergedCount}";
CRM_Utils_System::jsRedirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry), ts('Batch Merge Task in progress'), ts('The batch merge task is still in progress. This page will be refreshed automatically.'));
}
}
if ($action & CRM_Core_Action::UPDATE || $action & CRM_Core_Action::BROWSE) {
$cid = CRM_Utils_Request::retrieve('cid', 'Positive', $this, FALSE, 0);
$this->action = CRM_Core_Action::UPDATE;
$urlQry .= '&snippet=4';
if ($context == 'conflicts') {
$urlQry .= "&selected=1";
}
$this->assign('sourceUrl', CRM_Utils_System::url('civicrm/ajax/dedupefind', $urlQry, FALSE, NULL, FALSE));
//reload from cache table
$cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid);
$stats = CRM_Dedupe_Merger::getMergeStatsMsg($cacheKeyString);
if ($stats) {
CRM_Core_Session::setStatus($stats);
// reset so we not displaying same message again
CRM_Dedupe_Merger::resetMergeStats($cacheKeyString);
}
$join = CRM_Dedupe_Merger::getJoinOnDedupeTable();
$where = "de.id IS NULL";
if ($context == 'conflicts') {
$where .= " AND pn.is_selected = 1";
}
$this->_mainContacts = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where);
if (empty($this->_mainContacts)) {
if ($context == 'conflicts') {
// if the current screen was intended to list only selected contacts, move back to full dupe list
CRM_Utils_System::redirect(CRM_Utils_System::url(CRM_Utils_System::currentPath(), $urlQry . '&action=update'));
}
if ($gid) {
$foundDupes = $this->get("dedupe_dupes_{$gid}");
if (!$foundDupes) {
$foundDupes = CRM_Dedupe_Finder::dupesInGroup($rgid, $gid, $limit);
}
$this->set("dedupe_dupes_{$gid}", $foundDupes);
} elseif (!empty($contactIds)) {
$foundDupes = $this->get("search_dedupe_dupes_{$gid}");
if (!$foundDupes) {
$foundDupes = CRM_Dedupe_Finder::dupes($rgid, $contactIds);
}
$this->set("search_dedupe_dupes_{$gid}", $foundDupes);
} else {
$foundDupes = $this->get('dedupe_dupes');
if (!$foundDupes) {
$foundDupes = CRM_Dedupe_Finder::dupes($rgid, array(), TRUE, $limit);
}
$this->set('dedupe_dupes', $foundDupes);
}
if (!$foundDupes) {
$ruleGroup = new CRM_Dedupe_BAO_RuleGroup();
$ruleGroup->id = $rgid;
$ruleGroup->find(TRUE);
$session = CRM_Core_Session::singleton();
$session->setStatus(ts('No possible duplicates were found using %1 rule.', array(1 => $ruleGroup->name)), ts('None Found'), 'info');
$url = CRM_Utils_System::url('civicrm/contact/deduperules', 'reset=1');
if ($context == 'search') {
$url = $session->readUserContext();
}
CRM_Utils_System::redirect($url);
} else {
$mainContacts = CRM_Dedupe_Finder::parseAndStoreDupePairs($foundDupes, $cacheKeyString);
if ($cid) {
$this->_cid = $cid;
}
if ($gid) {
$this->_gid = $gid;
}
$this->_rgid = $rgid;
$this->_mainContacts = $mainContacts;
$session = CRM_Core_Session::singleton();
if ($this->_cid) {
$session->pushUserContext(CRM_Utils_System::url('civicrm/contact/deduperules', $urlQry . "&action=update&cid={$this->_cid}"));
} else {
$session->pushUserContext(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry . "&action=update"));
}
}
} else {
if ($cid) {
$this->_cid = $cid;
}
if ($gid) {
$this->_gid = $gid;
}
$this->_rgid = $rgid;
}
$this->assign('action', $this->action);
$this->browse();
} else {
$this->action = CRM_Core_Action::UPDATE;
$this->edit($this->action);
$this->assign('action', $this->action);
}
$this->assign('context', $context);
// parent run
return parent::run();
}