/** * Build a queue of tasks by dividing dupe pairs in batches. */ public static function getRunner() { $rgid = CRM_Utils_Request::retrieve('rgid', 'Positive'); $gid = CRM_Utils_Request::retrieve('gid', 'Positive'); $limit = CRM_Utils_Request::retrieve('limit', 'Positive'); $action = CRM_Utils_Request::retrieve('action', 'String', CRM_Core_DAO::$_nullObject); $mode = CRM_Utils_Request::retrieve('mode', 'String', CRM_Core_DAO::$_nullObject, FALSE, 'safe'); $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid); $urlQry = "reset=1&action=update&rgid={$rgid}&gid={$gid}&limit={$limit}"; if ($mode == 'aggressive' && !CRM_Core_Permission::check('force merge duplicate contacts')) { CRM_Core_Session::setStatus(ts('You do not have permission to force merge duplicate contact records'), ts('Permission Denied'), 'error'); CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry)); } // Setup the Queue $queue = CRM_Queue_Service::singleton()->create(array('name' => $cacheKeyString, 'type' => 'Sql', 'reset' => TRUE)); $where = NULL; if ($action == CRM_Core_Action::MAP) { $where = "pn.is_selected = 1"; $isSelected = 1; } else { // else merge all (2) $isSelected = 2; } $total = CRM_Core_BAO_PrevNextCache::getCount($cacheKeyString, NULL, $where); if ($total <= 0) { // Nothing to do. CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry)); } // reset merge stats, so we compute new stats CRM_Dedupe_Merger::resetMergeStats($cacheKeyString); for ($i = 1; $i <= ceil($total / self::BATCHLIMIT); $i++) { $task = new CRM_Queue_Task(array('CRM_Contact_Page_DedupeMerge', 'callBatchMerge'), array($rgid, $gid, $mode, FALSE, self::BATCHLIMIT, $isSelected), "Processed " . $i * self::BATCHLIMIT . " pair of duplicates out of " . $total); // Add the Task to the Queue $queue->createItem($task); } // Setup the Runner $urlQry .= "&context=conflicts"; $runner = new CRM_Queue_Runner(array('title' => ts('Merging Duplicates..'), 'queue' => $queue, 'errorMode' => CRM_Queue_Runner::ERROR_ABORT, 'onEndUrl' => CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry, TRUE, NULL, FALSE))); return $runner; }
/** * Mark dupe pairs as selected from un-selected state or vice-versa, in dupe cache table. */ public static function toggleDedupeSelect() { $rgid = CRM_Utils_Type::escape($_REQUEST['rgid'], 'Integer'); $gid = CRM_Utils_Type::escape($_REQUEST['gid'], 'Integer'); $pnid = $_REQUEST['pnid']; $isSelected = CRM_Utils_Type::escape($_REQUEST['is_selected'], 'Boolean'); $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid); $params = array(1 => array($isSelected, 'Boolean'), 3 => array("{$cacheKeyString}%", 'String')); //check pnid is_array or integer $whereClause = NULL; if (is_array($pnid) && !CRM_Utils_Array::crmIsEmptyArray($pnid)) { CRM_Utils_Type::escapeAll($pnid, 'Positive'); $pnid = implode(', ', $pnid); $whereClause = " id IN ( {$pnid} ) "; } else { $pnid = CRM_Utils_Type::escape($pnid, 'Integer'); $whereClause = " id = %2"; $params[2] = array($pnid, 'Integer'); } $sql = "UPDATE civicrm_prevnext_cache SET is_selected = %1 WHERE {$whereClause} AND cacheKey LIKE %3"; CRM_Core_DAO::executeQuery($sql, $params); CRM_Utils_System::civiExit(); }
public function postProcess() { $formValues = $this->exportValues(); // reset all selected contact ids from session // when we came from search context, CRM-3526 $session = CRM_Core_Session::singleton(); if ($session->get('selectedSearchContactIds')) { $session->resetScope('selectedSearchContactIds'); } $formValues['main_details'] = $this->_mainDetails; $formValues['other_details'] = $this->_otherDetails; $migrationData = array('migration_info' => $formValues); CRM_Utils_Hook::merge('form', $migrationData, $this->_cid, $this->_oid); CRM_Dedupe_Merger::moveAllBelongings($this->_cid, $this->_oid, $migrationData['migration_info']); $name = CRM_Core_DAO::getFieldValue('CRM_Contact_DAO_Contact', $this->_cid, 'display_name'); $message = '<ul><li>' . ts('%1 has been updated.', array(1 => $name)) . '</li><li>' . ts('Contact ID %1 has been deleted.', array(1 => $this->_oid)) . '</li></ul>'; CRM_Core_Session::setStatus($message, ts('Contacts Merged'), 'success'); $url = CRM_Utils_System::url('civicrm/contact/view', "reset=1&cid={$this->_cid}"); $urlParams = "reset=1&gid={$this->_gid}&rgid={$this->_rgid}&limit={$this->limit}"; if (!empty($formValues['_qf_Merge_submit'])) { $urlParams .= "&action=update"; $lisitingURL = CRM_Utils_System::url('civicrm/contact/dedupefind', $urlParams); CRM_Utils_System::redirect($lisitingURL); } if (!empty($formValues['_qf_Merge_done'])) { CRM_Utils_System::redirect($url); } if ($this->next && $this->_mergeId) { $cacheKey = CRM_Dedupe_Merger::getMergeCacheKeyString($this->_rgid, $this->_gid); $join = CRM_Dedupe_Merger::getJoinOnDedupeTable(); $where = "de.id IS NULL"; $pos = CRM_Core_BAO_PrevNextCache::getPositions($cacheKey, NULL, NULL, $this->_mergeId, $join, $where); if (!empty($pos) && $pos['next']['id1'] && $pos['next']['id2']) { $urlParams .= "&cid={$pos['next']['id1']}&oid={$pos['next']['id2']}&mergeId={$pos['next']['mergeId']}&action=update"; $url = CRM_Utils_System::url('civicrm/contact/merge', $urlParams); } } CRM_Utils_System::redirect($url); }
/** * Test the batch merge. */ public function testBatchMergeAllDuplicates() { $this->createDupeContacts(); // verify that all contacts have been created separately $this->assertEquals(count($this->_contactIds), 9, 'Check for number of contacts.'); $dao = new CRM_Dedupe_DAO_RuleGroup(); $dao->contact_type = 'Individual'; $dao->name = 'IndividualSupervised'; $dao->is_default = 1; $dao->find(TRUE); $foundDupes = CRM_Dedupe_Finder::dupesInGroup($dao->id, $this->_groupId); // ------------------------------------------------------------------------- // Name and Email (reserved) Matches ( 3 pairs ) // -------------------------------------------------------------------------- // robin - hood - robin@example.com // robin - hood - robin@example.com // little - dale - dale@example.com // little - dale - dale@example.com // will - dale - will@example.com // will - dale - will@example.com // so 3 pairs for - first + last + mail $this->assertEquals(count($foundDupes), 3, 'Check Individual-Supervised dupe rule for dupesInGroup().'); // Run dedupe finder as the browser would $_SERVER['REQUEST_METHOD'] = 'GET'; //avoid invalid key error $object = new CRM_Contact_Page_DedupeFind(); $object->set('gid', $this->_groupId); $object->set('rgid', $dao->id); $object->set('action', CRM_Core_Action::UPDATE); $object->setEmbedded(TRUE); @$object->run(); // Retrieve pairs from prev next cache table $select = array('pn.is_selected' => 'is_selected'); $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($dao->id, $this->_groupId); $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select); $this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.'); // batch merge all dupes $result = CRM_Dedupe_Merger::batchMerge($dao->id, $this->_groupId, 'safe', TRUE, 5, 2); $this->assertEquals(count($result['merged']), 3, 'Check number of merged pairs.'); // retrieve pairs from prev next cache table $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select); $this->assertEquals(count($pnDupePairs), 0, 'Check number of remaining dupe pairs in prev next cache.'); $this->deleteDupeContacts(); }
/** * Repopulate the cache of merge prospects. * * @param int $rgid * @param int $gid * @param NULL $cacheKeyString * @param array $criteria * Additional criteria to filter by. * * @param bool $checkPermissions * Respect logged in user's permissions. * * @return bool * @throws \CRM_Core_Exception * @throws \CiviCRM_API3_Exception */ public static function refillCache($rgid, $gid, $cacheKeyString, $criteria, $checkPermissions) { if (!$cacheKeyString && $rgid) { $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria, $checkPermissions); } if (!$cacheKeyString) { return FALSE; } // 1. Clear cache if any $sql = "DELETE FROM civicrm_prevnext_cache WHERE cacheKey LIKE %1"; CRM_Core_DAO::executeQuery($sql, array(1 => array("{$cacheKeyString}%", 'String'))); // FIXME: we need to start using temp tables / queries here instead of arrays. // And cleanup code in CRM/Contact/Page/DedupeFind.php // 2. FILL cache $foundDupes = array(); if ($rgid && $gid) { $foundDupes = CRM_Dedupe_Finder::dupesInGroup($rgid, $gid); } elseif ($rgid) { $contactIDs = array(); if (!empty($criteria)) { $contacts = civicrm_api3('Contact', 'get', array_merge(array('options' => array('limit' => 0), 'return' => 'id'), $criteria['contact'])); $contactIDs = array_keys($contacts['values']); } $foundDupes = CRM_Dedupe_Finder::dupes($rgid, $contactIDs, $checkPermissions); } if (!empty($foundDupes)) { CRM_Dedupe_Finder::parseAndStoreDupePairs($foundDupes, $cacheKeyString); } }
/** * Browse all rule groups. */ public function run() { $gid = CRM_Utils_Request::retrieve('gid', 'Positive', $this, FALSE, 0); $action = CRM_Utils_Request::retrieve('action', 'String', $this, FALSE, 0); $context = CRM_Utils_Request::retrieve('context', 'String', $this); $limit = CRM_Utils_Request::retrieve('limit', 'Integer', $this); $rgid = CRM_Utils_Request::retrieve('rgid', 'Positive'); $urlQry = "reset=1&rgid={$rgid}&gid={$gid}&limit={$limit}"; $this->assign('urlQuery', $urlQry); $session = CRM_Core_Session::singleton(); $contactIds = $session->get('selectedSearchContactIds'); if ($context == 'search' || !empty($contactIds)) { $context = 'search'; $this->assign('backURL', $session->readUserContext()); } if ($action & CRM_Core_Action::RENEW) { // empty cache if ($rgid) { CRM_Core_BAO_PrevNextCache::deleteItem(NULL, CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid)); } CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry . "&action=update")); } elseif ($action & CRM_Core_Action::MAP) { // do a batch merge if requested $result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', TRUE, 75); $skippedCount = CRM_Utils_Request::retrieve('skipped', 'Positive', $this, FALSE, 0); $skippedCount = $skippedCount + count($result['skipped']); $mergedCount = CRM_Utils_Request::retrieve('merged', 'Positive', $this, FALSE, 0); $mergedCount = $mergedCount + count($result['merged']); if (empty($result['merged']) && empty($result['skipped'])) { $message = ''; if ($mergedCount >= 1) { $message = ts("%1 pairs of duplicates were merged", array(1 => $mergedCount)); } if ($skippedCount >= 1) { $message = $message ? "{$message} and " : ''; $message .= ts("%1 pairs of duplicates were skipped due to conflict", array(1 => $skippedCount)); } $message .= ts(" during the batch merge process with safe mode."); CRM_Core_Session::setStatus($message, ts('Merge Complete'), 'success'); CRM_Utils_System::redirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry . "&action=update")); } else { $urlQry .= "&action=map&skipped={$skippedCount}&merged={$mergedCount}"; CRM_Utils_System::jsRedirect(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry), ts('Batch Merge Task in progress'), ts('The batch merge task is still in progress. This page will be refreshed automatically.')); } } if ($action & CRM_Core_Action::UPDATE || $action & CRM_Core_Action::BROWSE) { $cid = CRM_Utils_Request::retrieve('cid', 'Positive', $this, FALSE, 0); $this->action = CRM_Core_Action::UPDATE; $urlQry .= '&snippet=4'; if ($context == 'conflicts') { $urlQry .= "&selected=1"; } $this->assign('sourceUrl', CRM_Utils_System::url('civicrm/ajax/dedupefind', $urlQry, FALSE, NULL, FALSE)); //reload from cache table $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid); $stats = CRM_Dedupe_Merger::getMergeStatsMsg($cacheKeyString); if ($stats) { CRM_Core_Session::setStatus($stats); // reset so we not displaying same message again CRM_Dedupe_Merger::resetMergeStats($cacheKeyString); } $join = CRM_Dedupe_Merger::getJoinOnDedupeTable(); $where = "de.id IS NULL"; if ($context == 'conflicts') { $where .= " AND pn.is_selected = 1"; } $this->_mainContacts = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where); if (empty($this->_mainContacts)) { if ($context == 'conflicts') { // if the current screen was intended to list only selected contacts, move back to full dupe list CRM_Utils_System::redirect(CRM_Utils_System::url(CRM_Utils_System::currentPath(), $urlQry . '&action=update')); } if ($gid) { $foundDupes = $this->get("dedupe_dupes_{$gid}"); if (!$foundDupes) { $foundDupes = CRM_Dedupe_Finder::dupesInGroup($rgid, $gid, $limit); } $this->set("dedupe_dupes_{$gid}", $foundDupes); } elseif (!empty($contactIds)) { $foundDupes = $this->get("search_dedupe_dupes_{$gid}"); if (!$foundDupes) { $foundDupes = CRM_Dedupe_Finder::dupes($rgid, $contactIds); } $this->set("search_dedupe_dupes_{$gid}", $foundDupes); } else { $foundDupes = $this->get('dedupe_dupes'); if (!$foundDupes) { $foundDupes = CRM_Dedupe_Finder::dupes($rgid, array(), TRUE, $limit); } $this->set('dedupe_dupes', $foundDupes); } if (!$foundDupes) { $ruleGroup = new CRM_Dedupe_BAO_RuleGroup(); $ruleGroup->id = $rgid; $ruleGroup->find(TRUE); $session = CRM_Core_Session::singleton(); $session->setStatus(ts('No possible duplicates were found using %1 rule.', array(1 => $ruleGroup->name)), ts('None Found'), 'info'); $url = CRM_Utils_System::url('civicrm/contact/deduperules', 'reset=1'); if ($context == 'search') { $url = $session->readUserContext(); } CRM_Utils_System::redirect($url); } else { $mainContacts = CRM_Dedupe_Finder::parseAndStoreDupePairs($foundDupes, $cacheKeyString); if ($cid) { $this->_cid = $cid; } if ($gid) { $this->_gid = $gid; } $this->_rgid = $rgid; $this->_mainContacts = $mainContacts; $session = CRM_Core_Session::singleton(); if ($this->_cid) { $session->pushUserContext(CRM_Utils_System::url('civicrm/contact/deduperules', $urlQry . "&action=update&cid={$this->_cid}")); } else { $session->pushUserContext(CRM_Utils_System::url('civicrm/contact/dedupefind', $urlQry . "&action=update")); } } } else { if ($cid) { $this->_cid = $cid; } if ($gid) { $this->_gid = $gid; } $this->_rgid = $rgid; } $this->assign('action', $this->action); $this->browse(); } else { $this->action = CRM_Core_Action::UPDATE; $this->edit($this->action); $this->assign('action', $this->action); } $this->assign('context', $context); // parent run return parent::run(); }