public function testDupesByParams()
{
// make dupe checks based on based on following contact sets:
// FIRST - LAST - EMAIL
// ---------------------------------
// robin - hood - robin@example.com
// robin - hood - hood@example.com
// robin - dale - robin@example.com
// little - dale - dale@example.com
// will - dale - dale@example.com
// will - dale - will@example.com
// will - dale - will@example.com
// contact data set
// FIXME: move create params to separate function
$params = array(array('first_name' => 'robin', 'last_name' => 'hood', 'email' => '*****@*****.**', 'contact_type' => 'Individual'), array('first_name' => 'robin', 'last_name' => 'hood', 'email' => '*****@*****.**', 'contact_type' => 'Individual'), array('first_name' => 'robin', 'last_name' => 'dale', 'email' => '*****@*****.**', 'contact_type' => 'Individual'), array('first_name' => 'little', 'last_name' => 'dale', 'email' => '*****@*****.**', 'contact_type' => 'Individual'), array('first_name' => 'will', 'last_name' => 'dale', 'email' => '*****@*****.**', 'contact_type' => 'Individual'), array('first_name' => 'will', 'last_name' => 'dale', 'email' => '*****@*****.**', 'contact_type' => 'Individual'), array('first_name' => 'will', 'last_name' => 'dale', 'email' => '*****@*****.**', 'contact_type' => 'Individual'));
$count = 1;
foreach ($params as $param) {
$contact = $this->callAPISuccess('contact', 'create', $param);
$params = array('contact_id' => $contact['id'], 'street_address' => 'Ambachtstraat 23', 'location_type_id' => 1);
$this->callAPISuccess('address', 'create', $params);
$contactIds[$count++] = $contact['id'];
}
// verify that all contacts have been created separately
$this->assertEquals(count($contactIds), 7, 'Check for number of contacts.');
$dao = new CRM_Dedupe_DAO_RuleGroup();
$dao->contact_type = 'Individual';
$dao->used = 'General';
$dao->is_default = 1;
$dao->find(TRUE);
$fields = array('first_name' => 'robin', 'last_name' => 'hood', 'email' => '*****@*****.**', 'street_address' => 'Ambachtstraat 23');
CRM_Core_TemporaryErrorScope::useException();
$dedupeParams = CRM_Dedupe_Finder::formatParams($fields, 'Individual');
$ids = CRM_Dedupe_Finder::dupesByParams($dedupeParams, 'Individual', 'General');
// Check with default Individual-General rule
$this->assertEquals(count($ids), 2, 'Check Individual-General rule for dupesByParams().');
// delete all created contacts
foreach ($contactIds as $contactId) {
$this->contactDelete($contactId);
}
}
/**
* @param int $id
*/
public function delete($id)
{
$ruleDao = new CRM_Dedupe_DAO_Rule();
$ruleDao->dedupe_rule_group_id = $id;
$ruleDao->delete();
$rgDao = new CRM_Dedupe_DAO_RuleGroup();
$rgDao->id = $id;
$rgDao->delete();
}
/**
* Get an array of rule group id to rule group name
* for all th groups for that contactType. If contactType
* not specified, do it for all
*
* @param string $contactType
* Individual, Household or Organization.
*
*
* @return array
* id => "nice name" of rule group
*/
public static function getByType($contactType = NULL)
{
$dao = new CRM_Dedupe_DAO_RuleGroup();
if ($contactType) {
$dao->contact_type = $contactType;
}
$dao->find();
$result = array();
while ($dao->fetch()) {
$title = !empty($dao->title) ? $dao->title : (!empty($dao->name) ? $dao->name : $dao->contact_type);
$name = "{$title} - {$dao->used}";
$result[$dao->id] = $name;
}
return $result;
}
/**
* Process the form submission.
*
*
* @return void
*/
public function postProcess()
{
$values = $this->exportValues();
//FIXME: Handle logic to replace is_default column by usage
// reset used column to General (since there can only
// be one 'Supervised' or 'Unsupervised' rule)
if ($values['used'] != 'General') {
$query = "\nUPDATE civicrm_dedupe_rule_group\n SET used = 'General'\n WHERE contact_type = %1\n AND used = %2";
$queryParams = array(1 => array($this->_contactType, 'String'), 2 => array($values['used'], 'String'));
CRM_Core_DAO::executeQuery($query, $queryParams);
}
$rgDao = new CRM_Dedupe_DAO_RuleGroup();
if ($this->_action & CRM_Core_Action::UPDATE) {
$rgDao->id = $this->_rgid;
}
$rgDao->title = $values['title'];
$rgDao->is_reserved = CRM_Utils_Array::value('is_reserved', $values, FALSE);
$rgDao->used = $values['used'];
$rgDao->contact_type = $this->_contactType;
$rgDao->threshold = $values['threshold'];
$rgDao->save();
// make sure name is set only during insert
if ($this->_action & CRM_Core_Action::ADD) {
// generate name based on title
$rgDao->name = CRM_Utils_String::titleToVar($values['title']) . "_{$rgDao->id}";
$rgDao->save();
}
// lets skip updating of fields for reserved dedupe group
if ($rgDao->is_reserved) {
CRM_Core_Session::setStatus(ts("The rule '%1' has been saved.", array(1 => $rgDao->title)), ts('Saved'), 'success');
return;
}
$ruleDao = new CRM_Dedupe_DAO_Rule();
$ruleDao->dedupe_rule_group_id = $rgDao->id;
$ruleDao->delete();
$ruleDao->free();
$substrLenghts = array();
$tables = array();
$daoObj = new CRM_Core_DAO();
$database = $daoObj->database();
for ($count = 0; $count < self::RULES_COUNT; $count++) {
if (empty($values["where_{$count}"])) {
continue;
}
list($table, $field) = explode('.', CRM_Utils_Array::value("where_{$count}", $values));
$length = !empty($values["length_{$count}"]) ? CRM_Utils_Array::value("length_{$count}", $values) : NULL;
$weight = $values["weight_{$count}"];
if ($table and $field) {
$ruleDao = new CRM_Dedupe_DAO_Rule();
$ruleDao->dedupe_rule_group_id = $rgDao->id;
$ruleDao->rule_table = $table;
$ruleDao->rule_field = $field;
$ruleDao->rule_length = $length;
$ruleDao->rule_weight = $weight;
$ruleDao->save();
$ruleDao->free();
if (!array_key_exists($table, $tables)) {
$tables[$table] = array();
}
$tables[$table][] = $field;
}
// CRM-6245: we must pass table/field/length triples to the createIndexes() call below
if ($length) {
if (!isset($substrLenghts[$table])) {
$substrLenghts[$table] = array();
}
//CRM-13417 to avoid fatal error "Incorrect prefix key; the used key part isn't a string, the used length is longer than the key part, or the storage engine doesn't support unique prefix keys, 1089"
$schemaQuery = "SELECT * FROM INFORMATION_SCHEMA.COLUMNS\n WHERE TABLE_SCHEMA = '{$database}' AND\n TABLE_NAME = '{$table}' AND COLUMN_NAME = '{$field}';";
$dao = CRM_Core_DAO::executeQuery($schemaQuery);
if ($dao->fetch()) {
// set the length to null for all the fields where prefix length is not supported. eg. int,tinyint,date,enum etc dataTypes.
if ($dao->COLUMN_NAME == $field && !in_array($dao->DATA_TYPE, array('char', 'varchar', 'binary', 'varbinary', 'text', 'blob'))) {
$length = NULL;
} elseif ($dao->COLUMN_NAME == $field && !empty($dao->CHARACTER_MAXIMUM_LENGTH) && $length > $dao->CHARACTER_MAXIMUM_LENGTH) {
//set the length to CHARACTER_MAXIMUM_LENGTH in case the length provided by the user is greater than the limit
$length = $dao->CHARACTER_MAXIMUM_LENGTH;
}
}
$substrLenghts[$table][$field] = $length;
}
}
// also create an index for this dedupe rule
// CRM-3837
CRM_Utils_Hook::dupeQuery($ruleDao, 'dedupeIndexes', $tables);
CRM_Core_BAO_SchemaHandler::createIndexes($tables, 'dedupe_index', $substrLenghts);
//need to clear cache of deduped contacts
//based on the previous rule
$cacheKey = "merge {$this->_contactType}_{$this->_rgid}_%";
CRM_Core_BAO_PrevNextCache::deleteItem(NULL, $cacheKey);
CRM_Core_Session::setStatus(ts("The rule '%1' has been saved.", array(1 => $rgDao->title)), ts('Saved'), 'success');
}
/**
* Browse all rule groups
*
* @return void
* @access public
*/
function browse()
{
// get all rule groups
$ruleGroups = array();
$dao =& new CRM_Dedupe_DAO_RuleGroup();
$dao->orderBy('contact_type,level,is_default DESC');
$dao->find();
while ($dao->fetch()) {
$ruleGroups[$dao->id] = array();
CRM_Core_DAO::storeValues($dao, $ruleGroups[$dao->id]);
// form all action links
$action = array_sum(array_keys($this->links()));
$links = self::links();
if ($dao->is_default) {
unset($links[CRM_Core_Action::MAP]);
unset($links[CRM_Core_Action::DELETE]);
}
$ruleGroups[$dao->id]['action'] = CRM_Core_Action::formLink($links, $action, array('id' => $dao->id));
CRM_Dedupe_DAO_RuleGroup::addDisplayEnums($ruleGroups[$dao->id]);
}
$this->assign('rows', $ruleGroups);
}
public function testBatchMergeAllDuplicates()
{
$this->createDupeContacts();
// verify that all contacts have been created separately
$this->assertEquals(count($this->_contactIds), 9, 'Check for number of contacts.');
$dao = new CRM_Dedupe_DAO_RuleGroup();
$dao->contact_type = 'Individual';
$dao->name = 'IndividualSupervised';
$dao->is_default = 1;
$dao->find(TRUE);
$foundDupes = CRM_Dedupe_Finder::dupesInGroup($dao->id, $this->_groupId);
// -------------------------------------------------------------------------
// Name and Email (reserved) Matches ( 3 pairs )
// --------------------------------------------------------------------------
// robin - hood - robin@example.com
// robin - hood - robin@example.com
// little - dale - dale@example.com
// little - dale - dale@example.com
// will - dale - will@example.com
// will - dale - will@example.com
// so 3 pairs for - first + last + mail
$this->assertEquals(count($foundDupes), 3, 'Check Individual-Supervised dupe rule for dupesInGroup().');
// Run dedupe finder as the browser would
$_SERVER['REQUEST_METHOD'] = 'GET';
//avoid invalid key error
$object = new CRM_Contact_Page_DedupeFind();
$object->set('gid', $this->_groupId);
$object->set('rgid', $dao->id);
$object->set('action', CRM_Core_Action::UPDATE);
@$object->run();
// Retrieve pairs from prev next cache table
$select = array('pn.is_selected' => 'is_selected');
$cacheKeyString = "merge Individual_{$dao->id}_{$this->_groupId}";
$pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select);
$this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.');
// batch merge all dupes
$result = CRM_Dedupe_Merger::batchMerge($dao->id, $this->_groupId, 'safe', TRUE, 5, 2);
$this->assertEquals(count($result['merged']), 3, 'Check number of merged pairs.');
// retrieve pairs from prev next cache table
$pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select);
$this->assertEquals(count($pnDupePairs), 0, 'Check number of remaining dupe pairs in prev next cache.');
$this->deleteDupeContacts();
}
/**
* Get an array of rule group id to rule group name
* for all th groups for that contactType. If contactType
* not specified, do it for all
*
* @param string $contactType Individual, Household or Organization
*
* @static
*
* @return array id => "nice name" of rule group
*/
static function getByType($contactType = NULL)
{
$dao = new CRM_Dedupe_DAO_RuleGroup();
if ($contactType) {
$dao->contact_type = $contactType;
}
$dao->find();
$result = array();
while ($dao->fetch()) {
if (!empty($dao->name)) {
$name = "{$dao->name} - {$dao->level}";
} else {
$name = "{$dao->contact_type} - {$dao->level}";
}
$result[$dao->id] = $name;
}
return $result;
}
/**
* adds $value['foo_display'] for each $value['foo'] enum from civicrm_dedupe_rule_group
*
* @param array $values (reference) the array up for enhancing
* @return void
*/
static function addDisplayEnums(&$values)
{
$enumFields =& CRM_Dedupe_DAO_RuleGroup::getEnums();
foreach ($enumFields as $enum) {
if (isset($values[$enum])) {
$values[$enum . '_display'] = CRM_Dedupe_DAO_RuleGroup::tsEnum($enum, $values[$enum]);
}
}
}
/**
* build all the data structures needed to build the form
*
* @return void
* @access public
*/
function preProcess()
{
parent::preProcess();
$statusMsg = null;
$contactIds = array();
if (is_array($this->_contactIds)) {
$contactIds = array_unique($this->_contactIds);
}
if (count($contactIds) < 2) {
$statusMsg = ts('Minimum two contact records are required to perform merge operation.');
}
// do check for same contact type.
$contactTypes = array();
if (!$statusMsg) {
$sql = "SELECT contact_type FROM civicrm_contact WHERE id IN (" . implode(',', $contactIds) . ")";
$contact = CRM_Core_DAO::executeQuery($sql);
while ($contact->fetch()) {
$contactTypes[$contact->contact_type] = true;
if (count($contactTypes) > 1) {
break;
}
}
if (count($contactTypes) > 1) {
$statusMsg = ts('Please select same contact type records.');
}
}
if ($statusMsg) {
CRM_Core_Error::statusBounce($statusMsg);
}
$url = null;
// redirect to merge form directly.
if (count($contactIds) == 2) {
$cid = $contactIds[0];
$oid = $contactIds[1];
//don't allow to delete logged in user.
$session = CRM_Core_Session::singleton();
if ($oid == $session->get('userID')) {
$oid = $cid;
$cid = $session->get('userID');
}
$url = CRM_Utils_System::url('civicrm/contact/merge', "reset=1&cid={$cid}&oid={$oid}");
} else {
$level = 'Fuzzy';
$cType = key($contactTypes);
require_once 'CRM/Dedupe/DAO/RuleGroup.php';
$rgBao = new CRM_Dedupe_DAO_RuleGroup();
$rgBao->level = $level;
$rgBao->is_default = 1;
$rgBao->contact_type = $cType;
if (!$rgBao->find(true)) {
CRM_Core_Error::statusBounce("You can not merge contact records because {$level} rule for {$cType} does not exist.");
}
$ruleGroupID = $rgBao->id;
$session = CRM_Core_Session::singleton();
$session->set('selectedSearchContactIds', $contactIds);
// create a hidden group and poceed to merge
$url = CRM_Utils_System::url('civicrm/contact/dedupefind', "reset=1&action=update&rgid={$ruleGroupID}&context=search");
}
// redirect to merge page.
if ($url) {
CRM_Utils_System::redirect($url);
}
}
/**
* Function to process the form
*
* @access public
* @return None
*/
public function postProcess()
{
$values = $this->exportValues();
$isDefault = CRM_Utils_Array::value('is_default', $values, false);
// reset defaults
if ($isDefault) {
$query = "\nUPDATE civicrm_dedupe_rule_group \n SET is_default = 0\n WHERE contact_type = %1 \n AND level = %2";
$queryParams = array(1 => array($this->_contactType, 'String'), 2 => array($values['level'], 'String'));
CRM_Core_DAO::executeQuery($query, $queryParams);
}
$rgDao = new CRM_Dedupe_DAO_RuleGroup();
if ($this->_action & CRM_Core_Action::UPDATE) {
$rgDao->id = $this->_rgid;
}
$rgDao->threshold = $values['threshold'];
$rgDao->name = $values['name'];
$rgDao->level = $values['level'];
$rgDao->contact_type = $this->_contactType;
$rgDao->is_default = $isDefault;
$rgDao->save();
$ruleDao = new CRM_Dedupe_DAO_Rule();
$ruleDao->dedupe_rule_group_id = $rgDao->id;
$ruleDao->delete();
$ruleDao->free();
$substrLenghts = array();
$tables = array();
for ($count = 0; $count < self::RULES_COUNT; $count++) {
if (!CRM_Utils_Array::value("where_{$count}", $values)) {
continue;
}
list($table, $field) = explode('.', CRM_Utils_Array::value("where_{$count}", $values));
$length = CRM_Utils_Array::value("length_{$count}", $values) ? CRM_Utils_Array::value("length_{$count}", $values) : null;
$weight = $values["weight_{$count}"];
if ($table and $field) {
$ruleDao = new CRM_Dedupe_DAO_Rule();
$ruleDao->dedupe_rule_group_id = $rgDao->id;
$ruleDao->rule_table = $table;
$ruleDao->rule_field = $field;
$ruleDao->rule_length = $length;
$ruleDao->rule_weight = $weight;
$ruleDao->save();
$ruleDao->free();
if (!array_key_exists($table, $tables)) {
$tables[$table] = array();
}
$tables[$table][] = $field;
}
// CRM-6245: we must pass table/field/length triples to the createIndexes() call below
if ($length) {
if (!isset($substrLenghts[$table])) {
$substrLenghts[$table] = array();
}
$substrLenghts[$table][$field] = $length;
}
}
// also create an index for this dedupe rule
// CRM-3837
require_once 'CRM/Core/BAO/SchemaHandler.php';
CRM_Core_BAO_SchemaHandler::createIndexes($tables, 'dedupe_index', $substrLenghts);
}
/**
* Returns the list of fields that can be exported
*
* @param bool $prefix
*
* @return array
*/
static function &export($prefix = false)
{
if (!self::$_export) {
self::$_export = array();
$fields = self::fields();
foreach ($fields as $name => $field) {
if (CRM_Utils_Array::value('export', $field)) {
if ($prefix) {
self::$_export['dedupe_rule_group'] =& $fields[$name];
} else {
self::$_export[$name] =& $fields[$name];
}
}
}
}
return self::$_export;
}
/**
* Function to process the form
*
* @access public
*
* @return None
*/
public function postProcess()
{
$values = $this->exportValues();
$isDefault = CRM_Utils_Array::value('is_default', $values, FALSE);
// reset defaults
if ($isDefault) {
$query = "\nUPDATE civicrm_dedupe_rule_group \n SET is_default = 0\n WHERE contact_type = %1 \n AND level = %2";
$queryParams = array(1 => array($this->_contactType, 'String'), 2 => array($values['level'], 'String'));
CRM_Core_DAO::executeQuery($query, $queryParams);
}
$rgDao = new CRM_Dedupe_DAO_RuleGroup();
if ($this->_action & CRM_Core_Action::UPDATE) {
$rgDao->id = $this->_rgid;
}
$rgDao->title = $values['title'];
$rgDao->is_reserved = CRM_Utils_Array::value('is_reserved', $values, FALSE);
$rgDao->is_default = $isDefault;
$rgDao->level = $values['level'];
$rgDao->contact_type = $this->_contactType;
$rgDao->threshold = $values['threshold'];
$rgDao->save();
// make sure name is set only during insert
if ($this->_action & CRM_Core_Action::ADD) {
// generate name based on title
$rgDao->name = CRM_Utils_String::titleToVar($values['title']) . "_{$rgDao->id}";
$rgDao->save();
}
// lets skip updating of fields for reserved dedupe group
if ($rgDao->is_reserved) {
CRM_Core_Session::setStatus(ts('The rule \'%1\' has been saved.', array(1 => $rgDao->title)));
return;
}
$ruleDao = new CRM_Dedupe_DAO_Rule();
$ruleDao->dedupe_rule_group_id = $rgDao->id;
$ruleDao->delete();
$ruleDao->free();
$substrLenghts = array();
$tables = array();
for ($count = 0; $count < self::RULES_COUNT; $count++) {
if (!CRM_Utils_Array::value("where_{$count}", $values)) {
continue;
}
list($table, $field) = explode('.', CRM_Utils_Array::value("where_{$count}", $values));
$length = CRM_Utils_Array::value("length_{$count}", $values) ? CRM_Utils_Array::value("length_{$count}", $values) : NULL;
$weight = $values["weight_{$count}"];
if ($table and $field) {
$ruleDao = new CRM_Dedupe_DAO_Rule();
$ruleDao->dedupe_rule_group_id = $rgDao->id;
$ruleDao->rule_table = $table;
$ruleDao->rule_field = $field;
$ruleDao->rule_length = $length;
$ruleDao->rule_weight = $weight;
$ruleDao->save();
$ruleDao->free();
if (!array_key_exists($table, $tables)) {
$tables[$table] = array();
}
$tables[$table][] = $field;
}
// CRM-6245: we must pass table/field/length triples to the createIndexes() call below
if ($length) {
if (!isset($substrLenghts[$table])) {
$substrLenghts[$table] = array();
}
$substrLenghts[$table][$field] = $length;
}
}
// also create an index for this dedupe rule
// CRM-3837
CRM_Core_BAO_SchemaHandler::createIndexes($tables, 'dedupe_index', $substrLenghts);
//need to clear cache of deduped contacts
//based on the previous rule
$cacheKey = "merge {$this->_contactType}_{$this->_rgid}_%";
CRM_Core_BAO_PrevNextCache::deleteItem(NULL, $cacheKey);
CRM_Core_Session::setStatus(ts('The rule \'%1\' has been saved.', array(1 => $rgDao->title)));
}