function newAntiDuplicate($l_countries, $max = 100)
{
$dbConf = new Config("/home/eyezeek/db.conf", "ini");
$optionsTemplate = array('connectTimeout' => 5, 'arrayResult' => true, 'matchMode' => SPH_MATCH_EXTENDED2, 'limits' => $max, 'page' => 1, 'index' => "i_ref_jobs i_delta_jobs", 'groupBy' => array('duplicates', SPH_GROUPBY_ATTR, '@count desc'));
foreach ($l_countries as $target) {
dumpMsg('Starting anti-dup ' . $target['reg']);
$goOnWithNextPage = TRUE;
$page = 1;
$conf = new Conftmp($target['port']);
$sph = new CSphinx($conf, $target['server']);
//var_dump($target);
// ids to keep
$keepIDs = array();
// crc to delete
$crcIDs = array(1);
// ids to set inactive = 0
$duplicateIDs = array(0);
dumpMsg('Grouping ... ');
while ($goOnWithNextPage) {
sleep(1);
$options = $optionsTemplate;
$options['page'] = $page;
$page++;
$sph->resetSphinx();
$sph->setAttribute('what', 'jobs');
$sortParam = 'alps_promoted desc, l_postdate desc';
$options['sortMode'] = array(SPH_SORT_EXTENDED, $sortParam);
$sph->setOptions($options);
$res = $sph->search();
$spx_ok = TRUE;
if (!$res) {
dumpError('Grouping: ' . $sph->getError());
die;
}
$relevantGroups = 0;
if (isset($res['matches'])) {
foreach ($res['matches'] as $oneMatch) {
// if this is > 1 then there are duplicates
$matchCounter = $oneMatch['attrs']['@count'];
if ($matchCounter > 1) {
$keepIDs[] = $oneMatch['id'];
$crcIDs[] = $oneMatch['attrs']['duplicates'];
$relevantGroups++;
} else {
$goOnWithNextPage = FALSE;
}
}
// extra condition to check if there is no more need to traverse next page
if ($relevantGroups < $max) {
$goOnWithNextPage = FALSE;
}
} else {
// no more matches i.e. nothing to do
$goOnWithNextPage = FALSE;
}
// for now simple put manual cut-off to avoid offset overflows
$goOnWithNextPage = FALSE;
}
dumpMsg('Grouped ' . ($page - 1) . ' pages and ' . count($keepIDs) . ' listings have at least 1 duplicate... ');
dumpMsg('Processing IDs ... ');
$goOnWithNextPage = TRUE;
$page = 1;
if (count($keepIDs) == 0) {
dumpMsg('Nothing to do. No duplicated were detected ');
dumpMsg('Finished anti-dup ' . $target['reg']);
return TRUE;
}
while ($goOnWithNextPage) {
// find actual duplicate ids
$options = $optionsTemplate;
$options['page'] = $page;
$page++;
$options['limits'] = 2000;
unset($options['groupBy']);
//var_dump($options);die;
$sph->resetSphinx();
$sph->setAttribute('what', 'jobs');
$sph->setOptions($options);
$sph->setFilter($sph->VALUE_FILTER, array('theid', $keepIDs, TRUE));
$sph->setFilter($sph->VALUE_FILTER, array('duplicates', $crcIDs, FALSE));
$res = $sph->search();
if (!$res) {
dumpError('Error in fetching duplicates: ' . $sph->getError());
die;
}
$resultArray = $sph->getIDs();
if (sizeof($resultArray) < $options['limits']) {
$goOnWithNextPage = FALSE;
}
$duplicateIDs = array_merge($duplicateIDs, $sph->getIDs());
// for now simple put manual cut-off to avoid offset overflows
$goOnWithNextPage = FALSE;
}
dumpMsg('Doing SQL updates and deactivating ' . sizeof($duplicateIDs) . ' duplicates ... ');
$lastDump = 0;
$maxDump = 200;
$tmpIDs = array();
$liveDB = new DBAdapter2(array('host' => $dbConf->live->host, 'username' => $dbConf->live->username, 'password' => $dbConf->live->password, 'schema' => $target['schemali']));
//var_dump($liveDB);
for ($i = 0; $i < sizeof($duplicateIDs); $i++) {
$tmpIDs[] = $duplicateIDs[$i];
$lastDump++;
if ($lastDump == $maxDump || $i + 1 == sizeof($duplicateIDs)) {
sleep(1);
$lastDump = 0;
$sql = "UPDATE jobs SET l_isActive=0 WHERE l_id IN (" . implode(",", $tmpIDs) . ")";
$tmpIDs = array();
try {
$affectedRows = 0;
$lid = 0;
$liveDB->executeNoresSQL($sql, $affectedRows, $lid);
} catch (DBAdapter2Exception $e) {
dumpError(" MYSQL Error: " . $e->getMessage() . " *** QUERY: {$sql}");
die;
}
}
}
unset($sph);
dumpMsg('Finished anti-dup ' . $target['reg']);
}
}
foreach ($validation['keys'] as $key) {
if (!$app->settings->contains($key)) {
dumpError("setting <em>{$key}</em> is missing from <em>workbench.ini</em>");
}
$path = $app->settings->read($key);
if (!call_user_func($validation['callable'], $path)) {
dumpError(sprintf($validation['message'], $path, $key));
}
}
}
/** Validate php settings */
foreach ($app->settings->readAll('php_value') as $name => $expectedValue) {
$name = substr($name, 10);
$currentValue = ini_get($name);
if ($expectedValue != $currentValue) {
dumpError("php setting <em>{$name}</em> not be set to <em>{$expectedValue}</em> (found: <em>{$currentValue}</em>)");
}
}
/** All OK */
echoHtmlHead();
echo "<h3 style=\"background-color:LimeGreen;padding:10px;\">All OK!</h3>";
echo '<p>This file (' . basename(__FILE__) . ') can be safely removed.</p>';
echo '<p><a href="/">Continue to app</a></p>';
function echoHtmlHead()
{
global $app;
$templates = $app->dependencies->get('templates');
echo $templates->render('head');
echo "<h3>Workbench setup</h3>";
}
function dumpError($error)
public static function dbCleanValue($sValue)
{
$mResult = @mysql_real_escape_string($sValue);
if ($mResult === false) {
$mResult = mysql_real_escape_string($sValue, self::getDBInstance());
if ($mResult === false) {
dumpError("Unable to escpae due to missing connection");
}
}
return $mResult;
}
