}
}
$mysqli = new mysqli("192.168.240.15", "eyezeek", "p!n6access", 'ru_eyezeek');
/* check connection */
if ($mysqli->connect_errno) {
print "Connect failed: " . $mysqli->connect_error . "\n";
exit;
}
/* set character set */
if (!$mysqli->set_charset("utf8")) {
print "Error loading character set utf8: " . $mysqli->error;
exit;
}
$options = array('connectTimeout' => 5, 'arrayResult' => true, 'matchMode' => SPH_MATCH_EXTENDED2, 'limits' => 1, 'page' => 1, 'index' => "i_ref_vehicles i_delta_vehicles");
$conf = new Conftmp('3313');
$sph = new CSphinx($conf, '192.168.240.45');
print_text("Start");
$sql = "\n\t\tselect \n\t\t\tgroup_concat(p.id) as stat_ids,\n\t\t\tconcat(\n\t\t\t\t'http://pingola.ru/auto/search/all/',\n\t\t\t\tmake,\n\t\t\t\t'/',\n\t\t\t\tmodel,\n\t\t\t\t'/?year=',\n\t\t\t\tyear,'-',year\n\t\t\t) as url,\n\t\t\tmake as brand,model as series,year as year,sum(items_counted) as count,avg(avg_price) as avgPrice\n\t\tfrom ru_eyezeek.np_market_trends_vehicles_prices p join ru_eyezeek.np_market_trends_vehicles_items i on i.id=p.item_id\n\t\twhere `month`='" . date('Y-m-00', strtotime('-30 day')) . "' and items_counted>10\n\t\tgroup by make,model,year";
print_text("Fetching all relevant items");
if ($result = $mysqli->query($sql)) {
$writer = new XMLWriter();
$writer->openURI('/home/eyezeek/netup/ru/avtology/avtologystat.xml');
$writer->startDocument('1.0', 'UTF-8');
$writer->setIndent(TRUE);
$writer->startElement('report');
$writer->writeElement('creationDate', date("Y-m-d H:i:s", time()));
$writer->startElement('vehicles');
while ($arr = $result->fetch_assoc()) {
$writer->startElement('vehicle');
$stat_ids = $arr['stat_ids'];
unset($arr['stat_ids']);
$_passw = 'p!n6access';
$_db = "il_eyezeek";
PrintText("Connecting to DB...");
$link = mysql_connect($_host, $_login, $_passw);
if (!$link) {
die("\nCould not connect: " . mysql_error());
}
PrintText("Connected successfully");
mysqli_set_charset('utf8');
mysqli_select_db($_db);
PrintText("");
/*
2. Sphinx object creation
*/
$dbConf = new Config("/home/eyezeek/db_jobs_il.conf", "ini");
$sph = new CSphinx($dbConf->sphinx);
$ids = array();
/*
3. Find listings to update
*/
$SQL = "SELECT *\n\t\t\t\tFROM jobs_position2category\n\t\t\t\tORDER BY category_id";
$RS = mysql_query($SQL);
$categories_titles = array();
while ($ROW = mysql_fetch_assoc($RS)) {
// echo "\n".$ROW['category_id'].": ".$ROW['job_title'];
$categories_titles[$ROW['category_id']][] = $ROW['job_title'];
}
// var_dump($categories_titles);
// die();
$sph->resetSphinx();
foreach ($categories_titles as $cat_id => $category) {
class sph_conf
{
public $port;
public function __construct($port)
{
$this->port = $port;
}
}
$l_aTopRegions = array(array("name" => "Москва (обл.)", "icon" => "msk.png", "lat" => "55.632139", "lng" => "37.673753", "radius" => "218718", "count" => "0", "trnaslit" => "moskovskaya-oblast"), array("name" => "Томская область", "icon" => "tms.png", "lat" => "58.452311", "lng" => "82.217785", "radius" => "511810", "count" => "0", "trnaslit" => "tomskaya-oblast"), array("name" => "Алтайский край", "icon" => "ao.png", "lat" => "52.588139", "lng" => "82.530013", "radius" => "378222", "count" => "0", "trnaslit" => "altayskiy-kray"), array("name" => "Республика Коми", "icon" => "km.png", "lat" => "64.193025", "lng" => "55.826782", "radius" => "717622", "count" => "0", "trnaslit" => "respublika-komi"), array("name" => "Приморский край", "icon" => "vl.png", "lat" => "45.463955", "lng" => "134.713273", "radius" => "480605", "count" => "0", "trnaslit" => "primorskiy-kray"), array("name" => "Санкт-Петербург (обл.)", "icon" => "spb.png", "lat" => "59.906021", "lng" => "31.273652", "radius" => "307560", "count" => "0", "trnaslit" => "leningradskaya-oblast"), array("name" => "Кемеровская область", "icon" => "kmr.png", "lat" => "54.565874", "lng" => "86.927482", "radius" => "304813", "count" => "0", "trnaslit" => "kemerovskaya-oblast"), array("name" => "Республика Алтай", "icon" => "ar.png", "lat" => "50.900103", "lng" => "86.895789", "radius" => "288389", "count" => "0", "trnaslit" => "respublika-altay"), array("name" => "Челябинская область", "icon" => "chl.png", "lat" => "54.236353", "lng" => "60.241643", "radius" => "316202", "count" => "0", "trnaslit" => "chelyabinskaya-oblast"), array("name" => "Новосибирская область", "icon" => "nvs.png", "lat" => "55.312179", "lng" => "80.102208", "radius" => "385429", "count" => "0", "trnaslit" => "novosibirskaya-oblast"), array("name" => "Свердловская область", "icon" => "svr.png", "lat" => "59.126311", "lng" => "61.706319", "radius" => "415412", "count" => "0", "trnaslit" => "sverdlovskaya-oblast"), array("name" => "Ханты-Мансийский автономный округ - Югра", "icon" => "hm.png", "lat" => "62.36118", "lng" => "72.586768", "radius" => "791619", "count" => "0", "trnaslit" => "hanti-mansiyskiy-avtonomniy-okrug---yugra"), array("name" => "Ямало-Ненецкий автономный округ", "icon" => "yam.png", "lat" => "68.575798", "lng" => "74.4469", "radius" => "794010", "count" => "0", "trnaslit" => "yamalo-nenetskiy-avtonomniy-okrug"), array("name" => "Республика Бурятия", "icon" => "bur.png", "lat" => "53.659739", "lng" => "108.261679", "radius" => "723306", "count" => "0", "trnaslit" => "respublika-buryatiya"), array("name" => "Краснодарский край", "icon" => "krd.png", "lat" => "45.160042", "lng" => "39.170553", "radius" => "280074", "count" => "0", "trnaslit" => "krasnodarskiy-kray"), array("name" => "Иркутская область", "icon" => "irk.png", "lat" => "55.436642", "lng" => "102.455823", "radius" => "625258", "count" => "0", "trnaslit" => "irkutskaya-oblast"), array("name" => "Тюменская область", "icon" => "tmn.png", "lat" => "57.649009", "lng" => "70.013552", "radius" => "409270", "count" => "0", "trnaslit" => "tyumenskaya-oblast"), array("name" => "Омская область", "icon" => "oms.png", "lat" => "56.091426", "lng" => "73.331606", "radius" => "340131", "count" => "0", "trnaslit" => "omskaya-oblast"), array("name" => "Республика Татарстан", "icon" => "ttn.png", "lat" => "55.348304", "lng" => "50.763132", "radius" => "267614", "count" => "0", "trnaslit" => "respublika-tatarstan"), array("name" => "Республика Башкортостан", "icon" => "bas.png", "lat" => "54.127354", "lng" => "56.579526", "radius" => "354607", "count" => "0", "trnaslit" => "respublika-bashkortostan"), array("name" => "Оренбургская область", "icon" => "orb.png", "lat" => "52.472554", "lng" => "56.229273", "radius" => "427566", "count" => "0", "trnaslit" => "orenburgskaya-oblast"), array("name" => "Пермский край", "icon" => "prm.png", "lat" => "58.996154", "lng" => "55.627994", "radius" => "379658", "count" => "0", "trnaslit" => "permskiy-kray"), array("name" => "Курганская область", "icon" => "krg.png", "lat" => "55.537704", "lng" => "65.34534", "radius" => "258742", "count" => "0", "trnaslit" => "kurganskaya-oblast"), array("name" => "Самарская область", "icon" => "smr.png", "lat" => "53.248357", "lng" => "50.247481", "radius" => "223310", "count" => "0", "trnaslit" => "samarskaya-oblast"), array("name" => "Республика Адыгея", "icon" => "adg.png", "lat" => "44.486278", "lng" => "39.881489", "radius" => "115815", "count" => "0", "trnaslit" => "respublika-adigeya"), array("name" => "Красноярский край", "icon" => "ksn.png", "lat" => "55.007731", "lng" => "94.07049", "radius" => "495621", "count" => "0", "trnaslit" => "krasnoyarskiy-kray"), array("name" => "Хабаровский край", "icon" => "chb.png", "lat" => "51.191732", "lng" => "135.966288", "radius" => "619953", "count" => "0", "trnaslit" => "habarovskiy-kray"), array("name" => "Республика Тыва", "icon" => "tva.png", "lat" => "51.780788", "lng" => "94.032715", "radius" => "422879", "count" => "0", "trnaslit" => "respublika-tiva"), array("name" => "Саратовская область", "icon" => "srt.png", "lat" => "51.334903", "lng" => "46.669168", "radius" => "334233", "count" => "0", "trnaslit" => "saratovskaya-oblast"), array("name" => "Кировская область", "icon" => "krv.png", "lat" => "58.651988", "lng" => "50.097435", "radius" => "355802", "count" => "0", "trnaslit" => "kirovskaya-oblast"), array("name" => "Ростовская область", "icon" => "ros.png", "lat" => "48.127307", "lng" => "41.273491", "radius" => "327762", "count" => "0", "trnaslit" => "rostovskaya-oblast"), array("name" => "Ульяновская область", "icon" => "ul.png", "lat" => "53.728077", "lng" => "48.020045", "radius" => "196369", "count" => "0", "trnaslit" => "ulyanovskaya-oblast"), array("name" => "Вологодская область", "icon" => "vlg.png", "lat" => "60.078181", "lng" => "40.938725", "radius" => "385719", "count" => "0", "trnaslit" => "vologodskaya-oblast"), array("name" => "Волгоградская область", "icon" => "vgd.png", "lat" => "49.383009", "lng" => "44.281895", "radius" => "310930", "count" => "0", "trnaslit" => "volgogradskaya-oblast"), array("name" => "Амурская область", "icon" => "am.png", "lat" => "53.152586", "lng" => "127.288284", "radius" => "682574", "count" => "0", "trnaslit" => "amurskaya-oblast"), array("name" => "Нижегородская область", "icon" => "ngr.png", "lat" => "56.319899", "lng" => "44.773929", "radius" => "273567", "count" => "0", "trnaslit" => "nizhegorodskaya-oblast"), array("name" => "Республика Мордовия", "icon" => "mrd.png", "lat" => "54.424471", "lng" => "44.447365", "radius" => "170864", "count" => "0", "trnaslit" => "respublika-mordoviya"), array("name" => "Забайкальский край", "icon" => "zbl.png", "lat" => "54.029635", "lng" => "114.990357", "radius" => "697408", "count" => "0", "trnaslit" => "zabaykalskiy-kray"), array("name" => "Республика Калмыкия", "icon" => "klm.png", "lat" => "46.544747", "lng" => "44.619392", "radius" => "300276", "count" => "0", "trnaslit" => "respublika-kalmikiya"), array("name" => "Республика Марий Эл", "icon" => "mel.png", "lat" => "56.593423", "lng" => "47.914682", "radius" => "163428", "count" => "0", "trnaslit" => "respublika-mariy-el"), array("name" => "Республика Хакасия", "icon" => "hks.png", "lat" => "53.411623", "lng" => "89.896333", "radius" => "267543", "count" => "0", "trnaslit" => "respublika-hakasiya"), array("name" => "Владимирская область", "icon" => "vld.png", "lat" => "55.966907", "lng" => "40.624898", "radius" => "174193", "count" => "0", "trnaslit" => "vladimirskaya-oblast"), array("name" => "Костромская область", "icon" => "kst.png", "lat" => "58.426484", "lng" => "43.821472", "radius" => "260550", "count" => "0", "trnaslit" => "kostromskaya-oblast"), array("name" => "Ярославская область", "icon" => "yar.png", "lat" => "57.763852", "lng" => "39.26175", "radius" => "176638", "count" => "0", "trnaslit" => "yaroslavskaya-oblast"), array("name" => "Воронежская область", "icon" => "vrg.png", "lat" => "50.847334", "lng" => "40.542703", "radius" => "220262", "count" => "0", "trnaslit" => "voronezhskaya-oblast"), array("name" => "Удмуртская Республика", "icon" => "udm.png", "lat" => "57.225685", "lng" => "52.779499", "radius" => "179671", "count" => "0", "trnaslit" => "udmurtskaya-respublika"), array("name" => "Ставропольский край", "icon" => "stv.png", "lat" => "44.962334", "lng" => "43.280426", "radius" => "239321", "count" => "0", "trnaslit" => "stavropolskiy-kray"), array("name" => "Мурманская область", "icon" => "mrm.png", "lat" => "68.086178", "lng" => "34.911613", "radius" => "345453", "count" => "0", "trnaslit" => "murmanskaya-oblast"), array("name" => "Курская область", "icon" => "krs.png", "lat" => "51.680816", "lng" => "36.299924", "radius" => "175284", "count" => "0", "trnaslit" => "kurskaya-oblast"), array("name" => "Калининградская область", "icon" => "kln.png", "lat" => "54.809282", "lng" => "21.26324", "radius" => "117371", "count" => "0", "trnaslit" => "kaliningradskaya-oblast"), array("name" => "Рязанская область", "icon" => "ryz.png", "lat" => "54.348837", "lng" => "40.675411", "radius" => "173572", "count" => "0", "trnaslit" => "ryazanskaya-oblast"), array("name" => "Липецкая область", "icon" => "li.png", "lat" => "52.745899", "lng" => "39.242903", "radius" => "139334", "count" => "0", "trnaslit" => "lipetskaya-oblast"), array("name" => "Белгородская область", "icon" => "bel.png", "lat" => "50.620401", "lng" => "37.303521", "radius" => "166447", "count" => "0", "trnaslit" => "belgorodskaya-oblast"), array("name" => "Камчатский край", "icon" => "kmc.png", "lat" => "58.865297", "lng" => "164.626499", "radius" => "954321", "count" => "0", "trnaslit" => "kamchatskiy-kray"), array("name" => "Ивановская область", "icon" => "ivn.png", "lat" => "57.051228", "lng" => "41.350504", "radius" => "142173", "count" => "0", "trnaslit" => "ivanovskaya-oblast"), array("name" => "Калужская область", "icon" => "klg.png", "lat" => "54.322405", "lng" => "35.359666", "radius" => "169180", "count" => "0", "trnaslit" => "kaluzhskaya-oblast"), array("name" => "Пензенская область", "icon" => "pnz.png", "lat" => "53.169506", "lng" => "44.531681", "radius" => "189687", "count" => "0", "trnaslit" => "penzenskaya-oblast"), array("name" => "Тверская область", "icon" => "tvr.png", "lat" => "57.282926", "lng" => "34.548838", "radius" => "289442", "count" => "0", "trnaslit" => "tverskaya-oblast"), array("name" => "Смоленская область", "icon" => "sml.png", "lat" => "54.761748", "lng" => "33.069752", "radius" => "209603", "count" => "0", "trnaslit" => "smolenskaya-oblast"), array("name" => "Орловская область", "icon" => "orl.png", "lat" => "52.801729", "lng" => "36.43", "radius" => "145633", "count" => "0", "trnaslit" => "orlovskaya-oblast"), array("name" => "Тульская область", "icon" => "tul.png", "lat" => "53.910034", "lng" => "37.424659", "radius" => "145704", "count" => "0", "trnaslit" => "tulskaya-oblast"), array("name" => "Астраханская область", "icon" => "ash.png", "lat" => "47.12923", "lng" => "47.277417", "radius" => "262454", "count" => "0", "trnaslit" => "astrahanskaya-oblast"), array("name" => "Республика Саха (Якутия)", "icon" => "yak.png", "lat" => "59.266015", "lng" => "127.501086", "radius" => "445948", "count" => "0", "trnaslit" => "respublika-saha-yakutiya"), array("name" => "Республика Карелия", "icon" => "krl.png", "lat" => "63.833338", "lng" => "33.627004", "radius" => "394480", "count" => "0", "trnaslit" => "respublika-kareliya"), array("name" => "Карачаево-Черкесская Республика", "icon" => "kch.png", "lat" => "43.847074", "lng" => "41.678218", "radius" => "108532", "count" => "0", "trnaslit" => "karachaevo-cherkesskaya-respublika"), array("name" => "Чувашская Республика", "icon" => "chv.png", "lat" => "55.488126", "lng" => "47.165918", "radius" => "123151", "count" => "0", "trnaslit" => "chuvashskaya-respublika"), array("name" => "Кабардино-Балкарская Республика", "icon" => "kbr.png", "lat" => "43.457316", "lng" => "43.436805", "radius" => "104827", "count" => "0", "trnaslit" => "kabardino-balkarskaya-respublika"), array("name" => "Тамбовская область", "icon" => "tmb.png", "lat" => "52.720815", "lng" => "41.581811", "radius" => "167234", "count" => "0", "trnaslit" => "tambovskaya-oblast"), array("name" => "Республика Дагестан", "icon" => "dgs.png", "lat" => "43.11901", "lng" => "46.847843", "radius" => "254602", "count" => "0", "trnaslit" => "respublika-dagestan"), array("name" => "Псковская область", "icon" => "psk.png", "lat" => "57.345853", "lng" => "29.421119", "radius" => "228490", "count" => "0", "trnaslit" => "pskovskaya-oblast"), array("name" => "Новгородская область", "icon" => "ngd.png", "lat" => "58.195072", "lng" => "32.92445", "radius" => "239097", "count" => "0", "trnaslit" => "novgorodskaya-oblast"), array("name" => "Архангельская область", "icon" => "arh.png", "lat" => "63.87483", "lng" => "43.002344", "radius" => "500704", "count" => "0", "trnaslit" => "arhangelskaya-oblast"), array("name" => "Республика Северная Осетия-Алания", "icon" => "so.png", "lat" => "43.195482", "lng" => "44.177654", "radius" => "95602", "count" => "0", "trnaslit" => "respublika-severnaya-osetiya-alaniya"), array("name" => "Брянская область", "icon" => "br.png", "lat" => "52.952981", "lng" => "33.29275", "radius" => "182701", "count" => "0", "trnaslit" => "bryanskaya-oblast"), array("name" => "Сахалинская область", "icon" => "shl.png", "lat" => "48.450759", "lng" => "149.388314", "radius" => "921784", "count" => "0", "trnaslit" => "sahalinskaya-oblast"), array("name" => "Еврейская автономная область", "icon" => "evr.png", "lat" => "48.581949", "lng" => "132.756976", "radius" => "193660", "count" => "0", "trnaslit" => "evreyskaya-avtonomnaya-oblast"), array("name" => "Республика Ингушетия", "icon" => "ing.png", "lat" => "43.11722", "lng" => "44.885599", "radius" => "64749", "count" => "0", "trnaslit" => "respublika-ingushetiya"), array("name" => "Магаданская область", "icon" => "mgd.png", "lat" => "62.826558", "lng" => "154.103993", "radius" => "631784", "count" => "0", "trnaslit" => "magadanskaya-oblast"), array("name" => "Чеченская Республика", "icon" => "chn.png", "lat" => "43.247467", "lng" => "45.746912", "radius" => "112923", "count" => "0", "trnaslit" => "chechenskaya-respublika"), array("name" => "Ненецкий автономный округ", "icon" => "nnc.png", "lat" => "67.930797", "lng" => "54.469868", "radius" => "516893", "count" => "0", "trnaslit" => "nenetskiy-avtonomniy-okrug"), array("name" => "Чукотский автономный округ", "icon" => "chk.png", "lat" => "67.204936", "lng" => "174.034141", "radius" => "882254", "count" => "0", "trnaslit" => "chukotskiy-avtonomniy-okrug"), );
$l_sSection = 'vehicles';
$aSpxSectionWhereAttr = array('jobs' => array('city' => 'llocationcity', 'lng' => 'l_longitude', 'lat' => 'l_latitude'), 'vehicles' => array('city' => 'lgeocity', 'lng' => 'l_longitude', 'lat' => 'l_latitude'), 'realestate' => array('city' => 'lgeocity', 'lng' => 'l_longitude', 'lat' => 'l_latitude'));
$options = array('connectTimeout' => 1, 'arrayResult' => true, 'matchMode' => SPH_MATCH_EXTENDED2, 'limits' => 1, 'page' => 1, 'index' => "i_ref_vehicles i_delta_vehicles");
$id = 0;
$temp = array();
$l_aReturn = array();
$sph = new CSphinx(new sph_conf('3313'));
$sph->resetSphinx();
$sph->multiInit($options);
echo "\nPrforming sphinx search...\n";
foreach ($l_aTopRegions as $region) {
$spx_lngAttrName = $aSpxSectionWhereAttr[$l_sSection]['lng'];
$spx_latAttrName = $aSpxSectionWhereAttr[$l_sSection]['lat'];
$sph->setGeofilter($spx_lngAttrName, $spx_latAttrName, $region['lng'], $region['lat'], $region['radius']);
$sph->setAttribute('what', $l_sSection);
$sph->setOptions($options);
$temp[$id] = $region;
$temp[$id]['count'] = $sph->addQuery();
$id++;
if ($id == 31) {
$allResults = $sph->multiSearch();
for ($i = 0; $i < sizeof($allResults); $i++) {
function newAntiDuplicate($l_countries, $max = 100)
{
$dbConf = new Config("/home/eyezeek/db.conf", "ini");
$optionsTemplate = array('connectTimeout' => 5, 'arrayResult' => true, 'matchMode' => SPH_MATCH_EXTENDED2, 'limits' => $max, 'page' => 1, 'index' => "i_ref_jobs i_delta_jobs", 'groupBy' => array('duplicates', SPH_GROUPBY_ATTR, '@count desc'));
foreach ($l_countries as $target) {
dumpMsg('Starting anti-dup ' . $target['reg']);
$goOnWithNextPage = TRUE;
$page = 1;
$conf = new Conftmp($target['port']);
$sph = new CSphinx($conf, $target['server']);
//var_dump($target);
// ids to keep
$keepIDs = array();
// crc to delete
$crcIDs = array(1);
// ids to set inactive = 0
$duplicateIDs = array(0);
dumpMsg('Grouping ... ');
while ($goOnWithNextPage) {
sleep(1);
$options = $optionsTemplate;
$options['page'] = $page;
$page++;
$sph->resetSphinx();
$sph->setAttribute('what', 'jobs');
$sortParam = 'alps_promoted desc, l_postdate desc';
$options['sortMode'] = array(SPH_SORT_EXTENDED, $sortParam);
$sph->setOptions($options);
$res = $sph->search();
$spx_ok = TRUE;
if (!$res) {
dumpError('Grouping: ' . $sph->getError());
die;
}
$relevantGroups = 0;
if (isset($res['matches'])) {
foreach ($res['matches'] as $oneMatch) {
// if this is > 1 then there are duplicates
$matchCounter = $oneMatch['attrs']['@count'];
if ($matchCounter > 1) {
$keepIDs[] = $oneMatch['id'];
$crcIDs[] = $oneMatch['attrs']['duplicates'];
$relevantGroups++;
} else {
$goOnWithNextPage = FALSE;
}
}
// extra condition to check if there is no more need to traverse next page
if ($relevantGroups < $max) {
$goOnWithNextPage = FALSE;
}
} else {
// no more matches i.e. nothing to do
$goOnWithNextPage = FALSE;
}
// for now simple put manual cut-off to avoid offset overflows
$goOnWithNextPage = FALSE;
}
dumpMsg('Grouped ' . ($page - 1) . ' pages and ' . count($keepIDs) . ' listings have at least 1 duplicate... ');
dumpMsg('Processing IDs ... ');
$goOnWithNextPage = TRUE;
$page = 1;
if (count($keepIDs) == 0) {
dumpMsg('Nothing to do. No duplicated were detected ');
dumpMsg('Finished anti-dup ' . $target['reg']);
return TRUE;
}
while ($goOnWithNextPage) {
// find actual duplicate ids
$options = $optionsTemplate;
$options['page'] = $page;
$page++;
$options['limits'] = 2000;
unset($options['groupBy']);
//var_dump($options);die;
$sph->resetSphinx();
$sph->setAttribute('what', 'jobs');
$sph->setOptions($options);
$sph->setFilter($sph->VALUE_FILTER, array('theid', $keepIDs, TRUE));
$sph->setFilter($sph->VALUE_FILTER, array('duplicates', $crcIDs, FALSE));
$res = $sph->search();
if (!$res) {
dumpError('Error in fetching duplicates: ' . $sph->getError());
die;
}
$resultArray = $sph->getIDs();
if (sizeof($resultArray) < $options['limits']) {
$goOnWithNextPage = FALSE;
}
$duplicateIDs = array_merge($duplicateIDs, $sph->getIDs());
// for now simple put manual cut-off to avoid offset overflows
$goOnWithNextPage = FALSE;
}
dumpMsg('Doing SQL updates and deactivating ' . sizeof($duplicateIDs) . ' duplicates ... ');
$lastDump = 0;
$maxDump = 200;
$tmpIDs = array();
$liveDB = new DBAdapter2(array('host' => $dbConf->live->host, 'username' => $dbConf->live->username, 'password' => $dbConf->live->password, 'schema' => $target['schemali']));
//var_dump($liveDB);
for ($i = 0; $i < sizeof($duplicateIDs); $i++) {
$tmpIDs[] = $duplicateIDs[$i];
$lastDump++;
if ($lastDump == $maxDump || $i + 1 == sizeof($duplicateIDs)) {
sleep(1);
$lastDump = 0;
$sql = "UPDATE jobs SET l_isActive=0 WHERE l_id IN (" . implode(",", $tmpIDs) . ")";
$tmpIDs = array();
try {
$affectedRows = 0;
$lid = 0;
$liveDB->executeNoresSQL($sql, $affectedRows, $lid);
} catch (DBAdapter2Exception $e) {
dumpError(" MYSQL Error: " . $e->getMessage() . " *** QUERY: {$sql}");
die;
}
}
}
unset($sph);
dumpMsg('Finished anti-dup ' . $target['reg']);
}
}
public function getSavedSearchListings($queryArray, $limit, $maxMultiSearch, $sphinxConfig)
{
// queueArray passed by reference to prevent it copying into memory
$sphinx = new CSphinx($sphinxConfig, $sphinxConfig->host);
$sphinx->resetSphinx();
// Initialize search options first
$initOptions = array('connectTimeout' => 1, 'arrayResult' => true, 'matchMode' => SPH_MATCH_EXTENDED2, 'limits' => $limit, 'page' => 1);
$onedayago = strtotime('24 hours ago');
// iterrate all the listings per multi-search
$oneResultArray = array();
$resultHash = array();
$searchQueueCounter = 0;
foreach ($queryArray as $md5Hash => $usersArray) {
//$this->debugMode('enter hash: '.$md5Hash);
//$this->debugMode('with: '.sizeof($usersArray).' values');
if (sizeof($usersArray) > 0) {
// get only the first element of the array since
// elements are actual users who asked for the search with this md5
$oneUser = $usersArray[0];
// keep this counter to determine if sph has reached its defined limit
$userSection = $oneUser['data']['section'] === 'auto' ? 'vehicles' : $oneUser['data']['section'];
$initOptions['index'] = 'i_ref_' . $userSection . ' i_delta_' . $userSection;
$initOptions['sortMode'] = array(SPH_SORT_EXTENDED, 'alps_promoted desc');
//get only promoted on top
//var_dump($initOptions);die;
$sphinx->multiInit($initOptions);
//$sphinx->setOptions($initOptions);
$sphinx->setAttribute('what', $userSection);
if (!isset($this->searchResults[$md5Hash])) {
if (!($oneUser['data']['type'] == 'search')) {
$this->setCategory($sphinx, $oneUser['data']['section'], $oneUser['data']['what']);
} else {
$sphinx->setQuery($oneUser['data']['what']);
}
if (isset($oneUser['data']['refinments'])) {
// Get all the query refinments
foreach ($oneUser['data']['refinments'] as $oneRefinment) {
$oneRefinment = explode('=', $oneRefinment);
//TODO: check if refinment is a geo component in which case make sure u have all
//var_dump($oneRefinment);
$refinmentSettings = $this->getSpxOptionType($oneUser['data']['section'], $oneRefinment[0]);
//var_dump($refinmentSettings);die;
if ($refinmentSettings != FALSE && sizeof($refinmentSettings) == 2) {
// we got a proper refinment
$this->refinmentToSphinx($sphinx, $refinmentSettings, $oneRefinment[1]);
}
}
}
// var_dump($sphinx->getOptions());die;
// add additional filter which will order by date
$sphinx->setFilter($sphinx->VALUE_RANGE_INT, array('l_postdate', $onedayago, strtotime('now'), FALSE));
// add additional filter which will give only promoted
$sphinx->setFilter($sphinx->VALUE_FILTER, array('alps_promoted', array(1), FALSE));
// puush the query to be executed
$resultQueryNumber = $sphinx->addQuery();
//$resultQueryNumber = $sphinx->search();
//$allResults = $sphinx->multiSearch();
//var_dump($resultQueryNumber);die;
$oneResultArray[$resultQueryNumber] = $md5Hash;
// increment the current limit counter and do check
$searchQueueCounter = $searchQueueCounter + 1;
}
}
if ($searchQueueCounter >= $maxMultiSearch) {
//$this->debugMode('process a batch of '.$maxMultiSearch);
// Do search now - limit has been reached for this itterration
$allResults = $sphinx->multiSearch();
//var_dump($allResults);die;
foreach ($oneResultArray as $qNumber => $arrayElementMD5) {
$this->searchResults[$arrayElementMD5] = $sphinx->getIDs($qNumber);
}
// reset status counters
$oneResultArray = array();
$searchQueueCounter = 0;
$sphinx->resetSphinx();
}
}
//$this->debugMode('process remaining batch of '.sizeof($oneResultArray));
// Process any remaining batch
if (sizeof($oneResultArray) > 0) {
// Do search now - limit has been reached for this itterration
$allResults = $sphinx->multiSearch();
//var_dump($allResults);die;
foreach ($oneResultArray as $qNumber => $arrayElementMD5) {
$this->searchResults[$arrayElementMD5] = $sphinx->getIDs($qNumber);
}
// reset status counters
$oneResultArray = array();
$searchQueueCounter = 0;
$sphinx->resetSphinx();
}
//var_dump($this->searchResults);
//die('finished');
unset($sphinx);
return $this->searchResults;
}
<?php
include 'sphinxapi.php';
include 'CSphinx.class.php';
class sph_conf
{
public $port;
public function __construct($port)
{
$this->port = $port;
}
}
$cities = array(array("city_name" => "Россия", "city_icon" => "0.png", "count" => 0), array("city_name" => "Москва", "city_icon" => "2.png", "count" => 0), array("city_name" => "Санкт-Петербург", "city_icon" => "7.png", "count" => 0), array("city_name" => "Екатеринбург", "city_icon" => "12.png", "count" => 0), array("city_name" => "Казань", "city_icon" => "17.png", "count" => 0), array("city_name" => "Нижний Новгород", "city_icon" => "27.png", "count" => 0), array("city_name" => "Самара", "city_icon" => "37.png", "count" => 0), array("city_name" => "Тольятти", "city_icon" => "47.png", "count" => 0), array("city_name" => "Уфа", "city_icon" => "52.png", "count" => 0), array("city_name" => "Ярославль", "city_icon" => "57.png", "count" => 0), array("city_name" => "Челябинск", "city_icon" => "62.png", "count" => 0), array("city_name" => "Омск", "city_icon" => "67.png", "count" => 0), array("city_name" => "Ростов-на-Дону", "city_icon" => "72.png", "count" => 0), array("city_name" => "Новосибирск", "city_icon" => "77.png", "count" => 0), array("city_name" => "Воронеж", "city_icon" => "82.png", "count" => 0), array("city_name" => "Саратов", "city_icon" => "87.png", "count" => 0), array("city_name" => "Красноярск", "city_icon" => "92.png", "count" => 0), array("city_name" => "Краснодар", "city_icon" => "97.png", "count" => 0), array("city_name" => "Владивосток", "city_icon" => "102.png", "count" => 0), array("city_name" => "Хабаровск", "city_icon" => "107.png", "count" => 0), array("city_name" => "Иркутск", "city_icon" => "127.png", "count" => 0), array("city_name" => "Тюмень", "city_icon" => "132.png", "count" => 0), array("city_name" => "Балашиха", "city_icon" => "177.png", "count" => 0), array("city_name" => "Сочи", "city_icon" => "182.png", "count" => 0), array("city_name" => "Одинцово", "city_icon" => "192.png", "count" => 0), array("city_name" => "Подольск", "city_icon" => "197.png", "count" => 0), array("city_name" => "Красногорск", "city_icon" => "202.png", "count" => 0), array("city_name" => "Химки", "city_icon" => "219.png", "count" => 0), array("city_name" => "Ставрополь", "city_icon" => "244.png", "count" => 0), array("city_name" => "Железнодорожный", "city_icon" => "299.png", "count" => 0), array("city_name" => "Всеволожск", "city_icon" => "1457.png", "count" => 0));
$l_sSection = 'realestate';
$sph = new CSphinx(new sph_conf('3313'), '192.168.240.43');
$sph->resetSphinx();
/**
* Real estate
*/
echo "\nPerforming sphinx search for {$l_sSection}...\n";
$l_aDealTypes = array('rent' => 0, 'sale' => 1);
foreach ($l_aDealTypes as $deal_type_str => $deal_type_int) {
$options = array('connectTimeout' => 1, 'arrayResult' => TRUE, 'matchMode' => SPH_MATCH_EXTENDED2, 'index' => "i_ref_realestate i_delta_realestate", 'page' => 1, 'limits' => 1);
$sph->resetSphinx();
$sph->multiInit($options);
$tmp_arr = array();
for ($i = 0; $i < count($cities); $i++) {
$sph->setAttribute('what', $l_sSection);
//don't filter for all Russia
if ($i > 0) {
$sph->setAttribute('lgeocity', $cities[$i]['city_name']);
$this->port = $port;
}
}
/**
* Sphinx options
*/
$options = array('connectTimeout' => 5, 'arrayResult' => true, 'matchMode' => SPH_MATCH_EXTENDED2, 'limits' => 41, 'page' => 1, 'index' => "i_ref_vehicles i_delta_vehicles", 'groupBy' => array('l_geo_city', SPH_GROUPBY_ATTR, '@count DESC'));
$l_aSections = array('auto', 'realestate');
foreach ($l_aSections as $l_sSection) {
$l_aResult = array();
$l_aMatches = array();
if ($l_sSection == 'auto') {
$sph = new CSphinx(new sph_conf('3313'));
$table = 'vehicles';
} elseif ($l_sSection == 'realestate') {
$sph = new CSphinx(new sph_conf('3313'), '192.168.240.43');
$options['index'] = "i_ref_realestate i_delta_realestate";
$table = 'realestate';
}
$sph->resetSphinx();
$sph->setOptions($options);
echo "\nPrforming sphinx search...\n";
$l_aIds = array();
$l_aResult = $sph->search();
$l_aMatches = $l_aResult['matches'];
$l_aXmlData = array();
/**
* Excluding zero == no city
*/
foreach ($l_aMatches as $match) {
if ($match["attrs"]['l_geo_city'] != '0') {
<?php
include 'sphinxapi.php';
include 'CSphinx.class.php';
class sph_conf
{
public $port;
public function __construct($port)
{
$this->port = $port;
}
}
$sph = new CSphinx(new sph_conf('3313'));
$options = array('connectTimeout' => 5, 'arrayResult' => TRUE, 'matchMode' => SPH_MATCH_EXTENDED2, 'index' => 'i_ref_vehicles i_delta_vehicles', 'page' => 1, 'limit' => 1);
$l_sSection = 'vehicles';
$sph->resetSphinx();
$options['groupBy'] = array('l_www_id', SPH_GROUPBY_ATTR);
$sph->setAttribute('www_site_type', '2');
$sph->setAttribute('what', $l_sSection);
$sph->setFilter($sph->VALUE_FILTER, array('alps_promoted', array(1), FALSE));
$sph->setOptions($options);
$result = $sph->search(NULL, FALSE);
foreach ($result['matches'] as $one_promoted_dealer) {
$l_aReturnIDs[] = $one_promoted_dealer['attrs']['l_www_id'];
}
if (!empty($result)) {
$l_sXMLbase = '<?xml version="1.0" encoding="UTF-8"?><Default_top_dealers></Default_top_dealers>';
$l_sFile_name = "/home/eyezeek/netup/ru/seo/" . $l_sSection . "_default_top_dealers.xml";
$l_oXML = new SimpleXMLElement($l_sXMLbase);
foreach ($l_aReturnIDs as $dealer) {
$l_node = $l_oXML->AddChild('dealer');
public function multiInit($options)
{
//$cnf = $this->config->item('p_sphinx');
if ($options) {
foreach ($options as $key => $val) {
$this->lOptions[$key] = $val;
}
}
if (!isset($this->lOptions['index'])) {
$this->sError = 'Index not defined';
return FALSE;
}
$indexToSearch = trim($this->lOptions['index']);
/*
if(!isset($cnf['indexes'][$indexToSearch])){
$this->sError = 'Index '.$indexToSearch.' not found in configuration';
return FALSE;
}
*/
$serverToConnect = $this->sphinxServers[0];
//die($serverToConnect);
if (self::$oSphinx == NULL) {
// "opening ... ";
self::$oSphinx = new SphinxClient();
} else {
if ($serverToConnect != self::$lastUsedServer) {
self::$oSphinx->Close();
//echo "Setting server:".self::$lastUsedServer."<br />";
}
}
self::$lastUsedServer = $serverToConnect;
//echo "Using server:".self::$lastUsedServer."<br />";
$this->m_aDegugArry['server'] = $serverToConnect;
$this->m_aDegugArry['port'] = $this->cnf->port;
self::$oSphinx->SetServer($serverToConnect, (int) $this->cnf->port);
self::$oSphinx->ResetFilters();
self::$oSphinx->ResetGroupBy();
if (isset($this->lOptions['connectTimeout'])) {
self::$oSphinx->SetConnectTimeout((int) $this->lOptions['connectTimeout']);
} else {
self::$oSphinx->SetConnectTimeout(1);
}
}
private function make_search($city, $make, $group_by_field)
{
$options = array('connectTimeout' => 5, 'arrayResult' => TRUE, 'matchMode' => SPH_MATCH_EXTENDED2, 'index' => 'i_ref_vehicles i_delta_vehicles', 'page' => 1, 'limit' => 50);
$options['groupBy'] = array($group_by_field, SPH_GROUPBY_ATTR, '@count DESC');
$sph = new CSphinx(new sph_conf('3313'));
$sph->resetSphinx();
$sph->setAttribute('lgeocity', $city);
$sph->setAttribute('lmake', $make);
$sph->setAttribute('what', 'vehicles');
$sph->setOptions($options);
$result = $sph->search(NULL, FALSE);
return $result;
}
$this->port = $port;
}
}
/**
* Sphinx options
*/
$options = array('connectTimeout' => 5, 'arrayResult' => true, 'matchMode' => SPH_MATCH_EXTENDED2, 'limits' => 100, 'page' => 1, 'index' => 'i_ref_vehicles i_delta_vehicles', 'groupBy' => array('l_make', SPH_GROUPBY_ATTR, '@count DESC'));
$l_sPath = '/home/eyezeek/netup/ru/auto_php_files/';
//$l_sPath = '/var/www/projects/crawlers/netup_imitation/auto_php_files/';
$l_iNumOfMakers = 33;
$l_aImportantMakers = array('Great Wall' => array('translit' => 'great-wall', 'count' => 0), 'Geely' => array('translit' => 'geely', 'count' => 0));
$l_sNewAutoCount = '';
$l_sUsedAutoCount = '';
$l_aResult = array();
$l_aMatches = array();
$sph = new CSphinx(new sph_conf('3313'));
$sph->resetSphinx();
$sph->setOptions($options);
echo "Prforming sphinx search...\n";
$l_aIds = array();
$l_aResult = $sph->search();
$l_aMatches = $l_aResult['matches'];
$l_aData = array();
/**
* Excluding zero == no make
*/
foreach ($l_aMatches as $match) {
if ($match['attrs']['l_make'] != '0') {
$l_aIds[] = $match['id'];
$l_aData[$match['id']] = array('count' => $match['attrs']['@count'], 'name' => '', 'translit' => '');
}
private function make_search($city, $one_deal_type, $one_translitted_category, $group_by_field)
{
$options = array('connectTimeout' => 5, 'arrayResult' => TRUE, 'matchMode' => SPH_MATCH_EXTENDED2, 'index' => 'i_ref_realestate i_delta_realestate', 'page' => 1, 'limit' => 50);
if ($group_by_field != 'empty') {
$options['groupBy'] = array($group_by_field, SPH_GROUPBY_ATTR, '@count DESC');
}
$sph = new CSphinx(new sph_conf('3313'), '192.168.240.43');
$sph->resetSphinx();
$sph->setFilter($sph->VALUE_FILTER, array('l_deal_type', array($one_deal_type), FALSE));
$sph->setAttribute('lgeocity', $city);
foreach ($this->m_aCategoriesDetails[$one_translitted_category] as $asset_type) {
$sph->setAttribute('lassettype', $asset_type);
}
$sph->setAttribute('what', 'realestate');
$sph->setOptions($options);
$result = $sph->search(NULL, FALSE);
return $result;
}
