protected function standardize() { $CI =& get_instance(); $this->store_id = $CI->store_id; $CI->load->model('crawl_data_m', 'Crawl'); $this->merchants = ensure_array($this->merchants); $this->api_type = ensure_array($this->api_type); $this->flagDates24 = $this->date_from === $this->date_to; // get the last crawl data $this->last_crawl = $CI->Crawl->last_crawl(); $this->crawl_range = $CI->Crawl->last_crawl_range(); if ($this->time_frame) { // This report's dates are the last x many days $tF = getTimeFrame($this->time_frame); $this->date_from = $tF['date_from']; $this->date_to = $tF['date_to']; } switch ($this->report_type) { case 'violationoverview': $this->time_frame = 24; break; case 'pricingoverview': $this->time_frame = 24; break; case 'pricingviolation': break; case 'whois': break; } $this->set_filename(); }
protected function _run_cron_job($api, $from = 'no') { //if ( ! in_array($from, $this->fromAr)) { // echo "You are not allowed to run this cron"; // exit; //} // Count the active proxies // Send a warning if 5 or less are active $threshold = 10; $proxies = getProxyIPS(TRUE, $threshold + 1); if (count($proxies) <= $threshold) { $this->email_alert('Low on proxies', 'The active proxy count has reached ' . $threshold . ' or less as of ' . date('Y-m-d H:i:s')); } $apis = ensure_array($api); $storesData = $this->Store->get_stores(TRUE); $currentDate = date('Y-m-d H:i:s'); $this->Crowl_model->stats = array(); // Cycle Through API crons for ($j = 0, $n = count($apis); $j < $n; $j++) { $this->Crowl_model->stats[$apis[$j]] = array('data_found' => 0, 'price_found' => 0); $this->Crowl_model->load_sticky_api($apis[$j]); $data = array('api_type' => $apis[$j], 'key' => generate_rand(32), 'run_from' => $from, 'start_datetime' => $currentDate); try { // Update cron log information $lastLog = $this->getMaxCronLog($data['api_type']); $lastRun = empty($lastLog) ? '0000-00-00 00:00:00' : $lastLog->datetime; if (!empty($lastLog)) { if ($lastLog->datetime === "0000-00-00 00:00:00") { $update = array('datetime' => $currentDate, 'end_datetime' => $currentDate); $this->db->where('datetime', '0000-00-00 00:00:00')->where('api_type', $data['api_type'])->update('cron_log', $update); } } if ($lastRun == '0000-00-00 00:00:00' || $currentDate >= $lastRun) { // No entry in cron log table $insert_id = $this->create_cron_log($data); $this->resetViolationCount(); // Crawl the products for each store for ($i = 0, $z = count($storesData); $i < $z; $i++) { $this->index($storesData[$i]->id, $data['api_type'], $insert_id); } $data_edit['end_datetime'] = $data_edit['datetime'] = date("Y-m-d H:i:s"); $this->update_cron_log($insert_id, $data_edit); } } catch (Exception $e) { email_alertToTeam('Run Cron Job ' . ucwords($apis[$j]), $e->getMessage()); } // Now that the crawl has finished, // compare these crawl numbers with the last // and send out notifications if needed. $currentLog = $this->getMaxCronLog($data['api_type']); $lastCount = $lastLog ? $lastLog->google_count : 0; $currentCount = $currentLog->google_count; $percentThresh = 25; // send a notification if completed this percent of the last crawl $smallThresh = 10; // send a notification if complete less than this many products $msg = ''; $percentDrop = 100 - $percentThresh; if ($lastCount > 0 and $currentCount * 100 / $lastCount <= $percentThresh) { $msg = ' crawl completed ' . $percentDrop . '% less products than its previous crawl.'; } elseif ($currentCount < $smallThresh) { $msg = ' crawl completed less than ' . $smallThresh . ' products.'; } if (!empty($msg)) { $msg_txt = 'Crawl Warning: ' . $data['api_type'] . $msg; $msg_html = 'Crawl Warning: ' . '<b>' . $data['api_type'] . '</b>' . $msg; log_message('error', $msg_txt); send_email($this->config->item('alerts'), 'TrackStreet Errors', 'TrackStreet Crawl Warning', $msg_html, $msg_txt); } } $this->_checkNotFoundRatio(); // todo: should we not return to the main script here? exit; }