/**
* Add new keywords
* @return json
* @author Ruslan Ushakov
*/
function keyword_url_cron_job()
{
$this->load->model('sites_model');
$this->load->model('settings_model');
$this->load->model('ranking_model');
$parent_id = (int) $this->input->post('job');
$jobs_in_process = $this->ranking_model->getInProcessUrlCronJobsCount();
if ($jobs_in_process <= self::CRON_JOBS_TO_RUN) {
$cron_jobs = $this->ranking_model->getWaitingChildCronJobs($parent_id, self::CRON_JOBS_TO_RUN);
//change status of selected jobs to IN PROCESS
$cron_jobs_ids = array_map(function ($cron_job) {
return $cron_job->id;
}, $cron_jobs);
if ($cron_jobs_ids && $this->ranking_model->changeCronJobStatus($cron_jobs_ids, $this->ranking_model->CRON_STATUS_INPROCESS)) {
foreach ($cron_jobs as $key => $cron_job) {
$cron_jobs[$key]->status = $this->ranking_model->CRON_STATUS_INPROCESS;
}
}
// get parent jobs
$cron_jobs_parents_ids = array_values(array_filter(array_unique(array_map(function ($cron_job) {
return $cron_job->parent_id;
}, $cron_jobs))));
$cron_jobs_parents = array();
if (!empty($cron_jobs_parents_ids)) {
foreach ($this->ranking_model->get_('keyword_cron_jobs', array('id' => $cron_jobs_parents_ids)) as $cron_jobs_parent) {
$cron_jobs_parents[$cron_jobs_parent->id] = $cron_jobs_parent;
}
}
// get corresponding sites
$sites_ids = array_values(array_unique(array_filter(array_map(function ($cron_job) {
return $cron_job->site_id;
}, $cron_jobs_parents)), SORT_NUMERIC));
$sites = array();
if (!empty($sites_ids)) {
foreach ($this->ranking_model->get_('sites', array('id' => $sites_ids)) as $site) {
$sites[$site->id] = $site;
}
}
// get search_terms_groups
$cron_jobs_group_ids = array_values(array_unique(array_filter(array_map(function ($cron_job) {
return $cron_job->group_id;
}, $cron_jobs_parents)), SORT_NUMERIC));
$search_terms_groups = array();
if (!empty($cron_jobs_group_ids)) {
foreach ($this->ranking_model->get_('search_terms_groups', array('id' => $cron_jobs_group_ids)) as $search_terms_group) {
$search_terms_groups[$search_terms_group->id] = $search_terms_group;
}
}
// get product list items
$product_list_item_ids = array_filter(array_map(function ($cron_job) {
return $cron_job->product_list_item_id;
}, $cron_jobs));
$product_list_items = array();
if (!empty($product_list_item_ids)) {
foreach ($this->ranking_model->get_('product_list_items', array('id' => $product_list_item_ids)) as $product_list_item) {
$product_list_items[$product_list_item->id] = $product_list_item;
}
}
$this->load->library('email');
$config['protocol'] = 'sendmail';
$config['mailpath'] = '/usr/sbin/sendmail';
$config['charset'] = 'UTF-8';
$config['wordwrap'] = TRUE;
$config['mailtype'] = 'html';
$this->email->initialize($config);
$begin_requests_time = gmdate('Y-m-d H:i:s');
$imported_cron_jobs = 0;
if (!empty($cron_jobs)) {
$sent_cron_jobs = array();
foreach ($cron_jobs as $cron_job) {
// change status of current job to IN PROCESS
if (!empty($cron_job->id) && $this->ranking_model->changeCronJobStatus($cron_job->id, $this->ranking_model->CRON_STATUS_INPROCESS)) {
$cron_job->status = $this->ranking_model->CRON_STATUS_INPROCESS;
}
if (!$cron_job->url) {
continue;
}
$keyword_cron_jobs_log_data = array('job_id' => null, 'message' => null, 'child_job_id' => $cron_job->request_id, 'job_phase' => $this->ranking_model->JOB_PHASE_RESPONSE, 'job_phase_status' => 'true');
if (empty($cron_job->product_list_item_id)) {
// cron job per search term
$cron_job_parent = isset($cron_jobs_parents[$cron_job->parent_id]) ? $cron_jobs_parents[$cron_job->parent_id] : null;
if (!$cron_job_parent) {
continue;
}
$site = isset($sites[$cron_job_parent->site_id]) ? $sites[$cron_job_parent->site_id] : null;
$search_terms_group = isset($search_terms_groups[$cron_job_parent->group_id]) ? $search_terms_groups[$cron_job_parent->group_id] : null;
if (!($site && $search_terms_group)) {
continue;
}
$site_name = strtolower($site->name);
$location = '';
// get crawler name
$crawler_name = Ranking_model::getCrawlerName($site_name, $location);
$group_name = $search_terms_group->name;
$keyword_cron_jobs_log_message = array('id' => $cron_job->request_id, 'site' => $crawler_name, 'keyword' => $cron_job->keyword, 'group' => $group_name, 'status' => 'RESPONSE GET');
$keyword_cron_jobs_log_data['job_id'] = $cron_job_parent->id;
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
} else {
// cron job per product
$cron_job_parent = null;
$product_list_item = isset($product_list_items[$cron_job->product_list_item_id]) ? $product_list_items[$cron_job->product_list_item_id] : null;
$keyword_cron_jobs_log_message = array('id' => $cron_job->request_id, 'product_url' => empty($product_list_item->url) ? null : $product_list_item->url, 'status' => 'RESPONSE GET');
$keyword_cron_jobs_log_data['job_id'] = $cron_job->id;
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
}
$ch = curl_init($cron_job->url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$ch_result = curl_exec($ch);
$curl_log_data = array('type' => 'keyword_cron_job', 'url' => curl_getinfo($ch, CURLINFO_EFFECTIVE_URL), 'http_code' => curl_getinfo($ch, CURLINFO_HTTP_CODE), 'curl_info' => json_encode(curl_getinfo($ch)), 'result' => $ch_result);
$curl_log_id = $this->ranking_model->create_($this->ranking_model->tables['curl_logs'], $curl_log_data);
$request_time_seconds = round(curl_getinfo($ch, CURLINFO_TOTAL_TIME));
$response_request_time = floor($request_time_seconds / 60) . " min " . $request_time_seconds % 60 . " sec";
$keyword_cron_jobs_log_message['request time'] = $response_request_time;
$keyword_cron_jobs_log_data['curl_log_id'] = $curl_log_id;
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($http_code === 302) {
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
$this->ranking_model->create_('keyword_cron_jobs_log', $keyword_cron_jobs_log_data);
$redirect_url = curl_getinfo($ch, CURLINFO_REDIRECT_URL);
$ch2 = curl_init($redirect_url);
curl_setopt($ch2, CURLOPT_RETURNTRANSFER, true);
$results = curl_exec($ch2);
$curl_log_data = array('type' => 'keyword_cron_job', 'url' => curl_getinfo($ch2, CURLINFO_EFFECTIVE_URL), 'http_code' => curl_getinfo($ch2, CURLINFO_HTTP_CODE), 'curl_info' => json_encode(curl_getinfo($ch2)), 'result' => $results);
$curl2_log_id = $this->ranking_model->create_('curl_logs', $curl_log_data);
$results_code = curl_getinfo($ch2, CURLINFO_HTTP_CODE);
$request_time_seconds = round(curl_getinfo($ch2, CURLINFO_TOTAL_TIME));
$result_request_time = floor($request_time_seconds / 60) . " min " . $request_time_seconds % 60 . " sec";
$keyword_cron_jobs_log_message['request time'] = $result_request_time;
$keyword_cron_jobs_log_data['job_phase'] = $this->ranking_model->JOB_PHASE_IMPORT_BEGIN;
$keyword_cron_jobs_log_data['curl_log_id'] = $curl2_log_id;
if ($results_code === 200) {
if ($results) {
// got results, set CRON_STATUS_READY, make import
$keyword_cron_jobs_log_message['status'] = 'IMPORT BEGIN';
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
$this->ranking_model->create_('keyword_cron_jobs_log', $keyword_cron_jobs_log_data);
$import_time_begin = microtime(true);
if (empty($cron_job->product_list_item_id)) {
// import data per search term
$imported_count = $this->import_ranking_data($results, $cron_job, $search_terms_group->id, $site->id);
} else {
// import data per product
$imported_count = $this->import_ranking_data($results, $cron_job, null, null, $product_list_item);
}
$import_time = number_format(microtime(true) - $import_time_begin, 2);
$imported_cron_jobs++;
$this->ranking_model->changeCronJobStatus($cron_job->id, $this->ranking_model->CRON_STATUS_READY);
if (!empty($cron_job_parent) && $cron_job_parent->status != $this->ranking_model->CRON_STATUS_READY) {
if ($this->ranking_model->changeCronJobStatus($cron_job_parent->id, $this->ranking_model->CRON_STATUS_READY)) {
$cron_job_parent->status = $this->ranking_model->CRON_STATUS_READY;
}
}
$keyword_cron_jobs_log_message['status'] = 'STATUS READY';
unset($keyword_cron_jobs_log_message['request time']);
$keyword_cron_jobs_log_message['import time'] = "{$import_time} sec";
$keyword_cron_jobs_log_message['imported products'] = $imported_count;
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
$keyword_cron_jobs_log_data['job_phase'] = $this->ranking_model->JOB_PHASE_IMPORT_END;
$this->ranking_model->create_('keyword_cron_jobs_log', $keyword_cron_jobs_log_data);
} else {
// result is empty, set CRON_STATUS_WARNING
$this->ranking_model->changeCronJobStatus($cron_job->id, $this->ranking_model->CRON_STATUS_WARNING);
if (!empty($cron_job_parent) && $cron_job_parent->status != $this->ranking_model->CRON_STATUS_WARNING) {
if ($this->ranking_model->changeCronJobStatus($cron_job_parent->id, $this->ranking_model->CRON_STATUS_WARNING)) {
$cron_job_parent->status = $this->ranking_model->CRON_STATUS_WARNING;
}
}
$keyword_cron_jobs_log_message['status'] = 'STATUS WARNING - Response is empty';
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
$keyword_cron_jobs_log_data['job_phase_status'] = 'false';
$this->ranking_model->create_('keyword_cron_jobs_log', $keyword_cron_jobs_log_data);
}
} else {
// result wasn't received, set CRON_STATUS_ERROR
$this->ranking_model->changeCronJobStatus($cron_job->id, $this->ranking_model->CRON_STATUS_ERROR);
if (!empty($cron_job_parent) && $cron_job_parent->status != $this->ranking_model->CRON_STATUS_ERROR) {
if ($this->ranking_model->changeCronJobStatus($cron_job_parent->id, $this->ranking_model->CRON_STATUS_ERROR)) {
$cron_job_parent->status = $this->ranking_model->CRON_STATUS_ERROR;
}
}
$keyword_cron_jobs_log_message['status'] = "STATUS ERROR - Result wasn't received ({$results_code})";
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
$keyword_cron_jobs_log_data['job_phase_status'] = 'false';
$this->ranking_model->create_('keyword_cron_jobs_log', $keyword_cron_jobs_log_data);
}
} elseif ($http_code === 404 || preg_match('/[5]\\d\\d/', $http_code) || empty($ch_result)) {
// not found, set CRON_STATUS_ERROR
$this->ranking_model->changeCronJobStatus($cron_job->id, $this->ranking_model->CRON_STATUS_ERROR);
if (!empty($cron_job_parent) && $cron_job_parent->status != $this->ranking_model->CRON_STATUS_ERROR) {
if ($this->ranking_model->changeCronJobStatus($cron_job_parent->id, $this->ranking_model->CRON_STATUS_ERROR)) {
$cron_job_parent->status = $this->ranking_model->CRON_STATUS_ERROR;
}
}
if ($http_code === 404) {
$keyword_cron_jobs_log_message['status'] = 'STATUS ERROR - The resource could not be found.';
} elseif (!empty($ch_result)) {
$keyword_cron_jobs_log_message['status'] = "STATUS ERROR - REST API Server Error ({$http_code})";
} else {
$keyword_cron_jobs_log_message['status'] = "STATUS ERROR - Empty response from keyword server";
}
$keyword_cron_jobs_log_data['message'] = json_encode($keyword_cron_jobs_log_message);
$keyword_cron_jobs_log_data['job_phase_status'] = 'false';
$this->ranking_model->create_('keyword_cron_jobs_log', $keyword_cron_jobs_log_data);
} else {
$this->ranking_model->changeCronJobStatus($cron_job->id, $this->ranking_model->CRON_STATUS_WAITING);
}
}
}
$end_requests_time = gmdate('Y-m-d H:i:s');
$requests_count = empty($cron_jobs) ? 0 : count($cron_jobs);
if (count($cron_jobs) < 20) {
// $this->amazon_urls_unification(); no need for the moment
}
$this->load->model('ranking_timing_model');
$this->ranking_timing_model->add_request_timing($begin_requests_time, $end_requests_time, $requests_count, $imported_cron_jobs);
$result = array('status' => 'ok');
} else {
$result = array('status' => "too many jobs are in process ({$jobs_in_process})");
}
// fix 'stuck' jobs, reset their status to CRON_STATUS_WAITING
$stuck_jobs = $this->ranking_model->getStuckChildCronJobs();
if ($stuck_jobs) {
$stuck_jobs_ids = array_map(function ($stuck_job) {
return $stuck_job->id;
}, $stuck_jobs);
$this->ranking_model->changeCronJobStatus($stuck_jobs_ids, $this->ranking_model->CRON_STATUS_WAITING);
}
$this->output->set_content_type('application/json')->set_output(json_encode($result));
}