public function init()
{
parent::init();
/**
* The following settings are copied from worldjournal ajax request
*/
$this->_requestUrl = $this->_hostname . "/wp-content/themes/wjlife/includes/classified-core.php" . "?regions=" . $this->_currentRegionName . "&variant=" . $this->_wjlang . "&t=" . time();
$this->_optionVaules = ["relation" => "AND", "0" => ["relation" => "AND", "0" => ["key" => "wj_order_id"]]];
$this->_requestHeader = ['HTTP_X-Requested-With' => 'XMLHttpRequest', 'contentType' => 'application/x-www-form-urlencoded;charset=utf-8'];
}
/**
* crawl fetch data and insert it into database.
*
* @param BaseModel $model
*/
public function crawl(BaseModel $model)
{
// set 500 seconds time limit to run this program
set_time_limit(500);
$time_start = microtime(true);
$date = date('m/d/Y h:i:s a', time());
echo '[ ' . $date . ' ]: ';
$posts = $model->fetchAdData();
$time_end = microtime(true);
echo " [ time spent on crawling: " . ($time_end - $time_start) . ' ] ';
if (!empty($posts)) {
Post::batchInsert($posts);
echo "There are [ " . count($posts) . " ] ad inserted" . PHP_EOL;
} else {
echo "No action taken" . PHP_EOL;
}
}
