public function execute()
{
$i = 0;
foreach ($this->__urls as $id => $url) {
// fetch delay
if ($i > 0 && (double) self::$conf_delay_between_fetches > 0) {
sleep((double) self::$conf_delay_between_fetches);
}
if (!empty($url)) {
$md5 = md5($url);
// distinct documents only
if (!isset($this->__documents[$md5])) {
$this->total_documents++;
// add to document distinct count
$this->__documents[$md5] = new Document(Request::get($this->__formatUrl($url), self::$conf_default_timeout), $this->__document_fields, $id);
// set fetched counts
if ($this->__documents[$md5]->success) {
$this->total_documents_success++;
} else {
$this->total_documents_failed++;
}
}
} else {
$this->error = 'Invalid URL detected (empty URL with ID "' . $id . '")';
}
$i++;
}
// set success if no errors
$this->success = !$this->error;
}
/**
* Fetch documents from fetch URLs
*
* @return void
*/
public function execute()
{
$i = 0;
$this->__log('Executing bot URL fetches', __METHOD__);
foreach ($this->__urls as $id => $url) {
if ($i > 0 && (double) self::$conf_delay_between_fetches > 0) {
sleep((double) self::$conf_delay_between_fetches);
}
if (!empty($url)) {
$md5 = md5($url);
if (!isset($this->__documents[$md5])) {
$this->total_documents++;
// add to document distinct count
$this->__documents[$md5] = new Document(Request::get($this->__formatUrl($url), self::$conf_default_timeout), $id);
// set fetched counts
if ($this->__documents[$md5]->success) {
$this->total_documents_success++;
} else {
$this->total_documents_failed++;
}
}
} else {
$this->error = 'Invalid URL detected (empty URL with ID "' . $id . '")';
$this->__log($this->error, __METHOD__);
}
$i++;
}
$this->__log($this->total_documents . ' total documents', __METHOD__);
$this->__log($this->total_documents_success . ' documents fetched successfully', __METHOD__);
$this->__log($this->total_documents_failed . ' documents failed to fetch', __METHOD__);
// set success if no errors
$this->success = !$this->error;
}
