public function execute() { $i = 0; foreach ($this->__urls as $id => $url) { // fetch delay if ($i > 0 && (double) self::$conf_delay_between_fetches > 0) { sleep((double) self::$conf_delay_between_fetches); } if (!empty($url)) { $md5 = md5($url); // distinct documents only if (!isset($this->__documents[$md5])) { $this->total_documents++; // add to document distinct count $this->__documents[$md5] = new Document(Request::get($this->__formatUrl($url), self::$conf_default_timeout), $this->__document_fields, $id); // set fetched counts if ($this->__documents[$md5]->success) { $this->total_documents_success++; } else { $this->total_documents_failed++; } } } else { $this->error = 'Invalid URL detected (empty URL with ID "' . $id . '")'; } $i++; } // set success if no errors $this->success = !$this->error; }
/** * Fetch documents from fetch URLs * * @return void */ public function execute() { $i = 0; $this->__log('Executing bot URL fetches', __METHOD__); foreach ($this->__urls as $id => $url) { if ($i > 0 && (double) self::$conf_delay_between_fetches > 0) { sleep((double) self::$conf_delay_between_fetches); } if (!empty($url)) { $md5 = md5($url); if (!isset($this->__documents[$md5])) { $this->total_documents++; // add to document distinct count $this->__documents[$md5] = new Document(Request::get($this->__formatUrl($url), self::$conf_default_timeout), $id); // set fetched counts if ($this->__documents[$md5]->success) { $this->total_documents_success++; } else { $this->total_documents_failed++; } } } else { $this->error = 'Invalid URL detected (empty URL with ID "' . $id . '")'; $this->__log($this->error, __METHOD__); } $i++; } $this->__log($this->total_documents . ' total documents', __METHOD__); $this->__log($this->total_documents_success . ' documents fetched successfully', __METHOD__); $this->__log($this->total_documents_failed . ' documents failed to fetch', __METHOD__); // set success if no errors $this->success = !$this->error; }