/**
* Run Crawler
* @todo Cleanup code and code review
*
* @param Maverick_Crawler_Model_Crawler $crawler
* @param $mode
* @return array
*/
public function run(Maverick_Crawler_Model_Crawler $crawler, $mode = Maverick_Crawler_Model_Crawler::MODE_MANUAL)
{
$errors = array();
$urls = $this->getUrls();
$logEnabled = Mage::getStoreConfig('crawler/general/log_url');
$helper = Mage::helper('maverick_crawler');
$maxTime = Mage::getStoreConfig('crawler/general/max_time');
$startTime = Mage::getStoreConfigFlag('crawler/general/max_time') ? time() : false;
// Log Start
if ($logEnabled) {
$helper->log($helper->__('###### Starting Crawler ID %s ######', $crawler->getId()));
}
foreach ($urls as $url) {
$maxTimeExceeded = $startTime && $maxTime < time() - $startTime;
if ($maxTimeExceeded) {
$errMessage = Mage::helper('maverick_crawler')->__('--> ### Stoping Crawler, Maximum Time Of Crawling Exceeded (%s secondes)', $maxTime);
if ($logEnabled) {
Mage::helper('maverick_crawler')->log($errMessage);
}
$errors[] = $errMessage;
break;
}
if ($logEnabled) {
$helper->log($helper->__('--> Warming Up %s (%s time(s))', $url, $this->_nbrOfVisits));
}
$crawlerObj = $this->_crawlerHelper->visit($url, $this->_nbrOfVisits);
if (!is_object($crawlerObj)) {
$errors[] = Mage::helper('maverick_crawler')->__('Some errors encountered while crawling, check your log file');
continue;
}
if ($crawler->getScan() == '1') {
$pageLinks = $this->_crawlerHelper->getPageLinks($crawlerObj);
if ($logEnabled) {
$helper->log($helper->__('--> Scan Option is enabled, scanning %s', $url));
$helper->log($helper->__('--> Scan Option found %s urls', count($pageLinks)));
$helper->log($helper->__('--> Crawling Them ...'));
}
foreach ($pageLinks as $link) {
$maxTimeExceeded = $startTime && $maxTime < time() - $startTime;
if ($maxTimeExceeded) {
$errMessage = Mage::helper('maverick_crawler')->__('--> ### Stoping Scanning, Maximum Time Of Crawling Exceeded (%s secondes)', $maxTime);
if ($logEnabled) {
Mage::helper('maverick_crawler')->log($errMessage);
}
$errors[] = $errMessage;
break;
}
$start = time();
$this->_crawlerHelper->visit($link, $this->_nbrOfVisits);
$end = time() - $start;
if ($logEnabled) {
$helper->log($helper->__(' --> Warming Up %s (%s time(s), Took %s secondes)', $link, $this->_nbrOfVisits, $end));
}
}
}
}
if ($logEnabled) {
$helper->log($helper->__('###### End Process Crawler ID %s ######', $crawler->getId()));
}
$crawler->setLastExecutionMode($mode)->setLastExecutionAt(Mage::getModel('core/date')->date('Y-m-d H:i:s'))->save();
return $errors;
}
