/**
* Cleans up the crawler after it has finished.
*/
protected function cleanup()
{
// Free/unlock caches
$this->CookieCache->cleanup();
$this->LinkCache->cleanup();
// Delete working-dir
PHPCrawlerUtils::rmDir($this->working_directory);
// Remove semaphore (if multiprocess-mode)
if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_NONE) {
$sem_key = sem_get($this->crawler_uniqid);
sem_remove($sem_key);
}
}
/**
* Cleans up the crawler after it has finished.
*/
protected function cleanup()
{
$abort_reason = $this->checkForAbort();
if ($abort_reason != null && $abort_reason != PHPCrawlerAbortReasons::ABORTREASON_PASSEDTHROUGH) {
$crawler_status = $this->CrawlerStatusHandler->getCrawlerStatus();
$crawler_status->documents_received = 0;
$crawler_status->bytes_received = 0;
$crawler_status->links_followed = 0;
$crawler_status->previous_abort_reason = $abort_reason;
$crawler_status->abort_reason = null;
$this->CrawlerStatusHandler->setCrawlerStatus($crawler_status);
} else {
// Free/unlock caches
$this->CookieCache->cleanup();
$this->LinkCache->cleanup();
// Delete working-dir
PHPCrawlerUtils::rmDir($this->working_directory);
// Remove semaphore (if multiprocess-mode)
if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_NONE) {
$sem_key = sem_get($this->crawler_uniqid);
sem_remove($sem_key);
}
}
}
/**
* Receives and processes the given URL
*
* @param PHPCrawlerURLDescriptor $UrlDescriptor The URL as PHPCrawlerURLDescriptor-object
* @return bool TURE if the crawling-process should be aborted after processig the URL, otherwise FALSE.
*/
protected function processUrl(PHPCrawlerURLDescriptor $UrlDescriptor)
{
PHPCrawlerBenchmark::start("processing_url");
// Setup HTTP-request
$this->PageRequest->setUrl($UrlDescriptor);
// Add cookies to request
if ($this->cookie_handling_enabled == true) {
$this->PageRequest->addCookieDescriptors($this->CookieCache->getCookiesForUrl($UrlDescriptor->url_rebuild));
}
// Add basic-authentications to request
$authentication = $this->UserSendDataCache->getBasicAuthenticationForUrl($UrlDescriptor->url_rebuild);
if ($authentication != null) {
$this->PageRequest->setBasicAuthentication($authentication["username"], $authentication["password"]);
}
// Add post-data to request
$post_data = $this->UserSendDataCache->getPostDataForUrl($UrlDescriptor->url_rebuild);
while (list($post_key, $post_value) = @each($post_data)) {
$this->PageRequest->addPostData($post_key, $post_value);
}
// Do request
$PageInfo = $this->PageRequest->sendRequest();
if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) {
// Check for abort
$abort_reason = $this->checkForAbort();
if ($abort_reason !== null) {
return true;
}
$this->ProcessCommunication->updateCrawlerStatus($PageInfo);
}
// Remove post and cookie-data from request-object
$this->PageRequest->clearCookies();
$this->PageRequest->clearPostData();
// Call user-moethods if crawler doesn't run in MPMODE_PARENT_EXECUTES_USERCODE
if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) {
// Call the "abstract" method handlePageData
$user_abort = false;
$page_info = $PageInfo->toArray();
$user_return_value = $this->handlePageData($page_info);
if ($user_return_value < 0) {
$user_abort = true;
}
// Call the "abstract" method handleDocumentInfo
$user_return_value = $this->handleDocumentInfo($PageInfo);
if ($user_return_value < 0) {
$user_abort = true;
}
// Update status if user aborted process
if ($user_abort == true) {
$this->ProcessCommunication->updateCrawlerStatus(null, PHPCrawlerAbortReasons::ABORTREASON_USERABORT);
}
// Check for abort from other processes
if ($this->checkForAbort() !== null) {
return true;
}
}
// Filter found URLs by defined rules
if ($this->follow_redirects_till_content == true) {
$crawler_status = $this->ProcessCommunication->getCrawlerStatus();
// If content wasn't found so far and content was found NOW
if ($crawler_status->first_content_url == null && $PageInfo->http_status_code == 200) {
$this->ProcessCommunication->updateCrawlerStatus(null, null, $PageInfo->url);
$this->UrlFilter->setBaseURL($PageInfo->url);
// Set current page as base-URL
$this->UrlFilter->filterUrls($PageInfo);
$this->follow_redirects_till_content = false;
// Content was found, so this can be set to FALSE
} else {
if ($crawler_status->first_content_url == null) {
$this->UrlFilter->keepRedirectUrls($PageInfo);
// Content wasn't found so far, so just keep redirect-urls
} else {
if ($crawler_status->first_content_url != null) {
$this->follow_redirects_till_content = false;
$this->UrlFilter->filterUrls($PageInfo);
}
}
}
} else {
$this->UrlFilter->filterUrls($PageInfo);
}
// Add Cookies to Cookie-cache
if ($this->cookie_handling_enabled == true) {
$this->CookieCache->addCookies($PageInfo->cookies);
}
// Add filtered links to URL-cache
$this->LinkCache->addURLs($PageInfo->links_found_url_descriptors);
PHPCrawlerBenchmark::stop("processing_url");
// Complete PageInfo-Object with benchmarks
$PageInfo->benchmarks = PHPCrawlerBenchmark::getAllBenchmarks();
if ($this->multiprocess_mode == PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) {
$this->DocumentInfoQueue->addDocumentInfo($PageInfo);
}
// Mark URL as "followed"
$this->LinkCache->markUrlAsFollowed($UrlDescriptor);
PHPCrawlerBenchmark::resetAll(array("crawling_process"));
return false;
}