/**
  * Cleans up the crawler after it has finished.
  */
 protected function cleanup()
 {
     // Free/unlock caches
     $this->CookieCache->cleanup();
     $this->LinkCache->cleanup();
     // Delete working-dir
     PHPCrawlerUtils::rmDir($this->working_directory);
     // Remove semaphore (if multiprocess-mode)
     if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_NONE) {
         $sem_key = sem_get($this->crawler_uniqid);
         sem_remove($sem_key);
     }
 }
 /**
  * Cleans up the crawler after it has finished.
  */
 protected function cleanup()
 {
     $abort_reason = $this->checkForAbort();
     if ($abort_reason != null && $abort_reason != PHPCrawlerAbortReasons::ABORTREASON_PASSEDTHROUGH) {
         $crawler_status = $this->CrawlerStatusHandler->getCrawlerStatus();
         $crawler_status->documents_received = 0;
         $crawler_status->bytes_received = 0;
         $crawler_status->links_followed = 0;
         $crawler_status->previous_abort_reason = $abort_reason;
         $crawler_status->abort_reason = null;
         $this->CrawlerStatusHandler->setCrawlerStatus($crawler_status);
     } else {
         // Free/unlock caches
         $this->CookieCache->cleanup();
         $this->LinkCache->cleanup();
         // Delete working-dir
         PHPCrawlerUtils::rmDir($this->working_directory);
         // Remove semaphore (if multiprocess-mode)
         if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_NONE) {
             $sem_key = sem_get($this->crawler_uniqid);
             sem_remove($sem_key);
         }
     }
 }
Ejemplo n.º 3
0
 /**
  * Receives and processes the given URL
  *
  * @param PHPCrawlerURLDescriptor $UrlDescriptor The URL as PHPCrawlerURLDescriptor-object
  * @return bool TURE if the crawling-process should be aborted after processig the URL, otherwise FALSE.
  */
 protected function processUrl(PHPCrawlerURLDescriptor $UrlDescriptor)
 {
     PHPCrawlerBenchmark::start("processing_url");
     // Setup HTTP-request
     $this->PageRequest->setUrl($UrlDescriptor);
     // Add cookies to request
     if ($this->cookie_handling_enabled == true) {
         $this->PageRequest->addCookieDescriptors($this->CookieCache->getCookiesForUrl($UrlDescriptor->url_rebuild));
     }
     // Add basic-authentications to request
     $authentication = $this->UserSendDataCache->getBasicAuthenticationForUrl($UrlDescriptor->url_rebuild);
     if ($authentication != null) {
         $this->PageRequest->setBasicAuthentication($authentication["username"], $authentication["password"]);
     }
     // Add post-data to request
     $post_data = $this->UserSendDataCache->getPostDataForUrl($UrlDescriptor->url_rebuild);
     while (list($post_key, $post_value) = @each($post_data)) {
         $this->PageRequest->addPostData($post_key, $post_value);
     }
     // Do request
     $PageInfo = $this->PageRequest->sendRequest();
     if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) {
         // Check for abort
         $abort_reason = $this->checkForAbort();
         if ($abort_reason !== null) {
             return true;
         }
         $this->ProcessCommunication->updateCrawlerStatus($PageInfo);
     }
     // Remove post and cookie-data from request-object
     $this->PageRequest->clearCookies();
     $this->PageRequest->clearPostData();
     // Call user-moethods if crawler doesn't run in MPMODE_PARENT_EXECUTES_USERCODE
     if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) {
         // Call the "abstract" method handlePageData
         $user_abort = false;
         $page_info = $PageInfo->toArray();
         $user_return_value = $this->handlePageData($page_info);
         if ($user_return_value < 0) {
             $user_abort = true;
         }
         // Call the "abstract" method handleDocumentInfo
         $user_return_value = $this->handleDocumentInfo($PageInfo);
         if ($user_return_value < 0) {
             $user_abort = true;
         }
         // Update status if user aborted process
         if ($user_abort == true) {
             $this->ProcessCommunication->updateCrawlerStatus(null, PHPCrawlerAbortReasons::ABORTREASON_USERABORT);
         }
         // Check for abort from other processes
         if ($this->checkForAbort() !== null) {
             return true;
         }
     }
     // Filter found URLs by defined rules
     if ($this->follow_redirects_till_content == true) {
         $crawler_status = $this->ProcessCommunication->getCrawlerStatus();
         // If content wasn't found so far and content was found NOW
         if ($crawler_status->first_content_url == null && $PageInfo->http_status_code == 200) {
             $this->ProcessCommunication->updateCrawlerStatus(null, null, $PageInfo->url);
             $this->UrlFilter->setBaseURL($PageInfo->url);
             // Set current page as base-URL
             $this->UrlFilter->filterUrls($PageInfo);
             $this->follow_redirects_till_content = false;
             // Content was found, so this can be set to FALSE
         } else {
             if ($crawler_status->first_content_url == null) {
                 $this->UrlFilter->keepRedirectUrls($PageInfo);
                 // Content wasn't found so far, so just keep redirect-urls
             } else {
                 if ($crawler_status->first_content_url != null) {
                     $this->follow_redirects_till_content = false;
                     $this->UrlFilter->filterUrls($PageInfo);
                 }
             }
         }
     } else {
         $this->UrlFilter->filterUrls($PageInfo);
     }
     // Add Cookies to Cookie-cache
     if ($this->cookie_handling_enabled == true) {
         $this->CookieCache->addCookies($PageInfo->cookies);
     }
     // Add filtered links to URL-cache
     $this->LinkCache->addURLs($PageInfo->links_found_url_descriptors);
     PHPCrawlerBenchmark::stop("processing_url");
     // Complete PageInfo-Object with benchmarks
     $PageInfo->benchmarks = PHPCrawlerBenchmark::getAllBenchmarks();
     if ($this->multiprocess_mode == PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) {
         $this->DocumentInfoQueue->addDocumentInfo($PageInfo);
     }
     // Mark URL as "followed"
     $this->LinkCache->markUrlAsFollowed($UrlDescriptor);
     PHPCrawlerBenchmark::resetAll(array("crawling_process"));
     return false;
 }