/** * Receives and processes the given URL * * @param PHPCrawlerURLDescriptor $UrlDescriptor The URL as PHPCrawlerURLDescriptor-object * @return bool TURE if the crawling-process should be aborted after processig the URL, otherwise FALSE. */ protected function processUrl(PHPCrawlerURLDescriptor $UrlDescriptor) { // Check for abortion from other processes first if mode is MPMODE_CHILDS_EXECUTES_USERCODE if ($this->multiprocess_mode == PHPCrawlerMultiProcessModes::MPMODE_CHILDS_EXECUTES_USERCODE) { // Check for abortion (any limit reached?) if ($this->checkForAbort() !== null) { return true; } } PHPCrawlerBenchmark::start("processing_url"); // Setup HTTP-request $this->PageRequest->setUrl($UrlDescriptor); // Add cookies to request if ($this->cookie_handling_enabled == true) { $this->PageRequest->addCookieDescriptors($this->CookieCache->getCookiesForUrl($UrlDescriptor->url_rebuild)); } // Add basic-authentications to request $authentication = $this->UserSendDataCache->getBasicAuthenticationForUrl($UrlDescriptor->url_rebuild); if ($authentication != null) { $this->PageRequest->setBasicAuthentication($authentication["username"], $authentication["password"]); } // Add post-data to request $post_data = $this->UserSendDataCache->getPostDataForUrl($UrlDescriptor->url_rebuild); while (list($post_key, $post_value) = @each($post_data)) { $this->PageRequest->addPostData($post_key, $post_value); } // Do request $this->delayRequest(); $PageInfo = $this->PageRequest->sendRequest(); // Remove post and cookie-data from request-object $this->PageRequest->clearCookies(); $this->PageRequest->clearPostData(); // Complete PageInfo-Object with benchmarks PHPCrawlerBenchmark::stop("processing_url"); $PageInfo->benchmarks = PHPCrawlerBenchmark::getAllBenchmarks(); // Call user-methods, update craler-status and check for abortion here if crawler doesn't run in MPMODE_PARENT_EXECUTES_USERCODE if ($this->multiprocess_mode != PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) { // Check for abortion (any limit reached?) if ($this->checkForAbort() !== null) { return true; } // Update crawler-status $this->CrawlerStatusHandler->updateCrawlerStatus($PageInfo); $user_abort = false; // If defined by user -> call old handlePageData-method, otherwise don't (because of high memory-usage) if (method_exists($this, "handlePageData")) { $page_info = $PageInfo->toArray(); $user_return_value = $this->handlePageData($page_info); if ($user_return_value < 0) { $user_abort = true; } } // Call the "abstract" method handleDocumentInfo $user_return_value = $this->handleDocumentInfo($PageInfo); if ($user_return_value < 0) { $user_abort = true; } // Update status if user aborted process if ($user_abort == true) { $this->CrawlerStatusHandler->updateCrawlerStatus(null, PHPCrawlerAbortReasons::ABORTREASON_USERABORT); } // Check for abortion again (any limit reached?) if ($this->checkForAbort() !== null) { return true; } } // Add document to the DocumentInfoQueue if mode is MPMODE_PARENT_EXECUTES_USERCODE if ($this->multiprocess_mode == PHPCrawlerMultiProcessModes::MPMODE_PARENT_EXECUTES_USERCODE) { $this->DocumentInfoQueue->addDocumentInfo($PageInfo); } // Filter found URLs by defined rules if ($this->follow_redirects_till_content == true) { $crawler_status = $this->CrawlerStatusHandler->getCrawlerStatus(); // If content wasn't found so far and content was found NOW if ($crawler_status->first_content_url == null && $PageInfo->http_status_code == 200) { $this->CrawlerStatusHandler->updateCrawlerStatus(null, null, $PageInfo->url); $this->UrlFilter->setBaseURL($PageInfo->url); // Set current page as base-URL $this->UrlFilter->filterUrls($PageInfo); $this->follow_redirects_till_content = false; // Content was found, so this can be set to FALSE } else { if ($crawler_status->first_content_url == null) { $this->UrlFilter->keepRedirectUrls($PageInfo, true); // Content wasn't found so far, so just keep redirect-urls and // decrease lindepth } else { if ($crawler_status->first_content_url != null) { $this->follow_redirects_till_content = false; $this->UrlFilter->filterUrls($PageInfo); } } } } else { $this->UrlFilter->filterUrls($PageInfo); } // Add Cookies to Cookie-cache if ($this->cookie_handling_enabled == true) { $this->CookieCache->addCookies($PageInfo->cookies); } // Add filtered links to URL-cache $this->LinkCache->addURLs($PageInfo->links_found_url_descriptors); // Mark URL as "followed" $this->LinkCache->markUrlAsFollowed($UrlDescriptor); PHPCrawlerBenchmark::resetAll(array("crawling_process")); return false; }