コード例 #1
0
ファイル: bmp_processor.php プロジェクト: yakar/yioop
 /**
  * {@inheritDoc}
  *
  * @param string $page  the image represented as a character string
  * @param string $url  the url where the image was downloaded from
  * @return array summary information including a thumbnail and a
  *     description (where the description is just the url)
  */
 function process($page, $url)
 {
     if (is_string($page)) {
         $image = $this->imagecreatefrombmp($page);
         $thumb_string = self::createThumb($image);
         $summary[self::TITLE] = "";
         $summary[self::DESCRIPTION] = "Image of " . UrlParser::getDocumentFilename($url);
         $summary[self::LINKS] = array();
         $summary[self::PAGE] = "<html><body><div><img src='data:image/bmp;base64," . base64_encode($page) . "' alt='" . $summary[self::DESCRIPTION] . "' /></div></body></html>";
         $summary[self::THUMB] = 'data:image/jpeg;base64,' . base64_encode($thumb_string);
     }
     return $summary;
 }
コード例 #2
0
ファイル: fetcher.php プロジェクト: yakar/yioop
 /**
  * Prepare an array of up to NUM_MULTI_CURL_PAGES' worth of sites to be
  * downloaded in one go using the to_crawl array. Delete these sites
  * from the to_crawl array.
  *
  * @return array sites which are ready to be downloaded
  */
 function getFetchSites()
 {
     $web_archive = $this->web_archive;
     $start_time = microtime();
     $seeds = array();
     $delete_indices = array();
     $num_items = count($this->to_crawl);
     if ($num_items > 0) {
         $crawl_source =& $this->to_crawl;
         $to_crawl_flag = true;
     } else {
         crawlLog("...Trying to crawl sites which failed the first time");
         $num_items = count($this->to_crawl_again);
         $crawl_source =& $this->to_crawl_again;
         $to_crawl_flag = false;
     }
     reset($crawl_source);
     if ($num_items > NUM_MULTI_CURL_PAGES) {
         $num_items = NUM_MULTI_CURL_PAGES;
     }
     //DNS lookups take longer so try to get fewer in one go
     $num_ip_lookups = max($num_items / 3, 2);
     $i = 0;
     $ip_lookup_cnt = 0;
     $site_pair = each($crawl_source);
     while ($site_pair !== false && $i < $num_items && $ip_lookup_cnt < $num_ip_lookups) {
         $delete_indices[] = $site_pair['key'];
         if ($site_pair['value'][0] != self::DUMMY) {
             $host = UrlParser::getHost($site_pair['value'][0]);
             if (!strpos($site_pair['value'][0], "###")) {
                 $ip_lookup_cnt++;
             }
             // only download if host doesn't seem congested
             if (!isset($this->hosts_with_errors[$host]) || $this->hosts_with_errors[$host] < DOWNLOAD_ERROR_THRESHOLD) {
                 $url_to_check = $site_pair['value'][0];
                 $extension = UrlParser::getDocumentType($url_to_check);
                 $repository_indicator = FetchGitRepositoryUrls::checkForRepository($extension);
                 if ($repository_indicator == self::REPOSITORY_GIT) {
                     $git_internal_urls = FetchGitRepositoryUrls::setGitRepositoryUrl($url_to_check, $i, $seeds, $repository_indicator, $site_pair, $this->total_git_urls, $this->all_git_urls);
                     $i = $git_internal_urls['position'];
                     $git_url_index = $git_internal_urls['index'];
                     $seeds = $git_internal_urls['seeds'];
                     $repository_indicator = $git_internal_urls['indicator'];
                     $this->total_git_urls = $git_internal_urls['count'];
                     $this->all_git_urls = $git_internal_urls['all'];
                 } else {
                     $seeds[$i][self::URL] = $site_pair['value'][0];
                     $seeds[$i][self::WEIGHT] = $site_pair['value'][1];
                     $seeds[$i][self::CRAWL_DELAY] = $site_pair['value'][2];
                 }
                 /*
                   Crawl delay is only used in scheduling on the queue_server
                   on the fetcher, we only use crawl-delay to determine
                   if we will give a page a second try if it doesn't
                   download the first time
                 */
                 if (UrlParser::getDocumentFilename($seeds[$i][self::URL]) . "." . UrlParser::getDocumentType($seeds[$i][self::URL]) == "robots.txt") {
                     $seeds[$i][self::ROBOT_PATHS] = array();
                 }
                 $i++;
             }
         } else {
             break;
         }
         $site_pair = each($crawl_source);
     }
     //end while
     foreach ($delete_indices as $delete_index) {
         $git_set = false;
         if ($to_crawl_flag == true) {
             $extension = UrlParser::getDocumentType($this->to_crawl[$delete_index][0]);
             $repository_type = FetchGitRepositoryUrls::checkForRepository($extension);
             if ($repository_type != self::REPOSITORY_GIT) {
                 unset($this->to_crawl[$delete_index]);
             }
         } else {
             $extension = UrlParser::getDocumentType($this->to_crawl_again[$delete_index][0]);
             $repository_type = FetchGitRepositoryUrls::checkForRepository($extension);
             unset($this->to_crawl_again[$delete_index]);
         }
         if ($repository_type == self::REPOSITORY_GIT) {
             if (!$git_set) {
                 $next_url_start = $url_to_check . self::GIT_URL_CONTINUE . $git_url_index;
                 $git_set = true;
                 $this->to_crawl[$delete_index][0] = $next_url_start;
             }
             if ($repository_indicator == self::INDICATOR_NONE) {
                 unset($this->to_crawl[$delete_index]);
             }
         }
     }
     crawlLog("Fetch url list to download time " . changeInMicrotime($start_time));
     return $seeds;
 }
コード例 #3
0
ファイル: fetch_url.php プロジェクト: yakar/yioop
 /**
  * Make multi_curl requests for an array of sites with urls or onion urls
  *
  * @param array $sites  an array containing urls of pages to request
  * @param bool $timer  flag, true means print timing statistics to log
  * @param int $page_range_request maximum number of bytes to download/page
  *     0 means download all
  * @param string $temp_dir folder to store temporary ip header info
  * @param string $key  the component of $sites[$i] that has the value of
  *     a url to get defaults to URL
  * @param string $value component of $sites[$i] in which to store the
  *     page that was gotten
  * @param bool $minimal if true do a faster request of pages by not
  *     doing things like extract HTTP headers sent, etcs
  * @param array $post_data data to be POST'd to each site
  * @param bool $follow whether to follow redirects or not
  * @param string $tor_proxy url of a proxy that knows how to download
  *     .onion urls
  * @param array $proxy_servers if not array(), then an array of proxy
  *     server to use rather than to directly download web pages from
  *     the current machine
  *
  * @return array an updated array with the contents of those pages
  */
 static function getPages($sites, $timer = false, $page_range_request = PAGE_RANGE_REQUEST, $temp_dir = NULL, $key = CrawlConstants::URL, $value = CrawlConstants::PAGE, $minimal = false, $post_data = NULL, $follow = false, $tor_proxy = "", $proxy_servers = array())
 {
     $agent_handler = curl_multi_init();
     $active = NULL;
     $start_time = microtime();
     if (!$minimal && $temp_dir == NULL) {
         $temp_dir = CRAWL_DIR . "/temp";
         if (!file_exists($temp_dir)) {
             mkdir($temp_dir);
         }
     }
     //Set-up requests
     $num_sites = count($sites);
     for ($i = 0; $i < $num_sites; $i++) {
         $is_gopher = false;
         $sites[$i][CrawlConstants::IS_GOPHER_URL] = $is_gopher;
         if (isset($sites[$i][$key])) {
             list($sites[$i][$key], $url, $headers) = self::prepareUrlHeaders($sites[$i][$key], $minimal, $proxy_servers);
             if ($headers == "gopher") {
                 $is_gopher = true;
                 $sites[$i][CrawlConstants::IS_GOPHER_URL] = $is_gopher;
                 $headers = array();
             }
             $sites[$i][0] = curl_init();
             if (!$minimal) {
                 $ip_holder[$i] = fopen("{$temp_dir}/tmp{$i}.txt", 'w+');
                 curl_setopt($sites[$i][0], CURLOPT_STDERR, $ip_holder[$i]);
                 curl_setopt($sites[$i][0], CURLOPT_VERBOSE, true);
             }
             curl_setopt($sites[$i][0], CURLOPT_USERAGENT, USER_AGENT);
             curl_setopt($sites[$i][0], CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER);
             curl_setopt($sites[$i][0], CURLOPT_URL, $url);
             if (strcmp(substr($url, -10), "robots.txt") == 0) {
                 $sites[$i]['ROBOT'] = true;
                 $follow = true;
                 /*wikipedia redirects their robot page. grr
                     want to force this for robots pages
                   */
             }
             curl_setopt($sites[$i][0], CURLOPT_FOLLOWLOCATION, $follow);
             curl_setopt($sites[$i][0], CURLOPT_SSL_VERIFYHOST, 0);
             curl_setopt($sites[$i][0], CURLOPT_AUTOREFERER, true);
             curl_setopt($sites[$i][0], CURLOPT_RETURNTRANSFER, true);
             curl_setopt($sites[$i][0], CURLOPT_CONNECTTIMEOUT, PAGE_TIMEOUT);
             curl_setopt($sites[$i][0], CURLOPT_TIMEOUT, PAGE_TIMEOUT);
             if (stripos($url, '.onion') !== false && $tor_proxy != "") {
                 curl_setopt($sites[$i][0], CURLOPT_PROXY, $tor_proxy);
                 //CURLPROXY_SOCKS5_HOSTNAME = 7
                 curl_setopt($sites[$i][0], CURLOPT_PROXYTYPE, 7);
                 if ($timer) {
                     crawlLog("Using Tor proxy for {$url}..");
                 }
             } else {
                 if ($proxy_servers != array() && !$is_gopher) {
                     $select_proxy = rand(0, count($proxy_servers) - 1);
                     $proxy_server = $proxy_servers[$select_proxy];
                     $proxy_parts = explode(":", $proxy_server);
                     $proxy_ip = $proxy_parts[0];
                     if (!isset($proxy_parts[2]) || strtolower($proxy_parts[2]) == 'http') {
                         $proxy_type = CURLPROXY_HTTP;
                     } else {
                         if (strtolower($proxy_parts[2]) == 'socks5') {
                             $proxy_type = CURLPROXY_SOCKS5;
                         } else {
                             $proxy_type = $proxy_parts[2];
                         }
                     }
                     if (isset($proxy_parts[1])) {
                         $proxy_port = $proxy_parts[1];
                     } else {
                         $proxy_port = "80";
                     }
                     curl_setopt($sites[$i][0], CURLOPT_PROXY, "{$proxy_ip}:{$proxy_port}");
                     curl_setopt($sites[$i][0], CURLOPT_PROXYTYPE, $proxy_type);
                     if ($timer) {
                         crawlLog("Selecting proxy {$select_proxy} for {$url}");
                     }
                 }
             }
             if (!$minimal) {
                 curl_setopt($sites[$i][0], CURLOPT_HEADER, true);
             }
             //make lighttpd happier
             if (!$is_gopher) {
                 curl_setopt($sites[$i][0], CURLOPT_HTTPHEADER, $headers);
             }
             curl_setopt($sites[$i][0], CURLOPT_ENCODING, "");
             // ^ need to set for sites like att that use gzip
             if ($page_range_request > 0) {
                 curl_setopt($sites[$i][0], CURLOPT_RANGE, "0-" . $page_range_request);
             }
             if ($post_data != NULL) {
                 curl_setopt($sites[$i][0], CURLOPT_POST, true);
                 curl_setopt($sites[$i][0], CURLOPT_POSTFIELDS, $post_data[$i]);
             }
             curl_multi_add_handle($agent_handler, $sites[$i][0]);
         }
     }
     if ($timer) {
         crawlLog("  Init Get Pages " . changeInMicrotime($start_time));
     }
     $start_time = microtime();
     $start = time();
     //Wait for responses
     $running = NULL;
     $memory_limit = metricToInt(ini_get("memory_limit")) * 0.7;
     do {
         $mrc = curl_multi_exec($agent_handler, $running);
         $ready = curl_multi_select($agent_handler, 0.005);
     } while (memory_get_usage() < $memory_limit && time() - $start < PAGE_TIMEOUT && $running > 0);
     if (time() - $start > PAGE_TIMEOUT && $timer) {
         crawlLog("  TIMED OUT!!!");
     }
     if ($timer) {
         crawlLog("  Page Request time " . changeInMicrotime($start_time));
     }
     $start_time = microtime();
     //Process returned pages
     for ($i = 0; $i < $num_sites; $i++) {
         if ($timer) {
             crawlTimeoutLog("fetch_url initial processing of page %s of %s", $i, $num_sites);
         }
         if (!$minimal && isset($ip_holder[$i])) {
             rewind($ip_holder[$i]);
             $header = fread($ip_holder[$i], 8192);
             $ip_addresses = self::getCurlIp($header);
             fclose($ip_holder[$i]);
         }
         $is_gopher = false;
         if (isset($sites[$i][0]) && $sites[$i][0]) {
             // Get Data and Message Code
             $content = @curl_multi_getcontent($sites[$i][0]);
             $is_gopher = $sites[$i][CrawlConstants::IS_GOPHER_URL];
             /*
                If the Transfer-encoding was chunked then the Range header
                we sent was ignored. So we manually truncate the data
                here
             */
             if ($page_range_request > 0) {
                 $content = substr($content, 0, $page_range_request);
             }
             if (isset($content) && !$minimal && !$is_gopher) {
                 $site = self::parseHeaderPage($content, $value);
                 $sites[$i] = array_merge($sites[$i], $site);
                 if (isset($header)) {
                     $header = substr($header, 0, strpos($header, "\r\n\r\n") + 4);
                 } else {
                     $header = "";
                 }
                 $sites[$i][CrawlConstants::HEADER] = $header . $sites[$i][CrawlConstants::HEADER];
                 unset($header);
             } else {
                 if (isset($content) && !$minimal && $is_gopher) {
                     $sites[$i][CrawlConstants::HEADER] = $header;
                     $sites[$i][$value] = $content;
                     unset($header);
                 } else {
                     $sites[$i][$value] = $content;
                 }
             }
             if (!$minimal) {
                 $sites[$i][self::SIZE] = @curl_getinfo($sites[$i][0], CURLINFO_SIZE_DOWNLOAD);
                 $sites[$i][self::DNS_TIME] = @curl_getinfo($sites[$i][0], CURLINFO_NAMELOOKUP_TIME);
                 $sites[$i][self::TOTAL_TIME] = @curl_getinfo($sites[$i][0], CURLINFO_TOTAL_TIME);
                 $sites[$i][self::HTTP_CODE] = curl_getinfo($sites[$i][0], CURLINFO_HTTP_CODE);
                 if (!$sites[$i][self::HTTP_CODE] && !$is_gopher) {
                     $sites[$i][self::HTTP_CODE] = curl_error($sites[$i][0]);
                 } else {
                     $sites[$i][self::HTTP_CODE] = 200;
                 }
                 if ($ip_addresses) {
                     $sites[$i][self::IP_ADDRESSES] = $ip_addresses;
                 } else {
                     $sites[$i][self::IP_ADDRESSES] = array("0.0.0.0");
                 }
                 //Get Time, Mime type and Character encoding
                 $sites[$i][self::TIMESTAMP] = time();
                 if ($is_gopher) {
                     $path = UrlParser::getPath($sites[$i][self::URL]);
                     $filename = UrlParser::getDocumentFilename($sites[$i][self::URL]);
                     if (isset($path[1])) {
                         $gopher_type = $path[1];
                     } else {
                         $gopher_type = 1;
                     }
                     if ($gopher_type == 1) {
                         $sites[$i][self::TYPE] = "text/gopher";
                     } else {
                         if (in_array($gopher_type, array(0, 3, 6))) {
                             $sites[$i][self::TYPE] = "text/plain";
                             if ($gopher_type == 6) {
                                 $sites[$i][$value] = convert_uudecode($content);
                             }
                         } else {
                             if ($gopher_type == 'h') {
                                 $sites[$i][self::TYPE] = "text/html";
                             } else {
                                 if ($gopher_type == 'g') {
                                     $sites[$i][self::TYPE] = "image/gif";
                                 }
                             }
                         }
                     }
                     $path_info = pathinfo($filename);
                     if (!isset($sites[$i][self::TYPE]) && isset($path_info['extension'])) {
                         $sites[$i][self::TYPE] = UrlParser::guessMimeTypeFromFileName($filename);
                     } else {
                         if (!isset($sites[$i][self::TYPE])) {
                             $sites[$i][self::TYPE] = "unknown";
                         }
                     }
                 } else {
                     $type_parts = explode(";", curl_getinfo($sites[$i][0], CURLINFO_CONTENT_TYPE));
                     $sites[$i][self::TYPE] = strtolower(trim($type_parts[0]));
                 }
             }
             //curl_multi_remove_handle($agent_handler, $sites[$i][0]);
             curl_close($sites[$i][0]);
             if (isset($sites[$i]['ROBOT']) && $sites[$i]['ROBOT']) {
                 if (isset($sites[$i][self::TYPE]) && $sites[$i][self::TYPE] != "text/plain" && isset($sites[$i][CrawlConstants::LOCATION]) && count($site[CrawlConstants::LOCATION]) > 0) {
                     $sites[$i][self::TYPE] = "text/plain";
                     $sites[$i][self::HTTP_CODE] = "200";
                     $tmp = wordwrap($sites[$i][$value], 80);
                     $tmp_parts = explode("\n", $tmp);
                     $tmp = "# Suspect server misconfiguration\n";
                     $tmp .= "# Assume shouldn't crawl this site.\n";
                     $tmp .= "# Pretending got following robots.txt.\n";
                     $tmp .= "User-agent: *\n";
                     $tmp .= "Disallow: /\n";
                     $tmp .= "# Original error code: " . $sites[$i][self::HTTP_CODE] . "\n";
                     $tmp .= "# Original content:\n";
                     foreach ($tmp_parts as $part) {
                         $tmp = "#" . $part . "\n";
                     }
                     $sites[$i][$value] = $tmp;
                     $sites[$i][self::HTTP_CODE] = "200";
                     unset($site[CrawlConstants::LOCATION]);
                 }
             }
         }
         //end big if
     }
     //end for
     if ($timer) {
         crawlLog("  Get Page Content time " . changeInMicrotime($start_time));
     }
     curl_multi_close($agent_handler);
     return $sites;
 }
コード例 #4
0
ファイル: arc_tool.php プロジェクト: yakar/yioop
 /**
  * Used to recompute the dictionary of an index archive -- either from
  * scratch using the index shard data or just using the current dictionary
  * but merging the tiers into one tier
  *
  * @param string $path file path to dictionary of an IndexArchiveBundle
  * @param int $max_tier tier up to which the dictionary tiers should be
  *     merge (typically a value greater than the max_tier of the
  *     dictionary)
  */
 function reindexIndexArchive($path, $max_tier = -1)
 {
     if ($this->getArchiveKind($path) != "IndexArchiveBundle") {
         echo "\n{$path} ...\n" . "  is not an IndexArchiveBundle so cannot be re-indexed\n\n";
         exit;
     }
     $shards = glob($path . "/posting_doc_shards/index*");
     if (is_array($shards)) {
         if ($max_tier == -1) {
             $dbms_manager = DBMS . "Manager";
             $db = new $dbms_manager();
             $db->unlinkRecursive($path . "/dictionary", false);
             IndexDictionary::makePrefixLetters($path . "/dictionary");
         }
         $dictionary = new IndexDictionary($path . "/dictionary");
         if ($max_tier == -1) {
             $max_generation = 0;
             foreach ($shards as $shard_name) {
                 $file_name = UrlParser::getDocumentFilename($shard_name);
                 $generation = (int) substr($file_name, strlen("index"));
                 $max_generation = max($max_generation, $generation);
             }
             for ($i = 0; $i < $max_generation + 1; $i++) {
                 $shard_name = $path . "/posting_doc_shards/index{$i}";
                 echo "\nShard {$i}\n";
                 $shard = new IndexShard($shard_name, $i, NUM_DOCS_PER_GENERATION, true);
                 $dictionary->addShardDictionary($shard);
             }
             $max_tier = $dictionary->max_tier;
         }
         echo "\nFinal Merge Tiers\n";
         $dictionary->mergeAllTiers(NULL, $max_tier);
         $db->setWorldPermissionsRecursive($path . "/dictionary");
         echo "\nReindex complete!!\n";
     } else {
         echo "\n{$path} ...\n" . "  does not contain posting shards so cannot be re-indexed\n\n";
     }
 }
コード例 #5
0
ファイル: resource_controller.php プロジェクト: yakar/yioop
 /**
  * Gets the resource $_REQUEST['n'] from APP_DIR/$_REQUEST['f'] or
  * CRAWL_DIR/$_REQUEST['f']  after cleaning
  */
 function get()
 {
     if (!isset($_REQUEST['n']) || !isset($_REQUEST['f'])) {
         return;
     }
     $name = $this->clean($_REQUEST['n'], "string");
     if (in_array($_REQUEST['f'], array("css", "scripts", "resources"))) {
         /* notice in this case we don't check if request come from a
               legitimate source but we do try to restrict it to being
               a file (not a folder) in the above array
            */
         $base_dir = $this->getBaseFolder();
         if (!$base_dir) {
             header('HTTP/1.1 401 Unauthorized');
             echo "<html><head><title>401 Unauthorized</title></head>" . "<body><p>401 Unauthorized</p></body></html>";
             return;
         }
         $type = UrlParser::getDocumentType($name);
         $name = UrlParser::getDocumentFilename($name);
         $name = $type != "" ? "{$name}.{$type}" : $name;
         if (isset($_REQUEST['t'])) {
             $name .= ".jpg";
         }
     } else {
         if (in_array($_REQUEST['f'], array("cache"))) {
             /*  perform check since these request should come from a known
                     machine
                 */
             if (!$this->checkRequest()) {
                 return;
             }
             $folder = $_REQUEST['f'];
             $base_dir = CRAWL_DIR . "/{$folder}";
         } else {
             return;
         }
     }
     if (isset($_REQUEST['o']) && isset($_REQUEST['l'])) {
         $offset = $this->clean($_REQUEST['o'], "int");
         $limit = $this->clean($_REQUEST['l'], "int");
     }
     $path = "{$base_dir}/{$name}";
     if (file_exists($path)) {
         $mime_type = mimeType($path);
         $size = filesize($path);
         $start = 0;
         $end = $size - 1;
         header("Content-type: {$mime_type}");
         header("Accept-Ranges: bytes");
         if (isset($_SERVER['HTTP_RANGE'])) {
             $this->serveRangeRequest($path, $size, $start, $end);
             return;
         }
         header("Content-Length: " . $size);
         header("Content-Range: bytes {$start}-{$end}/{$size}");
         if (isset($offset) && isset($limit)) {
             echo file_get_contents($path, false, NULL, $offset, $limit);
         } else {
             readfile($path);
         }
     } else {
         header("Location:./error.php");
     }
 }