/**
* Builds an inverted index shard (word --> {docs it appears in})
* for the current batch of SEEN_URLS_BEFORE_UPDATE_SCHEDULER many pages.
* This inverted index shard is then merged by a queue_server
* into the inverted index of the current generation of the crawl.
* The complete inverted index for the whole crawl is built out of these
* inverted indexes for generations. The point of computing a partial
* inverted index on the fetcher is to reduce some of the computational
* burden on the queue server. The resulting mini index computed by
* buildMiniInvertedIndex() is stored in
* $this->found_sites[self::INVERTED_INDEX]
*
*/
function buildMiniInvertedIndex()
{
$start_time = microtime();
$keypad = "";
crawlLog(" Start building mini inverted index ... Current Memory:" . memory_get_usage());
$num_seen = count($this->found_sites[self::SEEN_URLS]);
$this->num_seen_sites += $num_seen;
/*
for the fetcher we are not saving the index shards so
name doesn't matter.
*/
if (!isset($this->found_sites[self::INVERTED_INDEX][$this->current_server])) {
$this->found_sites[self::INVERTED_INDEX][$this->current_server] = new IndexShard("fetcher_shard_{$this->current_server}");
}
for ($i = 0; $i < $num_seen; $i++) {
$interim_time = microtime();
$site = $this->found_sites[self::SEEN_URLS][$i];
if (!isset($site[self::HASH]) || isset($site[self::ROBOT_METAS]) && in_array("JUSTFOLLOW", $site[self::ROBOT_METAS])) {
continue;
}
$doc_rank = false;
if ($this->crawl_type == self::ARCHIVE_CRAWL && isset($this->archive_iterator)) {
$doc_rank = $this->archive_iterator->weight($site);
}
if (isset($site[self::TYPE]) && $site[self::TYPE] == "link") {
$is_link = true;
$doc_keys = $site[self::HTTP_CODE];
$site_url = $site[self::TITLE];
$host = UrlParser::getHost($site_url);
$link_parts = explode('|', $site[self::HASH]);
if (isset($link_parts[5])) {
$link_origin = $link_parts[5];
} else {
$link_origin = $site_url;
}
$meta_ids = PhraseParser::calculateLinkMetas($site_url, $host, $site[self::DESCRIPTION], $link_origin);
} else {
$is_link = false;
$site_url = str_replace('|', "%7C", $site[self::URL]);
$host = UrlParser::getHost($site_url);
$doc_keys = crawlHash($site_url, true) . $site[self::HASH] . "d" . substr(crawlHash($host . "/", true), 1);
$meta_ids = PhraseParser::calculateMetas($site, $this->video_sources);
}
$word_lists = array();
/*
self::JUST_METAS check to avoid getting sitemaps in results for
popular words
*/
$lang = NULL;
if (!isset($site[self::JUST_METAS])) {
$host_words = UrlParser::getWordsIfHostUrl($site_url);
$path_words = UrlParser::getWordsLastPathPartUrl($site_url);
if ($is_link) {
$phrase_string = $site[self::DESCRIPTION];
} else {
if (isset($site[self::LANG])) {
if (isset($this->programming_language_extension[$site[self::LANG]])) {
$phrase_string = $site[self::DESCRIPTION];
} else {
$phrase_string = $host_words . " " . $site[self::TITLE] . " " . $path_words . " " . $site[self::DESCRIPTION];
}
} else {
$phrase_string = $host_words . " " . $site[self::TITLE] . " " . $path_words . " " . $site[self::DESCRIPTION];
}
}
if (isset($site[self::LANG])) {
$lang = guessLocaleFromString(mb_substr($site[self::DESCRIPTION], 0, AD_HOC_TITLE_LENGTH), $site[self::LANG]);
}
$word_lists = PhraseParser::extractPhrasesInLists($phrase_string, $lang);
$len = strlen($phrase_string);
if (isset($this->programming_language_extension[$lang]) || PhraseParser::computeSafeSearchScore($word_lists, $len) < 0.012) {
$meta_ids[] = "safe:true";
$safe = true;
} else {
$meta_ids[] = "safe:false";
$safe = false;
}
}
if (!$is_link) {
//store inlinks so they can be searched by
$num_links = count($site[self::LINKS]);
if ($num_links > 0) {
$link_rank = false;
if ($doc_rank !== false) {
$link_rank = max($doc_rank - 1, 1);
}
} else {
$link_rank = false;
}
}
$num_queue_servers = count($this->queue_servers);
if (isset($site[self::USER_RANKS]) && count($site[self::USER_RANKS]) > 0) {
$score_keys = "";
foreach ($site[self::USER_RANKS] as $label => $score) {
$score_keys .= packInt($score);
}
if (strlen($score_keys) % 8 != 0) {
$score_keys .= $keypad;
}
$doc_keys .= $score_keys;
}
$this->found_sites[self::INVERTED_INDEX][$this->current_server]->addDocumentWords($doc_keys, self::NEEDS_OFFSET_FLAG, $word_lists, $meta_ids, PhraseParser::$materialized_metas, true, $doc_rank);
/*
$this->no_process_links is set when doing things like
mix recrawls. In this case links likely already will appear
in what indexing, so don't index again. $site[self::JUST_META]
is set when have a sitemap or robots.txt (this case set later).
In this case link info is not particularly useful for indexing
and can greatly slow building inverted index.
*/
if (!$this->no_process_links && !isset($site[self::JUST_METAS]) && !isset($this->programming_language_extension[$lang])) {
foreach ($site[self::LINKS] as $url => $link_text) {
/* this mysterious check means won't index links from
robots.txt. Sitemap will still be in TO_CRAWL, but that's
done elsewhere
*/
if (strlen($url) == 0 || is_numeric($url)) {
continue;
}
$link_host = UrlParser::getHost($url);
if (strlen($link_host) == 0) {
continue;
}
$part_num = calculatePartition($link_host, $num_queue_servers);
$summary = array();
if (!isset($this->found_sites[self::LINK_SEEN_URLS][$part_num])) {
$this->found_sites[self::LINK_SEEN_URLS][$part_num] = array();
}
$elink_flag = $link_host != $host ? true : false;
$link_text = strip_tags($link_text);
$ref = $elink_flag ? "eref" : "iref";
$url = str_replace('|', "%7C", $url);
$link_id = "url|" . $url . "|text|" . urlencode($link_text) . "|{$ref}|" . $site_url;
$elink_flag_string = $elink_flag ? "e" : "i";
$link_keys = crawlHash($url, true) . crawlHash($link_id, true) . $elink_flag_string . substr(crawlHash($host . "/", true), 1);
$summary[self::URL] = $link_id;
$summary[self::TITLE] = $url;
// stripping html to be on the safe side
$summary[self::DESCRIPTION] = $link_text;
$summary[self::TIMESTAMP] = $site[self::TIMESTAMP];
$summary[self::ENCODING] = $site[self::ENCODING];
$summary[self::HASH] = $link_id;
$summary[self::TYPE] = "link";
$summary[self::HTTP_CODE] = $link_keys;
$summary[self::LANG] = $lang;
$this->found_sites[self::LINK_SEEN_URLS][$part_num][] = $summary;
$link_word_lists = PhraseParser::extractPhrasesInLists($link_text, $lang);
$link_meta_ids = PhraseParser::calculateLinkMetas($url, $link_host, $link_text, $site_url);
if (!isset($this->found_sites[self::INVERTED_INDEX][$part_num])) {
$this->found_sites[self::INVERTED_INDEX][$part_num] = new IndexShard("fetcher_shard_{$part_num}");
}
$this->found_sites[self::INVERTED_INDEX][$part_num]->addDocumentWords($link_keys, self::NEEDS_OFFSET_FLAG, $link_word_lists, $link_meta_ids, PhraseParser::$materialized_metas, false, $link_rank);
}
}
$iterim_elapse = changeInMicrotime($interim_time);
if ($iterim_elapse > 5) {
crawlLog("..Inverting " . $site[self::URL] . "...took > 5s.");
}
crawlTimeoutLog("..Still building inverted index. Have processed " . "%s of %s documents.\nLast url processed was %s.", $i, $num_seen, $site[self::URL]);
}
if ($this->crawl_type == self::ARCHIVE_CRAWL) {
$this->recrawl_check_scheduler = true;
}
crawlLog(" Build mini inverted index time " . changeInMicrotime($start_time));
}
/**
* Used to recompute both the index shards and the dictionary
* of an index archive. The first step involves re-extracting the
* word into an inverted index from the summaries' web_archives.
* Then a reindex is done.
*
* @param string $archive_path file path to a IndexArchiveBundle
*/
function rebuildIndexArchive($archive_path)
{
$archive_type = $this->getArchiveKind($archive_path);
if ($archive_type != "IndexArchiveBundle") {
$this->badFormatMessageAndExit($archive_path);
}
$info = $archive_type::getArchiveInfo($archive_path);
$tmp = unserialize($info["DESCRIPTION"]);
$video_sources = $tmp[self::VIDEO_SOURCES];
$generation_info = unserialize(file_get_contents("{$archive_path}/generation.txt"));
$num_generations = $generation_info['ACTIVE'] + 1;
$archive = new WebArchiveBundle($archive_path . "/summaries");
$seen = 0;
$generation = 0;
$keypad = "";
while ($generation < $num_generations) {
$partition = $archive->getPartition($generation, false);
$shard_name = $archive_path . "/posting_doc_shards/index{$generation}";
crawlLog("Processing partition {$generation}");
if (file_exists($shard_name)) {
crawlLog("..Unlinking old shard {$generation}");
@unlink($shard_name);
}
$shard = new IndexShard($shard_name, $generation, NUM_DOCS_PER_GENERATION, true);
$seen_partition = 0;
while ($seen_partition < $partition->count) {
$num_to_get = min($partition->count - $seen_partition, 8000);
$offset = $partition->iterator_pos;
$objects = $partition->nextObjects($num_to_get);
$cnt = 0;
foreach ($objects as $object) {
$cnt++;
$site = $object[1];
if (isset($site[self::TYPE]) && $site[self::TYPE] == "link") {
$is_link = true;
$doc_keys = $site[self::HTTP_CODE];
$site_url = $site[self::TITLE];
$host = UrlParser::getHost($site_url);
$link_parts = explode('|', $site[self::HASH]);
if (isset($link_parts[5])) {
$link_origin = $link_parts[5];
} else {
$link_origin = $site_url;
}
$meta_ids = PhraseParser::calculateLinkMetas($site_url, $host, $site[self::DESCRIPTION], $link_origin);
$link_to = "LINK TO:";
} else {
$is_link = false;
$site_url = str_replace('|', "%7C", $site[self::URL]);
$host = UrlParser::getHost($site_url);
$doc_keys = crawlHash($site_url, true) . $site[self::HASH] . "d" . substr(crawlHash($host . "/", true), 1);
$meta_ids = PhraseParser::calculateMetas($site, $video_sources);
$link_to = "";
}
$so_far_cnt = $seen_partition + $cnt;
$time_out_message = "..still processing {$so_far_cnt} " . "of {$partition->count} in partition {$generation}." . "\n..Last processed was: " . ($seen + 1) . ". {$link_to}{$site_url}. ";
crawlTimeoutLog($time_out_message);
$seen++;
$word_lists = array();
/*
self::JUST_METAS check to avoid getting sitemaps in
results for popular words
*/
$lang = NULL;
if (!isset($site[self::JUST_METAS])) {
$host_words = UrlParser::getWordsIfHostUrl($site_url);
$path_words = UrlParser::getWordsLastPathPartUrl($site_url);
if ($is_link) {
$phrase_string = $site[self::DESCRIPTION];
} else {
$phrase_string = $host_words . " " . $site[self::TITLE] . " " . $path_words . " " . $site[self::DESCRIPTION];
}
if (isset($site[self::LANG])) {
$lang = guessLocaleFromString(mb_substr($site[self::DESCRIPTION], 0, AD_HOC_TITLE_LENGTH), $site[self::LANG]);
}
$word_lists = PhraseParser::extractPhrasesInLists($phrase_string, $lang);
$len = strlen($phrase_string);
if (PhraseParser::computeSafeSearchScore($word_lists, $len) < 0.012) {
$meta_ids[] = "safe:true";
$safe = true;
} else {
$meta_ids[] = "safe:false";
$safe = false;
}
}
if (isset($site[self::USER_RANKS]) && count($site[self::USER_RANKS]) > 0) {
$score_keys = "";
foreach ($site[self::USER_RANKS] as $label => $score) {
$score_keys .= packInt($score);
}
if (strlen($score_keys) % 8 != 0) {
$score_keys .= $keypad;
}
$doc_keys .= $score_keys;
}
$shard->addDocumentWords($doc_keys, $offset, $word_lists, $meta_ids, PhraseParser::$materialized_metas, true, false);
$offset = $object[0];
}
$seen_partition += $num_to_get;
}
$shard->save(false, true);
$generation++;
}
$this->reindexIndexArchive($archive_path);
}
