/**
* Searches for links in the given HTML-chunk and adds found links the the internal link-cache.
*/
public function findLinksInHTMLChunk(&$html_source)
{
PHPCrawlerBenchmark::start("searching_for_links_in_page");
// Check for meta-base-URL and meta-tags in top of HTML-source
if ($this->top_lines_processed == false) {
$meta_base_url = PHPCrawlerUtils::getBaseUrlFromMetaTag($html_source);
if ($meta_base_url != null) {
$base_url = PHPCrawlerUtils::buildURLFromLink($meta_base_url, $this->baseUrlParts);
$this->baseUrlParts = PHPCrawlerUrlPartsDescriptor::fromURL($base_url);
}
// Get all meta-tags
$this->meta_attributes = PHPCrawlerUtils::getMetaTagAttributes($html_source);
// Set flag that top-lines of source were processed
$this->top_lines_processed = true;
}
// Prepare HTML-chunk
$this->prepareHTMLChunk($html_source);
// Build the RegEx-part for html-tags to search links in
$tag_regex_part = "";
$cnt = count($this->extract_tags);
for ($x = 0; $x < $cnt; $x++) {
$tag_regex_part .= "|" . $this->extract_tags[$x];
}
$tag_regex_part = substr($tag_regex_part, 1);
// 1. <a href="...">LINKTEXT</a> (well formed link with </a> at the end and quotes around the link)
// Get the link AND the linktext from these tags
// This has to be done FIRST !!
preg_match_all("#<\\s*a\\s[^<>]*(?<=\\s)(?:" . $tag_regex_part . ")\\s*=\\s*" . "(?|\"([^\"]+)\"|'([^']+)'|([^\\s><'\"]+))[^<>]*>" . "((?:(?!<\\s*\\/a\\s*>).){0,500})" . "<\\s*\\/a\\s*># is", $html_source, $matches);
$cnt = count($matches[0]);
for ($x = 0; $x < $cnt; $x++) {
$link_raw = trim($matches[1][$x]);
$linktext = $matches[2][$x];
$linkcode = trim($matches[0][$x]);
if (!empty($link_raw)) {
$this->addLinkToCache($link_raw, $linkcode, $linktext);
}
}
// Second regex (everything that could be a link inside of <>-tags)
preg_match_all("#<[^<>]*\\s(?:" . $tag_regex_part . ")\\s*=\\s*" . "(?|\"([^\"]+)\"|'([^']+)'|([^\\s><'\"]+))[^<>]*># is", $html_source, $matches);
$cnt = count($matches[0]);
for ($x = 0; $x < $cnt; $x++) {
$link_raw = trim($matches[1][$x]);
$linktext = "";
$linkcode = trim($matches[0][$x]);
if (!empty($link_raw)) {
$this->addLinkToCache($link_raw, $linkcode, $linktext);
}
}
// Now, if agressive_mode is set to true, we look for some
// other things
$pregs = array();
if ($this->aggressive_search == true) {
// Links like "...:url("animage.gif")..."
$pregs[] = "/[\\s\\.:;](?:" . $tag_regex_part . ")\\s*\\(\\s*([\"|']{0,1})([^\"'\\) ]{1,500})['\"\\)]/ is";
// Everything like "...href="bla.html"..." with qoutes
$pregs[] = "/[\\s\\.:;\"'](?:" . $tag_regex_part . ")\\s*=\\s*([\"|'])(.{0,500}?)\\1/ is";
// Everything like "...href=bla.html..." without qoutes
$pregs[] = "/[\\s\\.:;](?:" . $tag_regex_part . ")\\s*(=)\\s*([^\\s\">']{1,500})/ is";
for ($x = 0; $x < count($pregs); $x++) {
unset($matches);
preg_match_all($pregs[$x], $html_source, $matches);
$cnt = count($matches[0]);
for ($y = 0; $y < $cnt; $y++) {
$link_raw = trim($matches[2][$y]);
$linkcode = trim($matches[0][$y]);
$linktext = "";
$this->addLinkToCache($link_raw, $linkcode, $linktext);
}
}
}
$this->found_links_map = array();
PHPCrawlerBenchmark::stop("searching_for_links_in_page");
}
