function parseurl_getsiteinfo($url) { $siteinfo = array(); $result = z_fetch_url($url, false, 0, array('novalidate' => true)); if (!$result['success']) { return $siteinfo; } $header = $result['header']; $body = $result['body']; $body = mb_convert_encoding($body, 'UTF-8', 'UTF-8'); $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8"); $doc = new DOMDocument(); @$doc->loadHTML($body); deletexnode($doc, 'style'); deletexnode($doc, 'script'); deletexnode($doc, 'option'); deletexnode($doc, 'h1'); deletexnode($doc, 'h2'); deletexnode($doc, 'h3'); deletexnode($doc, 'h4'); deletexnode($doc, 'h5'); deletexnode($doc, 'h6'); deletexnode($doc, 'ol'); deletexnode($doc, 'ul'); $xpath = new DomXPath($doc); //$list = $xpath->query("head/title"); $list = $xpath->query("//title"); foreach ($list as $node) { $siteinfo["title"] = html_entity_decode($node->nodeValue, ENT_QUOTES, "UTF-8"); } //$list = $xpath->query("head/meta[@name]"); $list = $xpath->query("//meta[@name]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"); switch (strtolower($attr["name"])) { case 'generator': $siteinfo['generator'] = $attr['content']; break; case "fulltitle": $siteinfo["title"] = $attr["content"]; break; case "description": $siteinfo["text"] = $attr["content"]; break; case "dc.title": $siteinfo["title"] = $attr["content"]; break; case "dc.description": $siteinfo["text"] = $attr["content"]; break; } } //$list = $xpath->query("head/meta[@property]"); $list = $xpath->query("//meta[@property]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"); switch (strtolower($attr["property"])) { case "og:image": $siteinfo["image"] = $attr["content"]; break; case "og:title": $siteinfo["title"] = $attr["content"]; break; case "og:description": $siteinfo["text"] = $attr["content"]; break; } } if ($siteinfo["image"] == "") { $list = $xpath->query("//img[@src]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $src = completeurl($attr["src"], $url); $photodata = @getimagesize($src); if ($photodata && $photodata[0] > 150 and $photodata[1] > 150) { if ($photodata[0] > 300) { $photodata[1] = round($photodata[1] * (300 / $photodata[0])); $photodata[0] = 300; } if ($photodata[1] > 300) { $photodata[0] = round($photodata[0] * (300 / $photodata[1])); $photodata[1] = 300; } $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]); } } } else { $src = completeurl($siteinfo["image"], $url); unset($siteinfo["image"]); $photodata = @getimagesize($src); if ($photodata && $photodata[0] > 10 and $photodata[1] > 10) { $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]); } } if ($siteinfo["text"] == "") { $text = ""; $list = $xpath->query("//div[@class='article']"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } if ($text == "") { $list = $xpath->query("//div[@class='content']"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } } // If none text was found then take the paragraph content if ($text == "") { $list = $xpath->query("//p"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } } if ($text != "") { $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text)); while (strpos($text, " ")) { $text = trim(str_replace(" ", " ", $text)); } $siteinfo["text"] = html_entity_decode(substr($text, 0, 350), ENT_QUOTES, "UTF-8") . '...'; } } return $siteinfo; }
function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) { require_once "include/network.php"; $a = get_app(); $siteinfo = array(); if ($count > 10) { logger("parseurl_getsiteinfo: Endless loop detected for " . $url, LOGGER_DEBUG); return $siteinfo; } $url = trim($url, "'"); $url = trim($url, '"'); $url = original_url($url); $siteinfo["url"] = $url; $siteinfo["type"] = "link"; $stamp1 = microtime(true); $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 1); curl_setopt($ch, CURLOPT_NOBODY, 1); curl_setopt($ch, CURLOPT_TIMEOUT, 3); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent()); $header = curl_exec($ch); $curl_info = @curl_getinfo($ch); $http_code = $curl_info['http_code']; curl_close($ch); $a->save_timestamp($stamp1, "network"); if (($curl_info['http_code'] == "301" or $curl_info['http_code'] == "302" or $curl_info['http_code'] == "303" or $curl_info['http_code'] == "307") and ($curl_info['redirect_url'] != "" or $curl_info['location'] != "")) { if ($curl_info['redirect_url'] != "") { $siteinfo = parseurl_getsiteinfo($curl_info['redirect_url'], $no_guessing, $do_oembed, ++$count); } else { $siteinfo = parseurl_getsiteinfo($curl_info['location'], $no_guessing, $do_oembed, ++$count); } return $siteinfo; } // if the file is too large then exit if ($curl_info["download_content_length"] > 1000000) { return $siteinfo; } // if it isn't a HTML file then exit if ($curl_info["content_type"] != "" and !strstr(strtolower($curl_info["content_type"]), "html")) { return $siteinfo; } if ($do_oembed) { require_once "include/oembed.php"; $oembed_data = oembed_fetch_url($url); if ($oembed_data->type != "error") { $siteinfo["type"] = $oembed_data->type; } if ($oembed_data->type == "link" and $siteinfo["type"] != "photo") { if (isset($oembed_data->title)) { $siteinfo["title"] = $oembed_data->title; } if (isset($oembed_data->description)) { $siteinfo["text"] = trim($oembed_data->description); } if (isset($oembed_data->thumbnail_url)) { $siteinfo["image"] = $oembed_data->thumbnail_url; } } } $stamp1 = microtime(true); // Now fetch the body as well $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 1); curl_setopt($ch, CURLOPT_NOBODY, 0); curl_setopt($ch, CURLOPT_TIMEOUT, 10); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent()); $header = curl_exec($ch); $curl_info = @curl_getinfo($ch); $http_code = $curl_info['http_code']; curl_close($ch); $a->save_timestamp($stamp1, "network"); // Fetch the first mentioned charset. Can be in body or header $charset = ""; if (preg_match('/charset=(.*?)[' . "'" . '"\\s\\n]/', $header, $matches)) { $charset = trim(trim(trim(array_pop($matches)), ';,')); } if ($charset == "") { $charset = "utf-8"; } $pos = strpos($header, "\r\n\r\n"); if ($pos) { $body = trim(substr($header, $pos)); } else { $body = $header; } if ($charset != '' and strtoupper($charset) != "UTF-8") { logger("parseurl_getsiteinfo: detected charset " . $charset, LOGGER_DEBUG); //$body = mb_convert_encoding($body, "UTF-8", $charset); $body = iconv($charset, "UTF-8//TRANSLIT", $body); } $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8"); $doc = new DOMDocument(); @$doc->loadHTML($body); deletenode($doc, 'style'); deletenode($doc, 'script'); deletenode($doc, 'option'); deletenode($doc, 'h1'); deletenode($doc, 'h2'); deletenode($doc, 'h3'); deletenode($doc, 'h4'); deletenode($doc, 'h5'); deletenode($doc, 'h6'); deletenode($doc, 'ol'); deletenode($doc, 'ul'); $xpath = new DomXPath($doc); $list = $xpath->query("//meta[@content]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } if (@$attr["http-equiv"] == 'refresh') { $path = $attr["content"]; $pathinfo = explode(";", $path); $content = ""; foreach ($pathinfo as $value) { if (substr(strtolower($value), 0, 4) == "url=") { $content = substr($value, 4); } } if ($content != "") { $siteinfo = parseurl_getsiteinfo($content, $no_guessing, $do_oembed, ++$count); return $siteinfo; } } } //$list = $xpath->query("head/title"); $list = $xpath->query("//title"); foreach ($list as $node) { $siteinfo["title"] = html_entity_decode($node->nodeValue, ENT_QUOTES, "UTF-8"); } //$list = $xpath->query("head/meta[@name]"); $list = $xpath->query("//meta[@name]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8")); if ($attr["content"] != "") { switch (strtolower($attr["name"])) { case "fulltitle": $siteinfo["title"] = $attr["content"]; break; case "description": $siteinfo["text"] = $attr["content"]; break; case "thumbnail": $siteinfo["image"] = $attr["content"]; break; case "twitter:image": $siteinfo["image"] = $attr["content"]; break; case "twitter:image:src": $siteinfo["image"] = $attr["content"]; break; case "twitter:card": if ($siteinfo["type"] == "" or $attr["content"] == "photo") { $siteinfo["type"] = $attr["content"]; } break; case "twitter:description": $siteinfo["text"] = $attr["content"]; break; case "twitter:title": $siteinfo["title"] = $attr["content"]; break; case "dc.title": $siteinfo["title"] = $attr["content"]; break; case "dc.description": $siteinfo["text"] = $attr["content"]; break; case "keywords": $keywords = explode(",", $attr["content"]); break; case "news_keywords": $keywords = explode(",", $attr["content"]); break; } } if ($siteinfo["type"] == "summary") { $siteinfo["type"] = "link"; } } if (isset($keywords)) { $siteinfo["keywords"] = array(); foreach ($keywords as $keyword) { $siteinfo["keywords"][] = trim($keyword); } } //$list = $xpath->query("head/meta[@property]"); $list = $xpath->query("//meta[@property]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8")); if ($attr["content"] != "") { switch (strtolower($attr["property"])) { case "og:image": $siteinfo["image"] = $attr["content"]; break; case "og:title": $siteinfo["title"] = $attr["content"]; break; case "og:description": $siteinfo["text"] = $attr["content"]; break; } } } if (@$siteinfo["image"] == "" and !$no_guessing) { $list = $xpath->query("//img[@src]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $src = completeurl($attr["src"], $url); $photodata = @getimagesize($src); if ($photodata && $photodata[0] > 150 and $photodata[1] > 150) { if ($photodata[0] > 300) { $photodata[1] = round($photodata[1] * (300 / $photodata[0])); $photodata[0] = 300; } if ($photodata[1] > 300) { $photodata[0] = round($photodata[0] * (300 / $photodata[1])); $photodata[1] = 300; } $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]); } } } else { $src = completeurl($siteinfo["image"], $url); unset($siteinfo["image"]); $photodata = @getimagesize($src); if ($photodata && $photodata[0] > 10 and $photodata[1] > 10) { $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]); } } if (@$siteinfo["text"] == "" and @$siteinfo["title"] != "" and !$no_guessing) { $text = ""; $list = $xpath->query("//div[@class='article']"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } if ($text == "") { $list = $xpath->query("//div[@class='content']"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } } // If none text was found then take the paragraph content if ($text == "") { $list = $xpath->query("//p"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } } if ($text != "") { $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text)); while (strpos($text, " ")) { $text = trim(str_replace(" ", " ", $text)); } $siteinfo["text"] = trim(html_entity_decode(substr($text, 0, 350), ENT_QUOTES, "UTF-8") . '...'); } } logger("parseurl_getsiteinfo: Siteinfo for " . $url . " " . print_r($siteinfo, true), LOGGER_DEBUG); call_hooks('getsiteinfo', $siteinfo); return $siteinfo; }
function parseurl_getsiteinfo($url) { $siteinfo = array(); $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 1); curl_setopt($ch, CURLOPT_NOBODY, 0); curl_setopt($ch, CURLOPT_TIMEOUT, 3); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt($ch, CURLOPT_USERAGENT, 'Opera/9.64(Windows NT 5.1; U; de) Presto/2.1.1'); $header = curl_exec($ch); $curl_info = @curl_getinfo($ch); $http_code = $curl_info['http_code']; curl_close($ch); if (($curl_info['http_code'] == "301" or $curl_info['http_code'] == "302") and ($curl_info['redirect_url'] != "" or $curl_info['location'] != "")) { if ($curl_info['redirect_url'] != "") { $siteinfo = parseurl_getsiteinfo($curl_info['redirect_url']); } else { $siteinfo = parseurl_getsiteinfo($curl_info['location']); } return $siteinfo; } // Fetch the first mentioned charset. Can be in body or header if (preg_match('/charset=(.*?)[' . "'" . '"\\s\\n]/', $header, $matches)) { $charset = trim(array_pop($matches)); } else { $charset = "utf-8"; } $pos = strpos($header, "\r\n\r\n"); if ($pos) { $body = trim(substr($header, $pos)); } else { $body = $header; } $body = mb_convert_encoding($body, "UTF-8", $charset); $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8"); $doc = new DOMDocument(); @$doc->loadHTML($body); deletenode($doc, 'style'); deletenode($doc, 'script'); deletenode($doc, 'option'); deletenode($doc, 'h1'); deletenode($doc, 'h2'); deletenode($doc, 'h3'); deletenode($doc, 'h4'); deletenode($doc, 'h5'); deletenode($doc, 'h6'); deletenode($doc, 'ol'); deletenode($doc, 'ul'); $xpath = new DomXPath($doc); $list = $xpath->query("//meta[@content]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } if (@$attr["http-equiv"] == 'refresh') { $path = $attr["content"]; $pathinfo = explode(";", $path); $content = ""; foreach ($pathinfo as $value) { if (substr(strtolower($value), 0, 4) == "url=") { $content = substr($value, 4); } } if ($content != "") { $siteinfo = parseurl_getsiteinfo($content); return $siteinfo; } } } //$list = $xpath->query("head/title"); $list = $xpath->query("//title"); foreach ($list as $node) { $siteinfo["title"] = html_entity_decode($node->nodeValue, ENT_QUOTES, "UTF-8"); } //$list = $xpath->query("head/meta[@name]"); $list = $xpath->query("//meta[@name]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"); switch (strtolower($attr["name"])) { case "fulltitle": $siteinfo["title"] = $attr["content"]; break; case "description": $siteinfo["text"] = $attr["content"]; break; case "dc.title": $siteinfo["title"] = $attr["content"]; break; case "dc.description": $siteinfo["text"] = $attr["content"]; break; } } //$list = $xpath->query("head/meta[@property]"); $list = $xpath->query("//meta[@property]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"); switch (strtolower($attr["property"])) { case "og:image": $siteinfo["image"] = $attr["content"]; break; case "og:title": $siteinfo["title"] = $attr["content"]; break; case "og:description": $siteinfo["text"] = $attr["content"]; break; } } if (@$siteinfo["image"] == "") { $list = $xpath->query("//img[@src]"); foreach ($list as $node) { $attr = array(); if ($node->attributes->length) { foreach ($node->attributes as $attribute) { $attr[$attribute->name] = $attribute->value; } } $src = completeurl($attr["src"], $url); $photodata = @getimagesize($src); if ($photodata && $photodata[0] > 150 and $photodata[1] > 150) { if ($photodata[0] > 300) { $photodata[1] = round($photodata[1] * (300 / $photodata[0])); $photodata[0] = 300; } if ($photodata[1] > 300) { $photodata[0] = round($photodata[0] * (300 / $photodata[1])); $photodata[1] = 300; } $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]); } } } else { $src = completeurl($siteinfo["image"], $url); unset($siteinfo["image"]); $photodata = @getimagesize($src); if ($photodata && $photodata[0] > 10 and $photodata[1] > 10) { $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]); } } if (@$siteinfo["text"] == "") { $text = ""; $list = $xpath->query("//div[@class='article']"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } if ($text == "") { $list = $xpath->query("//div[@class='content']"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } } // If none text was found then take the paragraph content if ($text == "") { $list = $xpath->query("//p"); foreach ($list as $node) { if (strlen($node->nodeValue) > 40) { $text .= " " . trim($node->nodeValue); } } } if ($text != "") { $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text)); while (strpos($text, " ")) { $text = trim(str_replace(" ", " ", $text)); } $siteinfo["text"] = html_entity_decode(substr($text, 0, 350), ENT_QUOTES, "UTF-8") . '...'; } } return $siteinfo; }