Esempio n. 1
0
function parseurl_getsiteinfo($url)
{
    $siteinfo = array();
    $result = z_fetch_url($url, false, 0, array('novalidate' => true));
    if (!$result['success']) {
        return $siteinfo;
    }
    $header = $result['header'];
    $body = $result['body'];
    $body = mb_convert_encoding($body, 'UTF-8', 'UTF-8');
    $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");
    $doc = new DOMDocument();
    @$doc->loadHTML($body);
    deletexnode($doc, 'style');
    deletexnode($doc, 'script');
    deletexnode($doc, 'option');
    deletexnode($doc, 'h1');
    deletexnode($doc, 'h2');
    deletexnode($doc, 'h3');
    deletexnode($doc, 'h4');
    deletexnode($doc, 'h5');
    deletexnode($doc, 'h6');
    deletexnode($doc, 'ol');
    deletexnode($doc, 'ul');
    $xpath = new DomXPath($doc);
    //$list = $xpath->query("head/title");
    $list = $xpath->query("//title");
    foreach ($list as $node) {
        $siteinfo["title"] = html_entity_decode($node->nodeValue, ENT_QUOTES, "UTF-8");
    }
    //$list = $xpath->query("head/meta[@name]");
    $list = $xpath->query("//meta[@name]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8");
        switch (strtolower($attr["name"])) {
            case 'generator':
                $siteinfo['generator'] = $attr['content'];
                break;
            case "fulltitle":
                $siteinfo["title"] = $attr["content"];
                break;
            case "description":
                $siteinfo["text"] = $attr["content"];
                break;
            case "dc.title":
                $siteinfo["title"] = $attr["content"];
                break;
            case "dc.description":
                $siteinfo["text"] = $attr["content"];
                break;
        }
    }
    //$list = $xpath->query("head/meta[@property]");
    $list = $xpath->query("//meta[@property]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8");
        switch (strtolower($attr["property"])) {
            case "og:image":
                $siteinfo["image"] = $attr["content"];
                break;
            case "og:title":
                $siteinfo["title"] = $attr["content"];
                break;
            case "og:description":
                $siteinfo["text"] = $attr["content"];
                break;
        }
    }
    if ($siteinfo["image"] == "") {
        $list = $xpath->query("//img[@src]");
        foreach ($list as $node) {
            $attr = array();
            if ($node->attributes->length) {
                foreach ($node->attributes as $attribute) {
                    $attr[$attribute->name] = $attribute->value;
                }
            }
            $src = completeurl($attr["src"], $url);
            $photodata = @getimagesize($src);
            if ($photodata && $photodata[0] > 150 and $photodata[1] > 150) {
                if ($photodata[0] > 300) {
                    $photodata[1] = round($photodata[1] * (300 / $photodata[0]));
                    $photodata[0] = 300;
                }
                if ($photodata[1] > 300) {
                    $photodata[0] = round($photodata[0] * (300 / $photodata[1]));
                    $photodata[1] = 300;
                }
                $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]);
            }
        }
    } else {
        $src = completeurl($siteinfo["image"], $url);
        unset($siteinfo["image"]);
        $photodata = @getimagesize($src);
        if ($photodata && $photodata[0] > 10 and $photodata[1] > 10) {
            $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]);
        }
    }
    if ($siteinfo["text"] == "") {
        $text = "";
        $list = $xpath->query("//div[@class='article']");
        foreach ($list as $node) {
            if (strlen($node->nodeValue) > 40) {
                $text .= " " . trim($node->nodeValue);
            }
        }
        if ($text == "") {
            $list = $xpath->query("//div[@class='content']");
            foreach ($list as $node) {
                if (strlen($node->nodeValue) > 40) {
                    $text .= " " . trim($node->nodeValue);
                }
            }
        }
        // If none text was found then take the paragraph content
        if ($text == "") {
            $list = $xpath->query("//p");
            foreach ($list as $node) {
                if (strlen($node->nodeValue) > 40) {
                    $text .= " " . trim($node->nodeValue);
                }
            }
        }
        if ($text != "") {
            $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text));
            while (strpos($text, "  ")) {
                $text = trim(str_replace("  ", " ", $text));
            }
            $siteinfo["text"] = html_entity_decode(substr($text, 0, 350), ENT_QUOTES, "UTF-8") . '...';
        }
    }
    return $siteinfo;
}
Esempio n. 2
0
function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1)
{
    require_once "include/network.php";
    $a = get_app();
    $siteinfo = array();
    if ($count > 10) {
        logger("parseurl_getsiteinfo: Endless loop detected for " . $url, LOGGER_DEBUG);
        return $siteinfo;
    }
    $url = trim($url, "'");
    $url = trim($url, '"');
    $url = original_url($url);
    $siteinfo["url"] = $url;
    $siteinfo["type"] = "link";
    $stamp1 = microtime(true);
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_HEADER, 1);
    curl_setopt($ch, CURLOPT_NOBODY, 1);
    curl_setopt($ch, CURLOPT_TIMEOUT, 3);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
    curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
    $header = curl_exec($ch);
    $curl_info = @curl_getinfo($ch);
    $http_code = $curl_info['http_code'];
    curl_close($ch);
    $a->save_timestamp($stamp1, "network");
    if (($curl_info['http_code'] == "301" or $curl_info['http_code'] == "302" or $curl_info['http_code'] == "303" or $curl_info['http_code'] == "307") and ($curl_info['redirect_url'] != "" or $curl_info['location'] != "")) {
        if ($curl_info['redirect_url'] != "") {
            $siteinfo = parseurl_getsiteinfo($curl_info['redirect_url'], $no_guessing, $do_oembed, ++$count);
        } else {
            $siteinfo = parseurl_getsiteinfo($curl_info['location'], $no_guessing, $do_oembed, ++$count);
        }
        return $siteinfo;
    }
    // if the file is too large then exit
    if ($curl_info["download_content_length"] > 1000000) {
        return $siteinfo;
    }
    // if it isn't a HTML file then exit
    if ($curl_info["content_type"] != "" and !strstr(strtolower($curl_info["content_type"]), "html")) {
        return $siteinfo;
    }
    if ($do_oembed) {
        require_once "include/oembed.php";
        $oembed_data = oembed_fetch_url($url);
        if ($oembed_data->type != "error") {
            $siteinfo["type"] = $oembed_data->type;
        }
        if ($oembed_data->type == "link" and $siteinfo["type"] != "photo") {
            if (isset($oembed_data->title)) {
                $siteinfo["title"] = $oembed_data->title;
            }
            if (isset($oembed_data->description)) {
                $siteinfo["text"] = trim($oembed_data->description);
            }
            if (isset($oembed_data->thumbnail_url)) {
                $siteinfo["image"] = $oembed_data->thumbnail_url;
            }
        }
    }
    $stamp1 = microtime(true);
    // Now fetch the body as well
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_HEADER, 1);
    curl_setopt($ch, CURLOPT_NOBODY, 0);
    curl_setopt($ch, CURLOPT_TIMEOUT, 10);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
    $header = curl_exec($ch);
    $curl_info = @curl_getinfo($ch);
    $http_code = $curl_info['http_code'];
    curl_close($ch);
    $a->save_timestamp($stamp1, "network");
    // Fetch the first mentioned charset. Can be in body or header
    $charset = "";
    if (preg_match('/charset=(.*?)[' . "'" . '"\\s\\n]/', $header, $matches)) {
        $charset = trim(trim(trim(array_pop($matches)), ';,'));
    }
    if ($charset == "") {
        $charset = "utf-8";
    }
    $pos = strpos($header, "\r\n\r\n");
    if ($pos) {
        $body = trim(substr($header, $pos));
    } else {
        $body = $header;
    }
    if ($charset != '' and strtoupper($charset) != "UTF-8") {
        logger("parseurl_getsiteinfo: detected charset " . $charset, LOGGER_DEBUG);
        //$body = mb_convert_encoding($body, "UTF-8", $charset);
        $body = iconv($charset, "UTF-8//TRANSLIT", $body);
    }
    $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");
    $doc = new DOMDocument();
    @$doc->loadHTML($body);
    deletenode($doc, 'style');
    deletenode($doc, 'script');
    deletenode($doc, 'option');
    deletenode($doc, 'h1');
    deletenode($doc, 'h2');
    deletenode($doc, 'h3');
    deletenode($doc, 'h4');
    deletenode($doc, 'h5');
    deletenode($doc, 'h6');
    deletenode($doc, 'ol');
    deletenode($doc, 'ul');
    $xpath = new DomXPath($doc);
    $list = $xpath->query("//meta[@content]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        if (@$attr["http-equiv"] == 'refresh') {
            $path = $attr["content"];
            $pathinfo = explode(";", $path);
            $content = "";
            foreach ($pathinfo as $value) {
                if (substr(strtolower($value), 0, 4) == "url=") {
                    $content = substr($value, 4);
                }
            }
            if ($content != "") {
                $siteinfo = parseurl_getsiteinfo($content, $no_guessing, $do_oembed, ++$count);
                return $siteinfo;
            }
        }
    }
    //$list = $xpath->query("head/title");
    $list = $xpath->query("//title");
    foreach ($list as $node) {
        $siteinfo["title"] = html_entity_decode($node->nodeValue, ENT_QUOTES, "UTF-8");
    }
    //$list = $xpath->query("head/meta[@name]");
    $list = $xpath->query("//meta[@name]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"));
        if ($attr["content"] != "") {
            switch (strtolower($attr["name"])) {
                case "fulltitle":
                    $siteinfo["title"] = $attr["content"];
                    break;
                case "description":
                    $siteinfo["text"] = $attr["content"];
                    break;
                case "thumbnail":
                    $siteinfo["image"] = $attr["content"];
                    break;
                case "twitter:image":
                    $siteinfo["image"] = $attr["content"];
                    break;
                case "twitter:image:src":
                    $siteinfo["image"] = $attr["content"];
                    break;
                case "twitter:card":
                    if ($siteinfo["type"] == "" or $attr["content"] == "photo") {
                        $siteinfo["type"] = $attr["content"];
                    }
                    break;
                case "twitter:description":
                    $siteinfo["text"] = $attr["content"];
                    break;
                case "twitter:title":
                    $siteinfo["title"] = $attr["content"];
                    break;
                case "dc.title":
                    $siteinfo["title"] = $attr["content"];
                    break;
                case "dc.description":
                    $siteinfo["text"] = $attr["content"];
                    break;
                case "keywords":
                    $keywords = explode(",", $attr["content"]);
                    break;
                case "news_keywords":
                    $keywords = explode(",", $attr["content"]);
                    break;
            }
        }
        if ($siteinfo["type"] == "summary") {
            $siteinfo["type"] = "link";
        }
    }
    if (isset($keywords)) {
        $siteinfo["keywords"] = array();
        foreach ($keywords as $keyword) {
            $siteinfo["keywords"][] = trim($keyword);
        }
    }
    //$list = $xpath->query("head/meta[@property]");
    $list = $xpath->query("//meta[@property]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8"));
        if ($attr["content"] != "") {
            switch (strtolower($attr["property"])) {
                case "og:image":
                    $siteinfo["image"] = $attr["content"];
                    break;
                case "og:title":
                    $siteinfo["title"] = $attr["content"];
                    break;
                case "og:description":
                    $siteinfo["text"] = $attr["content"];
                    break;
            }
        }
    }
    if (@$siteinfo["image"] == "" and !$no_guessing) {
        $list = $xpath->query("//img[@src]");
        foreach ($list as $node) {
            $attr = array();
            if ($node->attributes->length) {
                foreach ($node->attributes as $attribute) {
                    $attr[$attribute->name] = $attribute->value;
                }
            }
            $src = completeurl($attr["src"], $url);
            $photodata = @getimagesize($src);
            if ($photodata && $photodata[0] > 150 and $photodata[1] > 150) {
                if ($photodata[0] > 300) {
                    $photodata[1] = round($photodata[1] * (300 / $photodata[0]));
                    $photodata[0] = 300;
                }
                if ($photodata[1] > 300) {
                    $photodata[0] = round($photodata[0] * (300 / $photodata[1]));
                    $photodata[1] = 300;
                }
                $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]);
            }
        }
    } else {
        $src = completeurl($siteinfo["image"], $url);
        unset($siteinfo["image"]);
        $photodata = @getimagesize($src);
        if ($photodata && $photodata[0] > 10 and $photodata[1] > 10) {
            $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]);
        }
    }
    if (@$siteinfo["text"] == "" and @$siteinfo["title"] != "" and !$no_guessing) {
        $text = "";
        $list = $xpath->query("//div[@class='article']");
        foreach ($list as $node) {
            if (strlen($node->nodeValue) > 40) {
                $text .= " " . trim($node->nodeValue);
            }
        }
        if ($text == "") {
            $list = $xpath->query("//div[@class='content']");
            foreach ($list as $node) {
                if (strlen($node->nodeValue) > 40) {
                    $text .= " " . trim($node->nodeValue);
                }
            }
        }
        // If none text was found then take the paragraph content
        if ($text == "") {
            $list = $xpath->query("//p");
            foreach ($list as $node) {
                if (strlen($node->nodeValue) > 40) {
                    $text .= " " . trim($node->nodeValue);
                }
            }
        }
        if ($text != "") {
            $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text));
            while (strpos($text, "  ")) {
                $text = trim(str_replace("  ", " ", $text));
            }
            $siteinfo["text"] = trim(html_entity_decode(substr($text, 0, 350), ENT_QUOTES, "UTF-8") . '...');
        }
    }
    logger("parseurl_getsiteinfo: Siteinfo for " . $url . " " . print_r($siteinfo, true), LOGGER_DEBUG);
    call_hooks('getsiteinfo', $siteinfo);
    return $siteinfo;
}
Esempio n. 3
0
function parseurl_getsiteinfo($url)
{
    $siteinfo = array();
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_HEADER, 1);
    curl_setopt($ch, CURLOPT_NOBODY, 0);
    curl_setopt($ch, CURLOPT_TIMEOUT, 3);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
    curl_setopt($ch, CURLOPT_USERAGENT, 'Opera/9.64(Windows NT 5.1; U; de) Presto/2.1.1');
    $header = curl_exec($ch);
    $curl_info = @curl_getinfo($ch);
    $http_code = $curl_info['http_code'];
    curl_close($ch);
    if (($curl_info['http_code'] == "301" or $curl_info['http_code'] == "302") and ($curl_info['redirect_url'] != "" or $curl_info['location'] != "")) {
        if ($curl_info['redirect_url'] != "") {
            $siteinfo = parseurl_getsiteinfo($curl_info['redirect_url']);
        } else {
            $siteinfo = parseurl_getsiteinfo($curl_info['location']);
        }
        return $siteinfo;
    }
    // Fetch the first mentioned charset. Can be in body or header
    if (preg_match('/charset=(.*?)[' . "'" . '"\\s\\n]/', $header, $matches)) {
        $charset = trim(array_pop($matches));
    } else {
        $charset = "utf-8";
    }
    $pos = strpos($header, "\r\n\r\n");
    if ($pos) {
        $body = trim(substr($header, $pos));
    } else {
        $body = $header;
    }
    $body = mb_convert_encoding($body, "UTF-8", $charset);
    $body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");
    $doc = new DOMDocument();
    @$doc->loadHTML($body);
    deletenode($doc, 'style');
    deletenode($doc, 'script');
    deletenode($doc, 'option');
    deletenode($doc, 'h1');
    deletenode($doc, 'h2');
    deletenode($doc, 'h3');
    deletenode($doc, 'h4');
    deletenode($doc, 'h5');
    deletenode($doc, 'h6');
    deletenode($doc, 'ol');
    deletenode($doc, 'ul');
    $xpath = new DomXPath($doc);
    $list = $xpath->query("//meta[@content]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        if (@$attr["http-equiv"] == 'refresh') {
            $path = $attr["content"];
            $pathinfo = explode(";", $path);
            $content = "";
            foreach ($pathinfo as $value) {
                if (substr(strtolower($value), 0, 4) == "url=") {
                    $content = substr($value, 4);
                }
            }
            if ($content != "") {
                $siteinfo = parseurl_getsiteinfo($content);
                return $siteinfo;
            }
        }
    }
    //$list = $xpath->query("head/title");
    $list = $xpath->query("//title");
    foreach ($list as $node) {
        $siteinfo["title"] = html_entity_decode($node->nodeValue, ENT_QUOTES, "UTF-8");
    }
    //$list = $xpath->query("head/meta[@name]");
    $list = $xpath->query("//meta[@name]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8");
        switch (strtolower($attr["name"])) {
            case "fulltitle":
                $siteinfo["title"] = $attr["content"];
                break;
            case "description":
                $siteinfo["text"] = $attr["content"];
                break;
            case "dc.title":
                $siteinfo["title"] = $attr["content"];
                break;
            case "dc.description":
                $siteinfo["text"] = $attr["content"];
                break;
        }
    }
    //$list = $xpath->query("head/meta[@property]");
    $list = $xpath->query("//meta[@property]");
    foreach ($list as $node) {
        $attr = array();
        if ($node->attributes->length) {
            foreach ($node->attributes as $attribute) {
                $attr[$attribute->name] = $attribute->value;
            }
        }
        $attr["content"] = html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8");
        switch (strtolower($attr["property"])) {
            case "og:image":
                $siteinfo["image"] = $attr["content"];
                break;
            case "og:title":
                $siteinfo["title"] = $attr["content"];
                break;
            case "og:description":
                $siteinfo["text"] = $attr["content"];
                break;
        }
    }
    if (@$siteinfo["image"] == "") {
        $list = $xpath->query("//img[@src]");
        foreach ($list as $node) {
            $attr = array();
            if ($node->attributes->length) {
                foreach ($node->attributes as $attribute) {
                    $attr[$attribute->name] = $attribute->value;
                }
            }
            $src = completeurl($attr["src"], $url);
            $photodata = @getimagesize($src);
            if ($photodata && $photodata[0] > 150 and $photodata[1] > 150) {
                if ($photodata[0] > 300) {
                    $photodata[1] = round($photodata[1] * (300 / $photodata[0]));
                    $photodata[0] = 300;
                }
                if ($photodata[1] > 300) {
                    $photodata[0] = round($photodata[0] * (300 / $photodata[1]));
                    $photodata[1] = 300;
                }
                $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]);
            }
        }
    } else {
        $src = completeurl($siteinfo["image"], $url);
        unset($siteinfo["image"]);
        $photodata = @getimagesize($src);
        if ($photodata && $photodata[0] > 10 and $photodata[1] > 10) {
            $siteinfo["images"][] = array("src" => $src, "width" => $photodata[0], "height" => $photodata[1]);
        }
    }
    if (@$siteinfo["text"] == "") {
        $text = "";
        $list = $xpath->query("//div[@class='article']");
        foreach ($list as $node) {
            if (strlen($node->nodeValue) > 40) {
                $text .= " " . trim($node->nodeValue);
            }
        }
        if ($text == "") {
            $list = $xpath->query("//div[@class='content']");
            foreach ($list as $node) {
                if (strlen($node->nodeValue) > 40) {
                    $text .= " " . trim($node->nodeValue);
                }
            }
        }
        // If none text was found then take the paragraph content
        if ($text == "") {
            $list = $xpath->query("//p");
            foreach ($list as $node) {
                if (strlen($node->nodeValue) > 40) {
                    $text .= " " . trim($node->nodeValue);
                }
            }
        }
        if ($text != "") {
            $text = trim(str_replace(array("\n", "\r"), array(" ", " "), $text));
            while (strpos($text, "  ")) {
                $text = trim(str_replace("  ", " ", $text));
            }
            $siteinfo["text"] = html_entity_decode(substr($text, 0, 350), ENT_QUOTES, "UTF-8") . '...';
        }
    }
    return $siteinfo;
}