Пример #1
0
function printHeader($omit, $url, $cl)
{
    global $db_con, $log_format, $template_path, $copy, $no_log;
    ob_start();
    if ($omit) {
        $urlparts = parse_addr($url);
        $omits = array();
        foreach ($omit as $dir) {
            if ($cl != 1) {
                $omits[] = "<li>" . $urlparts['scheme'] . "://" . $urlparts['host'] . $dir . "</li>";
            } else {
                $omits[] = $urlparts['scheme'] . "://" . $urlparts['host'] . $dir;
            }
        }
    }
    $log_msg_txt = "\r\nSpidering {$url}\r\n\r\n";
    if ($omits) {
        $log_msg_txt .= "Disallowed files and directories in robots.txt:\r\n";
        $log_msg_txt .= implode("\n", $omit);
        $log_msg_txt .= "\r\n";
    }
    $log_msg_html_1 = "     <h1>Spidering: <span class='warnok'>{$url}</span></h1>\n";
    $log_msg_html_link = "\t<p>[Go back to <span  class='em'><a href='admin.php'>admin</a></span>]</p>";
    if ($omits) {
        $log_msg_html_2 = "     <div class='alert'>\n\t\t<p class='em'>Disallowed files and directories in robots.txt:</p>\n\t\t<ul class='txt'>\n";
        $log_msg_html_2 .= implode("\n", $omits);
        $log_msg_html_2 .= "\n</ul>\n</div>\n";
    }
    if ($no_log == '0') {
        if ($cl != 1) {
            echo $log_msg_html_1 . $log_msg_html_2;
        } else {
            print $log_msg_txt;
        }
        @ob_flush();
        @flush();
    }
    if ($log_format == "html") {
        writeToLog($log_msg_html_1 . $log_msg_html_2, $copy);
    } else {
        writeToLog($log_msg_txt, $copy);
    }
}
Пример #2
0
$http = array();
$p2p = array();
$Count = array();
// Accumulator
$a_total = array();
$a_icmp = array();
$a_udp = array();
$a_tcp = array();
$a_ftp = array();
$a_http = array();
$a_p2p = array();
if ($dbtype == DB_PGSQL) {
    $sql_ip = "and ip <<= '{$ip}'";
} else {
    if ($dbtype == DB_MYSQL) {
        $p = parse_addr($ip);
        $net = $p["ip"] & $p["mask"];
        $sql_ip = "and (ip & " . sprintf("%u", $p["mask"]) . ") = " . sprintf("%u", $net);
    }
}
$pg_sql = "select *, extract(epoch from timestamp) as ts from sensors, {$table} where sensors.sensor_id = " . $table . ".sensor_id {$sql_ip} and sensor_name = '" . bd_escape_string($sensor_name) . "' and timestamp > {$timestamp}::abstime and timestamp < " . ($timestamp + $interval) . "::abstime order by ip;";
//original query
//$my_sql = "select *, unix_timestamp(timestamp) as ts from sensors, $table where sensors.sensor_id = ".$table.".sensor_id $sql_ip and sensor_name = '".bd_escape_string($sensor_name)."' and unix_timestamp(timestamp) > $timestamp and unix_timestamp(timestamp) < ".($timestamp+$interval)." order by ip";
//optimized query
$minTime = '';
$maxTime = '';
$my_sql = "select *, unix_timestamp(timestamp) as ts from sensors, {$table} where sensors.sensor_id = " . $table . ".sensor_id {$sql_ip} and sensor_name = '" . bd_escape_string($sensor_name) . "' and unix_timestamp(timestamp) > {$timestamp} and unix_timestamp(timestamp) < " . ($timestamp + $interval) . " order by ip";
if (DEBUG) {
    file_put_contents('debug_q.log', $my_sql);
}
//echo $my_sql."<br>"; exit(1);
Пример #3
0
function index_site($url, $reindex, $maxlevel, $soption, $url_inc, $url_not_inc, $can_leave, $use_robot, $use_nofollow, $cl, $all, $use_pref)
{
    global $db_con, $mysql_table_prefix, $command_line, $mainurl, $tmp_urls, $domain_arr, $all_keywords, $smp, $follow_sitemap;
    global $link_check, $smap_dir, $index_media, $clear, $create_sitemap, $tmp_dir, $domaincb;
    global $max_links, $realnum, $debug, $no_log, $dba_act, $add_auth, $interrupt, $index_media, $thumb_folder;
    if (!$can_leave) {
        $can_leave = $domaincb;
    }
    $can_leave_domain = $can_leave;
    $starttime = getmicrotime();
    //  start time to index this site
    $black = '0';
    //  will become counter for hits of blacklist
    $site_id = '';
    $skip = '';
    $smp = '0';
    $omit = array();
    $url = $db_con->real_escape_string(stripslashes($url));
    if (strstr($interrupt, "-")) {
        //  if indexer should not be interrupted periodically
        $interrupt = '999999';
        //  never
    }
    $int_count = $interrupt;
    //  $int_count will be decreased by each indexed link until $int_count = 1
    printStandardReport('starting', $command_line, $no_log);
    if (!isset($all_keywords)) {
        mysqltest();
        $sql_query = "SELECT keyword_ID, keyword from " . $mysql_table_prefix . "keywords";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        while ($row = $result->fetch_array(MYSQLI_NUM)) {
            $all_keywords[addslashes($row[1])] = $row[0];
        }
        if ($clear == 1) {
            clean_resource($result, '06');
        }
    }
    $url = convert_url($url);
    $compurl = parse_addr($url);
    if ($compurl['path'] == '') {
        $url = $url . "/";
    }
    $t = microtime();
    $a = getenv("REMOTE_ADDR");
    $sessid = md5($t . $a);
    if ($url != '/') {
        //      ignore dummies
        $urlparts = parse_addr($url);
        $domain = $urlparts['host'];
        if (isset($urlparts['port'])) {
            $port = (int) $urlparts['port'];
        } else {
            $port = 80;
        }
        if (strpos($url, "?")) {
            $url_bas = substr($url, 0, strpos($url, "?"));
        } else {
            $url_bas = $url;
        }
        mysqltest();
        $sql_query = "SELECT * from " . $mysql_table_prefix . "sites where url like '{$url_bas}%'";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $row = $result->fetch_array(MYSQLI_NUM);
        $site_id = $row[0];
        $authent = $row[2];
        if ($add_auth && $authent) {
            //  for sites with authentication we need to verify the value
            $url_status = url_status($url, $site_id, $sessid);
            $url_parts = parse_all_url($url);
            if ($url_status['state'] == 'ok' && $url_status['content'] == 'text') {
                if ($url_status['relocate']) {
                    //  if relocated,  print message and redirect to new URL
                    printRedirected($url_status['relocate'], $url_status['path'], $cl);
                    if (strstr($url_status['path'], "//")) {
                        //  if redirected to absolute URL, use this for further usage
                        $url = $url_status['path'];
                    } else {
                        $relo_url = str_replace($url_parts['query'], "", $url);
                        //  url without query
                        $relo_url = substr($url, 0, strrpos($relo_url, "/") + 1);
                        //  url without file name
                        if (strpos($url_status['path'], "./") === 0) {
                            //  if redirected relativ to same folder depth
                            $url_status['path'] = str_replace("./", "", $url_status['path']);
                            $url = "" . $relo_url . "" . $url_status['path'] . "";
                        }
                        if (strpos($url_status['path'], "../") === 0) {
                            //  if redirected relativ and one folder up
                            $url_status['path'] = str_replace("./", "", $url_status['path']);
                            $relo_url = substr($url, 0, strpos($url_parts['path']));
                            //  url without file name
                            $relo_url = substr($url, 0, strrpos($relo_url, "/") + 1);
                            //  url without last folder
                            $url = "" . $relo_url . "" . $url_status['path'] . "";
                        }
                    }
                }
                //  read file
                $contents = array();
                $file = '';
                $file = file_get_contents($url);
                if ($file === FALSE) {
                    //  we know another way to get the content
                    $get_charset = '';
                    $contents = getFileContents($url, $get_charset);
                    $file = $contents['file'];
                }
                //  parse header only
                preg_match("@<head[^>]*>(.*?)<\\/head>@si", $file, $regs);
                $headdata = $regs[1];
                //  fetch the tag value
                preg_match("/<meta +name *=[\"']?Sphider-plus[\"']? *content=[\"'](.*?)[\"']/i", $headdata, $res);
                if (isset($res)) {
                    if ($authent != $res[1]) {
                        //  invalid value in authentication tag
                        $skip = '1';
                        printHeader($omit, $url, $command_line);
                        printStandardReport('Skipped_03', $command_line, $no_log);
                    }
                } else {
                    //  no authentication tag found in header
                    $skip = '1';
                    printHeader($omit, $url, $command_line);
                    printStandardReport('Skipped_02', $command_line, $no_log);
                }
            } else {
                $skip = '1';
                printHeader($omit, $url, $command_line);
                printStandardReport('statError', $command_line, $no_log);
            }
        }
        if (!$skip) {
            if ($site_id != "" && $reindex == 1) {
                mysqltest();
                $sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id) values ('{$url}', 0, '{$sessid}')";
                $db_con->query($sql_query);
                if ($debug && $db_con->errno) {
                    $err_row = __LINE__ - 2;
                    printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                    if (__FUNCTION__) {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                    } else {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                    }
                    printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                    printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                    echo "<p> {$sql_query} </p>";
                    exit;
                }
                $sql_query = "SELECT url, level from " . $mysql_table_prefix . "links where site_id = {$site_id}";
                $result = $db_con->query($sql_query);
                while ($row = $result->fetch_array(MYSQLI_ASSOC)) {
                    $site_link = $row['url'];
                    $link_level = $row['level'];
                    if ($site_link != $url) {
                        $sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id) values ('{$site_link}', '{$link_level}', '{$sessid}')";
                        $db_con->query($sql_query);
                    }
                }
                $sql_query = "UPDATE " . $mysql_table_prefix . "sites set indexdate=now(), spider_depth ='{$maxlevel}', required = '{$url_inc}'," . "disallowed = '{$url_not_inc}', can_leave_domain='{$can_leave}', use_prefcharset='{$use_pref}' where site_id='{$site_id}'";
                mysqltest();
                $db_con->query($sql_query);
                if ($debug && $db_con->errno) {
                    $err_row = __LINE__ - 2;
                    printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                    if (__FUNCTION__) {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                    } else {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                    }
                    printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                    printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                    echo "<p> {$sql_query} </p>";
                    exit;
                }
            } else {
                if ($site_id == '') {
                    mysqltest();
                    $sql_query = "INSERT into " . $mysql_table_prefix . "sites (url, indexdate, spider_depth, required, disallowed, can_leave_domain, use_prefcharset) " . "values ('{$url}', now(), '{$maxlevel}', '{$url_inc}', '{$url_not_inc}', '{$can_leave_domain}', '{$use_pref}')";
                    $db_con->query($sql_query);
                    if ($debug && $db_con->errno) {
                        $err_row = __LINE__ - 2;
                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                        if (__FUNCTION__) {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                        } else {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                        }
                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                        echo "<p> {$sql_query} </p>";
                        exit;
                    }
                    $sql_query = "SELECT site_ID from " . $mysql_table_prefix . "sites where url='{$url}'";
                    $result = $db_con->query($sql_query);
                    if ($debug && $db_con->errno) {
                        $err_row = __LINE__ - 2;
                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                        if (__FUNCTION__) {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                        } else {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                        }
                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                        echo "<p> {$sql_query} </p>";
                        exit;
                    }
                    $row = $result->fetch_array(MYSQLI_NUM);
                    $site_id = $row[0];
                    if ($clear == 1) {
                        clean_resource($result, '09');
                    }
                } else {
                    mysqltest();
                    $sql_query = "UPDATE " . $mysql_table_prefix . "sites set indexdate=now(), spider_depth ='{$maxlevel}', required = '{$url_inc}'," . "disallowed = '{$url_not_inc}', can_leave_domain='{$can_leave_domain}', use_prefcharset='{$use_pref}' where site_id='{$site_id}'";
                    $db_con->query($sql_query);
                    if ($debug && $db_con->errno) {
                        $err_row = __LINE__ - 2;
                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                        if (__FUNCTION__) {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                        } else {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                        }
                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                        echo "<p> {$sql_query} </p>";
                        exit;
                    }
                }
            }
            $pending = array();
            mysqltest();
            $sql_query = "SELECT site_id, temp_id, level, count, num from " . $mysql_table_prefix . "pending where site_id='{$site_id}'";
            $result = $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
            $row = $result->fetch_array(MYSQLI_NUM);
            $pending = $row[0];
            $level = '0';
            $count = '0';
            if ($clear == 1) {
                clean_resource($result, '10');
            }
            $domain_arr = get_domains();
            if ($pending == '') {
                mysqltest();
                $sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id) values ('{$url}', 0, '{$sessid}')";
                $db_con->query($sql_query);
                if ($debug && $db_con->errno) {
                    $err_row = __LINE__ - 2;
                    printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                    if (__FUNCTION__) {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                    } else {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                    }
                    printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                    printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                    echo "<p> {$sql_query} </p>";
                    exit;
                }
            } else {
                if ($pending != '') {
                    printStandardReport('continueSuspended', $command_line, $no_log);
                    mysqltest();
                    $pend_count = '0';
                    //$result = $db_con->query("SELECT temp_id, level, count from ".$mysql_table_prefix."pending where site_id='$site_id'");
                    $sql_query = "SELECT * from " . $mysql_table_prefix . "pending where site_id='{$site_id}'";
                    $result = $db_con->query($sql_query);
                    if ($debug && $db_con->errno) {
                        $err_row = __LINE__ - 2;
                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                        if (__FUNCTION__) {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                        } else {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                        }
                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                        echo "<p> {$sql_query} </p>";
                        exit;
                    }
                    $row = $result->fetch_array(MYSQLI_NUM);
                    if ($row) {
                        $sessid = $row[1];
                        $level = $row[2];
                        $pend_count = $row[3] + 1;
                        $num = $row[4];
                        $pending = 1;
                        $tmp_urls = get_temp_urls($sessid);
                        if ($clear == 1) {
                            clean_resource($result, '11');
                        }
                    }
                }
            }
            if ($pending != 1) {
                mysqltest();
                $sql_query = "INSERT into " . $mysql_table_prefix . "pending (site_id, temp_id, level, count) values ('{$site_id}', '{$sessid}', '0', '0')";
                $db_con->query($sql_query);
                if ($debug && $db_con->errno) {
                    $err_row = __LINE__ - 2;
                    printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                    if (__FUNCTION__) {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                    } else {
                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                    }
                    printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                    printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                    echo "<p> {$sql_query} </p>";
                    exit;
                }
            }
            $time = time();
            $robots = "robots.txt";
            // standardname of robots file
            if ($use_robot == '1') {
                $omit = check_robot_txt($url, $robots);
            }
            printHeader($omit, $url, $command_line);
            if ($link_check == 1) {
                printStandardReport('start_link_check', $command_line, $no_log);
            }
            if ($link_check == 0 && $reindex == 1) {
                printStandardReport('start_reindex', $command_line, $no_log);
            }
            if ($link_check == 0 && $reindex == 0) {
                printStandardReport('starting', $command_line, $no_log);
            }
            $mainurl = $url;
            $realnum = $num;
            $num = 0;
            while ($level <= $maxlevel && $soption == 'level' || $soption == 'full') {
                if ($pending == 1) {
                    $count = $pend_count;
                    $pending = 0;
                } else {
                    $count = 0;
                }
                $links = array();
                mysqltest();
                $sql_query = "SELECT distinct link from " . $mysql_table_prefix . "temp where level={$level} && id='{$sessid}' order by link";
                $result = $db_con->query($sql_query);
                $rows = $result->num_rows;
                if ($rows == 0) {
                    break;
                }
                while ($row = $result->fetch_array(MYSQLI_ASSOC)) {
                    $links[] = $row['link'];
                }
                //  now loop through all available links(pages)
                while ($count < count($links)) {
                    $num++;
                    $realnum++;
                    if ($realnum > $max_links) {
                        //  if max. links per page reached
                        mysqltest();
                        $sql_query = "DELETE from " . $mysql_table_prefix . "temp where id = '{$sessid}'";
                        $db_con->query($sql_query);
                        $sql_query = "DELETE from " . $mysql_table_prefix . "pending where site_id = '{$site_id}'";
                        $db_con->query($sql_query);
                        printMaxLinks($max_links, $cl);
                        printStandardReport('completed', $command_line, $no_log);
                        return;
                    }
                    $thislink = $db_con->real_escape_string(stripslashes($links[$count]));
                    $urlparts = parse_addr($thislink);
                    $forbidden = 0;
                    if (is_array($omit)) {
                        //      if valid robots.txt  was found
                        reset($omit);
                        foreach ($omit as $omiturl) {
                            $omiturl = trim($omiturl);
                            $omiturl_parts = array();
                            $omiturl_parts = parse_addr($omiturl);
                            if (@$omiturl_parts['scheme'] == '') {
                                $check_omit = $urlparts['host'] . $omiturl;
                            } else {
                                $check_omit = $omiturl;
                            }
                            if (strpos($thislink, $check_omit)) {
                                printRobotsReport($num, $thislink, $command_line);
                                $realnum--;
                                check_for_removal($thislink);
                                $forbidden = 1;
                                break;
                            }
                        }
                    }
                    if (!check_include($thislink, $url_inc, $url_not_inc)) {
                        $realnum--;
                        printUrlStringReport($num, $thislink, $command_line);
                        //printUrlStringReport($realnum, $thislink, $command_line);
                        check_for_removal($thislink);
                        $forbidden = 1;
                    }
                    if ($forbidden == 0) {
                        printRetrieving($num, stripslashes(rawurldecode($thislink)), $command_line);
                        //printRetrieving($realnum, $thislink, $command_line);
                        mysqltest();
                        $sql_query = "SELECT md5sum, indexdate from " . $mysql_table_prefix . "links where url='{$thislink}'";
                        $result = $db_con->query($sql_query);
                        $rows = $result->num_rows;
                        if ($rows == 0) {
                            $url_status = index_url($thislink, $level + 1, $site_id, '', $domain, '', $sessid, $can_leave_domain, $reindex, $use_nofollow, $cl, $use_robot, $use_pref, $url_inc, $url_not_inc, $num);
                            //  check for touching the blacklist and its count against limit
                            if ($url_status['black'] == "1") {
                                $black++;
                                if ($black > 20) {
                                    //  limit until aborting the indexation of this site
                                    $url_status['aborted'] = "1";
                                    $url_status['state'] = "<br /><br />Indexation aborted for this site, as it met too often the blacklist.";
                                }
                            } else {
                                $black = 0;
                                //  reset counter, as should count only on continuous hits
                            }
                            //  check for emergency exit
                            if ($url_status['aborted'] == "1") {
                                //  delete all links from the temp table, which might be left for this site
                                mysqltest();
                                $sql_query = "DELETE from " . $mysql_table_prefix . "temp where id = '{$sessid}'";
                                $db_con->query($sql_query);
                                $sql_query = "DELETE from " . $mysql_table_prefix . "pending where site_id = '{$site_id}'";
                                $db_con->query($sql_query);
                                $sql_query = "UPDATE " . $mysql_table_prefix . "sites set indexdate=now() where url = '{$url}'";
                                $db_con->query($sql_query);
                                //  end all loops
                                $forbidden = '1';
                                $omit = '';
                                $reindex = '';
                                $count = '9999999999';
                                $pending = array();
                                if (!stristr($url_status['state'], "NOHOST") && !stristr($url_status['state'], "black")) {
                                    //  NOHOST warning will be printed separately
                                    printWarning($url_status['state'], $command_line, $no_log);
                                }
                            }
                            if (stristr($url_status['state'], "NOHOST")) {
                                //  delete all links from the temp table, which might be left for this site,  etc
                                mysqltest();
                                $sql_query = "DELETE from " . $mysql_table_prefix . "temp where id = '{$sessid}'";
                                $db_con->query($sql_query);
                                $sql_query = "DELETE from " . $mysql_table_prefix . "pending where site_id = '{$site_id}'";
                                $db_con->query($sql_query);
                                $sql_query = "UPDATE " . $mysql_table_prefix . "sites set indexdate=now() where url = '{$url}'";
                                $db_con->query($sql_query);
                                //  end all loops
                                $forbidden = '1';
                                $omit = '';
                                $reindex = '';
                                $count = '9999999999';
                                $pending = array();
                                printWarning($url_status['state'], $command_line, $no_log);
                                return;
                            }
                            //  check for UFO file or invalid suffix (by redirected URL)
                            if (stristr($url_status['state'], "ufo")) {
                                //printWarning($url_status['state'],$command_line, $no_log);
                            }
                            if ($url_status['state'] != "ok") {
                                printWarning($url_status['state'], $command_line, $no_log);
                            }
                            mysqltest();
                            $sql_query = "UPDATE " . $mysql_table_prefix . "pending set level ='{$level}', count='{$count}', num='{$realnum}' where site_id='{$site_id}'";
                            $db_con->query($sql_query);
                        } else {
                            if ($rows != 0 && $reindex == 1) {
                                $row = $result->fetch_array(MYSQLI_ASSOC);
                                $md5sum = $row['md5sum'];
                                $indexdate = $row['indexdate'];
                                if ($link_check == 1 && $reindex == 1) {
                                    link_check($thislink, $level + 1, $sessid, $can_leave_domain, $reindex, $site_id);
                                } else {
                                    $url_status = index_url($thislink, $level + 1, $site_id, $md5sum, $domain, $indexdate, $sessid, $can_leave_domain, $reindex, $use_nofollow, $cl, $use_robot, $use_pref, $url_inc, $url_not_inc, $num);
                                    //  check for emergency exit
                                    if ($url_status['aborted']) {
                                        //  delete all links from the temp table, which might be left for this site
                                        mysqltest();
                                        $sql_query = "DELETE from " . $mysql_table_prefix . "temp where id = '{$sessid}'";
                                        $db_con->query($sql_query);
                                        //  end all loops
                                        $forbidden = '1';
                                        $omit = '';
                                        $reindex = '';
                                        $count = '9999999999';
                                        $pending = array();
                                        printWarning($url_status['state'], $command_line, $no_log);
                                    }
                                }
                            } else {
                                printStandardReport('inDatabase', $command_line, $no_log);
                                $realnum--;
                                //$num--;
                            }
                        }
                        if ($rows != 0) {
                            mysqltest();
                            $sql_query = "UPDATE " . $mysql_table_prefix . "pending set level ='{$level}', count='{$count}', num='{$realnum}' where site_id='{$site_id}'";
                            $db_con->query($sql_query);
                        }
                        if ($clear == 1) {
                            clean_resource($result, '13');
                        }
                    }
                    //  check for interrupt counter
                    if ($int_count == '1') {
                        //  interrupt the index procedure until interactive resume
                        $sql_query = "UPDATE " . $mysql_table_prefix . "pending set level ='{$level}', count='{$count}', num='{$realnum}' where site_id='{$site_id}'";
                        $db_con->query($sql_query);
                        printInterrupt($interrupt, $url, $cl);
                        die;
                    }
                    $count++;
                    $int_count--;
                }
                $level++;
            }
        }
        mysqltest();
        $sql_query = "DELETE from " . $mysql_table_prefix . "temp where id = '{$sessid}'";
        $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $sql_query = "DELETE from " . $mysql_table_prefix . "pending where site_id = '{$site_id}'";
        $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        if ($create_sitemap == 1) {
            create_sitemap($site_id, $url);
        }
        printStandardReport('completed', $command_line, $no_log);
        $stats = get_Stats();
        printDatabase($stats, $cl);
    }
    if ($index_media) {
        //  delete all thumbnails in .../admin/tmp/thumbs/ folder
        clear_folder("." . $thumb_folder);
    }
}
Пример #4
0
// Print Title
if (isset($limit)) {
    echo "<h2>Top {$limit} - {$sensor_name}</h2>";
} else {
    echo "<h2>All Records - {$sensor_name}</h2>";
}
if (isset($last)) {
    echo "Last connection: " . $last . "</BR>\n";
}
// Sqlize the incomming variables
if (isset($subnet)) {
    if ($dbtype == DB_PGSQL) {
        $sql_subnet = "and ip <<= '{$subnet}'";
    } else {
        if ($dbtype == DB_MYSQL) {
            $p = parse_addr($subnet);
            $net = $p["ip"] & $p["mask"];
            $sql_subnet = "and (ip & " . sprintf("%u", $p["mask"]) . ") = " . sprintf("%u", $net);
        }
    }
}
// Sql Statement
$pg_sql = "select tx.ip, rx.scale as rxscale, tx.scale as txscale, tx.total+rx.total as total, tx.total as sent, \nrx.total as received, tx.tcp+rx.tcp as tcp, tx.udp+rx.udp as udp,\ntx.icmp+rx.icmp as icmp, tx.http+rx.http as http,\ntx.p2p+rx.p2p as p2p, tx.ftp+rx.ftp as ftp\nfrom\n\n(SELECT ip, max(total/sample_duration)*8 as scale, sum(total) as total, sum(tcp) as tcp, sum(udp) as udp, sum(icmp) as icmp,\nsum(http) as http, sum(p2p) as p2p, sum(ftp) as ftp\nfrom sensors, bd_tx_log\nwhere sensor_name = '" . bd_escape_string($sensor_name) . "'\nand sensors.sensor_id = bd_tx_log.sensor_id\n{$sql_subnet}\nand timestamp > {$timestamp}::abstime and timestamp < " . ($timestamp + $interval) . "::abstime\ngroup by ip) as tx,\n\n(SELECT ip, max(total/sample_duration)*8 as scale, sum(total) as total, sum(tcp) as tcp, sum(udp) as udp, sum(icmp) as icmp,\nsum(http) as http, sum(p2p) as p2p, sum(ftp) as ftp\nfrom sensors, bd_rx_log\nwhere sensor_name = '" . bd_escape_string($sensor_name) . "'\nand sensors.sensor_id = bd_rx_log.sensor_id\n{$sql_subnet}\nand timestamp > {$timestamp}::abstime and timestamp < " . ($timestamp + $interval) . "::abstime\ngroup by ip) as rx\n\nwhere tx.ip = rx.ip\norder by total desc;";
$my_sql = "select inet_ntoa(tx.ip) as ip, rx.scale as rxscale, tx.scale as txscale, tx.total+rx.total as total, tx.total as sent, \nrx.total as received, tx.tcp+rx.tcp as tcp, tx.udp+rx.udp as udp,\ntx.icmp+rx.icmp as icmp, tx.http+rx.http as http,\ntx.p2p+rx.p2p as p2p, tx.ftp+rx.ftp as ftp\nfrom\n\n(SELECT ip, max(total/sample_duration)*8 as scale, sum(total) as total, sum(tcp) as tcp, sum(udp) as udp, sum(icmp) as icmp,\nsum(http) as http, sum(p2p) as p2p, sum(ftp) as ftp\nfrom sensors, bd_tx_log\nwhere sensor_name = '" . bd_escape_string($sensor_name) . "'\nand sensors.sensor_id = bd_tx_log.sensor_id\n{$sql_subnet}\nand unix_timestamp(timestamp) > {$timestamp} and unix_timestamp(timestamp) < " . ($timestamp + $interval) . "\ngroup by ip) as tx,\n\n(SELECT ip, max(total/sample_duration)*8 as scale, sum(total) as total, sum(tcp) as tcp, sum(udp) as udp, sum(icmp) as icmp,\nsum(http) as http, sum(p2p) as p2p, sum(ftp) as ftp\nfrom sensors, bd_rx_log\nwhere sensor_name = '" . bd_escape_string($sensor_name) . "'\nand sensors.sensor_id = bd_rx_log.sensor_id\n{$sql_subnet}\nand unix_timestamp(timestamp) > {$timestamp} and unix_timestamp(timestamp) < " . ($timestamp + $interval) . "\ngroup by ip) as rx\n\nwhere tx.ip = rx.ip\norder by total desc";
//echo "</center><pre>$my_sql</pre><center>"; exit(0);
if ($dbtype == DB_PGSQL) {
    pg_query("SET sort_mem TO 30000;");
    $result = pg_query($pg_sql);
} else {
    if ($dbtype == DB_MYSQL) {
        $result = mysql_query($my_sql);