$maxlevel = $this_url[1];
$in = $this_url[2];
$out = $this_url[3];
$can_leave = $this_url[4];
$use_pref = $this_url[5];
if ($can_leave == '') {
$can_leave = 0;
}
if ($maxlevel == -1) {
$soption = 'full';
} else {
$soption = 'level';
}
}
if ($clear == 1) {
clean_resource($result, '01');
}
}
if (!isset($in)) {
$in = "";
}
if (!isset($out)) {
$out = "";
}
$started = time();
index_site($url, $reindex, $maxlevel, $soption, $in, $out, $can_leave, $use_robot, $use_nofollow, $cl, $all, $use_pref);
$ended = time();
$consumed = $ended - $started;
printConsumedReport('consumed', $cl, '0', $consumed);
printStandardReport('ReindexFinish', $command_line, '0');
}
function writeToLog($msg, $copy)
{
global $db_con, $keep_log, $log_handle, $real_handle, $real_log, $copy, $mysql_table_prefix, $refresh, $debug, $db_con;
// rebuild stylesheet of external css file, which is not available here
$msg = preg_replace("/<link rel='stylesheet'(.*?)type='text\\/css' \\/>/i", "<style>\n a { padding: 6px; text-decoration: underline; }\n a:active \\{ background: transparent; color: #00F; text-decoration: underline; }\n a:link { text-decoration: underline; }\n a:hover { background: #5F5F5F; color: #FF0; padding: 6px; text-decoration: none; }\n a:visited, .sphome a { text-decoration: none; }\n body { background: #DCDCDC; color: #007; font: 100% Verdana,Arial,Helvetica,sans-serif; margin: 0 auto; padding: 0; text-align: left; width: 750px; border:1px solid #FFFFCC;}\n h1 { background: #BDE4D4; color: #007; font-size: 1em; margin: 0px auto; padding: 10px; text-align: center; border:2px solid #FFFFCC; }\n hr, hr.powered { background-color: #91A681; border: none; color: #91A682; height: 2px; margin-left: 0px; margin-right: 0px; }\n html { height: 100%; margin-bottom: 1px; }\n li { font-size: 0.8em; line-height: 1.1em; list-style: none; margin: 0; padding: 0px 0px 8px 0px; vertical-align: baseline; }\n li.indented { margin-left: 6em; }\n p, .txt { font-size: 0.8em; font-weight: normal; text-align: left; }\n table { border: 1px solid #070; border-collapse: collapse; border-spacing: 2px; empty-cells: show; margin: 0 auto; padding: 0; }\n table tr td { font-size: 0.8em; padding: 5px; }\n table tr td.bd { text-align: right; vertical-align: baseline; }\n td { border: 1px solid #070; }\n ul { margin: 0; padding: 4px; }\n .tblhead { background: url(hdline.jpg) #D0E3D1; border: 1px solid #070; color: #007; font-weight: bold; padding: 6px; text-align: center; }\n .title { font-size: 1em; line-height: 1.4em; margin: 0; padding: 3px 2px 3px 0px; text-align: left; }\n .url { background: transparent; font-size: 0.7em; color: #7E7E7E; margin: 2px 0 5px 20px; padding: 0; }\n .w60 { margin: 0 auto; width: 60%; }\n .w75 { margin: 0 auto; width: 75%; }\n .warnadmin, .red { background: #FEFF04; color: #EE3C00; }\n .warn{ background: #bbb; color: #EE3C00; }\n .warn, .red .warnok, .green, .links, .blue { font-weight: bold; padding: 0px 2px; }\n .warnok, .green { background: transparent; color: #008001; }\n </style>", $msg);
// kill links to admin.php. Links are not available for log-file in subfolder xyz
$msg = str_replace("<p class='evrow'><br /><a class='bkbtn' href='admin.php?cancel=1' title='Abort and jump back to Admin'><strong>Cancel this index procedure</strong></a><br /><br /> Database will be updated, repaired, optimized and flushed. Please remain patient, as it may take some time to work out all.</p>", '', $msg);
$msg = str_replace("<p class='evrow'><a class='bkbtn' href='admin.php' title='Go back to Admin'>Back to admin</a></p>", '', $msg);
// Replace the 'Start indexer' buttons for multithreaded indexing with a 'Close window' button
$msg = preg_replace("@<div class='no_write'>.*?</div>@si", "<form class='cntr'>\n <table class='closeWindow'>\n <tr>\n <td>\n <input type='submit' value='Close this window' 'title='Return to Log File output' onclick='window.close()'>\n </td>\n </tr>\n </table>\n </form>\n <br /><br />", $msg);
//$end =strpos($msg, '</body>');
if (strpos($msg, '</body>') > '1') {
// create 'close window' button for log output.
$msg = "<br /><br />\n <form class='cntr'>\n <table class='closeBox'>\n <tr>\n <td>\n <input type='submit' value='Close this window' 'title='Return to Admin site' onclick='window.close()'>\n </td>\n </tr>\n </table>\n </form>\n <br /><br />\n </body>\n </html>";
}
if ($keep_log) {
if (!$log_handle) {
die("Cannot open file for logging. ");
}
if (fwrite($log_handle, $msg) === FALSE) {
die("Cannot write to file for logging. ");
}
if ($real_log == '1' && $copy == '1') {
// if selected,, update also the real-time log
//$msg = preg_replace("@\[.*?\]@si", "<br /><p><a href='javascript:ScrollDown()'>Jump to end of page</a></p><br />",$msg); // replace 'Back to Admin' links with 'Jump to end of page' for rel-time output
$msg = preg_replace("@\\[.*?\\]@si", "", $msg);
// supress 'Back to Admin' links for rel-time output
$msg = preg_replace("@<p class='evrow'><a class='bkbtn'.*?Back</a></p>@si", "", $msg);
// supress 'Back' links for rel-time output
if (strpos($msg, "</html>")) {
// create 'close window' button for real-time output.
$msg = "\n <a class='navup' href='javascript:JumpUp()' title='Jump to Page Top'>Top</a><br /><br />\n <form class='cntr'>\n <table class='closeWindow'>\n <tr>\n <td>\n <input type='submit' value='Close this window' 'title='Return to Log File output' onclick='window.close()'>\n </td>\n </tr>\n </table>\n </form>\n <br /><br />\n ";
}
if ($db_con) {
$sql_query = "SELECT * from " . $mysql_table_prefix . "real_log LIMIT 1";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$log = $result->fetch_array(MYSQLI_ASSOC);
$old_log = stripslashes($log["real_log"]);
// get previous real-log data
$msg = addslashes("" . $old_log . "" . $msg . "");
// build updated real-log data
// save the updated log data
$sql_query = "UPDATE " . $mysql_table_prefix . "real_log set `real_log`='{$msg}' LIMIT 1";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
clean_resource($result);
unset($old_log, $msg);
error_reporting(E_ALL & ~E_DEPRECATED & ~E_WARNING & ~E_NOTICE & ~E_STRICT);
}
}
}
}
function index_suspended()
{
global $db_con, $mysql_table_prefix, $command_line, $debug, $use_robot, $use_nofollow, $no_log, $clear, $started, $cl;
$started = time();
$reindex = 0;
printStandardReport('SuspendedStart', $command_line, $no_log);
// get ID and URL of all sites
$sql_query = "SELECT site_id, url from " . $mysql_table_prefix . "sites ORDER by url";
$result1 = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
while ($row1 = $result1->fetch_array(MYSQLI_NUM)) {
$url = $row1[1];
$site_id = $row1[0];
// check whether this site is pending
$sql_query = "SELECT site_id from " . $mysql_table_prefix . "pending where site_id ={$site_id}";
$result2 = $db_con->query($sql_query);
$row2 = $result2->fetch_array(MYSQLI_ASSOC);
// if pending, continue indexing this URL
if ($row2['site_id'] == $site_id) {
// fetch all important data of this site
$sql_query = "SELECT url, spider_depth, required, disallowed, can_leave_domain, use_prefcharset from " . $mysql_table_prefix . "sites where url='{$url}'";
$result = $db_con->query($sql_query);
if ($row = $result->fetch_array(MYSQLI_NUM)) {
$maxlevel = $row[1];
$in = $row[2];
$out = $row[3];
$domaincb = $row[4];
$use_prefcharset = $row[5];
if ($domaincb == '') {
$domaincb = 0;
}
if ($maxlevel == -1) {
$soption = 'full';
} else {
$soption = 'level';
}
}
if ($clear == 1) {
clean_resource($result, '21');
}
if (!isset($in)) {
$in = "";
}
if (!isset($out)) {
$out = "";
}
// now indnex the rest of this site
index_site($url, $reindex, $maxlevel, $soption, $in, $out, $domaincb, $use_robot, $use_nofollow, $cl, $all, $use_prefcharset);
}
}
if ($clear == 1) {
clean_resource($result, '20');
}
$ended = time();
$consumed = $ended - $started;
printConsumedReport('consumed', $cl, '0', $consumed);
printStandardReport('SuspendedFinish', $command_line, '0');
create_footer();
}
function index_new()
{
global $command_line, $use_robot;
$reindex == 0;
//include "admin_header.php";
printStandardReport('NewStart', $command_line);
$result = mysql_query("select url, indexdate, spider_depth, required, disallowed, can_leave_domain from " . TABLE_PREFIX . "sites");
if (DEBUG > '0') {
echo mysql_error();
}
while ($row = mysql_fetch_row($result)) {
$url = $row[0];
$indexdate = $row[1];
$depth = $row[2];
$include = $row[3];
$not_include = $row[4];
$can_leave_domain = $row[5];
if ($can_leave_domain == '') {
$can_leave_domain = 0;
}
if ($depth == -1) {
$soption = 'full';
} else {
$soption = 'level';
}
if ($indexdate == '') {
index_site($url, 1, $depth, $soption, $include, $not_include, $can_leave_domain, $use_robot);
}
}
clean_resource($result);
printStandardReport('NewFinish', $command_line);
create_footer();
}
function create_sitemap($site_id, $url)
{
global $mysql_table_prefix, $smap_dir;
$changefreq = "monthly";
// individualize this variable
$priority = "0.50";
// individualize this variable
// Below this only change something, if you are sure to remain compatible to http://www.sitemaps.org/schemas/sitemap/0.9
$date = date("Y-m-d");
$time = date("h:i:s");
$modtime = "T{$time}+01:00";
$version = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
$urlset = "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.google.com/schemas/sitemap/0.84 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">";
$copyright = "<!-- Generated by Sphider-plus created by Tec (v.1.2 rev.3) -->";
$update = "<!-- Last update of this sitemap: {$date} / {$time} -->";
$all_links = '';
$res = mysql_query("select * from " . $mysql_table_prefix . "links where site_id = {$site_id}");
echo mysql_error();
$num_rows = mysql_num_rows($res);
// Get all links of the current domain
for ($i = 0; $i < $num_rows; $i++) {
// Create individual rows for XML-file
$link = mysql_result($res, $i, "url");
$link = str_replace("&", "&", $link);
// URL should become XML conform
$all_links = "{$all_links}<url><loc>{$link}</loc><lastmod>{$date}{$modtime}</lastmod><changefreq>{$changefreq}</changefreq><priority>{$priority}</priority></url>\n";
}
clean_resource($res);
$name = parse_url($url);
// Create filename and open file
$hostname = $name[host];
if ($hostname == 'localhost') {
// if we run a localhost system extract the domain
$pathname = $name[path];
// get path, domain and filename
$pos = strpos($pathname, "/", 1);
// extract domain from path and forget first / by +1 offset
$pathname = substr($pathname, $pos + 1);
// suppress /localhost/
$pos = strrpos($pathname, "/");
if ($pos) {
$pathname = substr(str_replace("/", "_", $pathname), 0, $pos);
// if exists, suppress folder, filename and suffix
}
if (!is_dir($smap_dir)) {
mkdir($smap_dir, 0766);
// if new, create directory
}
$filename = "./{$smap_dir}/sitemap_localhost_{$pathname}.xml";
if (!($handle = fopen($filename, "w"))) {
printInvalidFile($filename);
die;
}
} else {
// if we run in the wild
if (!is_dir($smap_dir)) {
mkdir($smap_dir, 0766);
// if new, create directory
}
$filename = "./{$smap_dir}/sitemap_{$hostname}.xml";
if (!($handle = fopen($filename, "w"))) {
printInvalidFile($filename);
die('');
}
}
// Now write all to XML-file
if (!fwrite($handle, "{$version}\n{$urlset}\n{$copyright}\n{$update}\n{$all_links}</urlset>\n")) {
printInvalidFile($filename);
die('');
}
fclose($handle);
// sitemap.xml done! Now final printout
printSitemapCreated($filename);
}
function writeToLog($msg, $copy)
{
global $log_handle, $real_handle, $copy;
if (Configure::read('keep_log')) {
if (!$log_handle) {
die("Cannot open file for logging. ");
}
if (fwrite($log_handle, $msg) === FALSE) {
die("Cannot write to file for logging. ");
}
if (Configure::read('real_log') == '1' && $copy == '1') {
// if selected,, update also the real-time log
//$msg = preg_replace("@\[.*?\]@si", "<br /><p><a href='javascript:ScrollDown()'>Jump to end of page</a></p><br />",$msg); // replace 'Back to Admin' links with 'Jump to end of page' for rel-time output
$msg = preg_replace("@\\[.*?\\]@si", "", $msg);
// supress 'Back to Admin' links for rel-time output
$msg = preg_replace("@<p class='evrow'><a class='bkbtn'.*?Back</a></p>@si", "", $msg);
// supress 'Back' links for rel-time output
if (strpos($msg, "</html>")) {
// create 'close window' button for real-time output.
$msg = " \r\n <a class='navup' href='javascript:JumpUp()' title='Jump to Page Top'>Top</a><br /><br /> \r\n <form class='cntr'> \r\n <table class='searchBox'>\r\n <tr>\r\n <td> \r\n <input type='submit' value='Close this window' 'title='Return to Log File output' onclick='window.close()'>\r\n </td>\r\n </tr>\r\n </table> \r\n </form>\r\n <br /><br />\r\n </div>\r\n ";
}
$result = mysql_query("select real_log from " . TABLE_PREFIX . "real_log LIMIT 1");
if (DEBUG > '0') {
echo mysql_error();
}
$old_log = stripslashes(mysql_result($result, '0'));
// get previous real-log data
$msg = addslashes("{$old_log}{$msg}");
// build updated real-log data and save it
mysql_query("update " . TABLE_PREFIX . "real_log set `real_log`='{$msg}' LIMIT 1");
if (DEBUG > '0') {
echo mysql_error();
}
clean_resource($result);
unset($old_log, $msg);
}
}
}