//sleep(5);
//retrieve list of links to index from this level
$query = "SELECT id,path,file,indexed FROM " . PHPDIG_DB_PREFIX . "tempspider WHERE level = {$level} AND indexed = 0 AND site_id={$site_id} AND error = 0 limit 1";
$result_id = mysql_query($query, $id_connect);
$n_links = mysql_num_rows($result_id);
if ($n_links > 0) {
while ($new_links = mysql_fetch_array($result_id)) {
$query_check_stop = "SELECT stopped FROM " . PHPDIG_DB_PREFIX . "sites WHERE stopped=1";
$result_id_stop = mysql_query($query_check_stop, $id_connect);
if (mysql_num_rows($result_id_stop) > 0) {
$query = "SELECT id,path,file,indexed FROM " . PHPDIG_DB_PREFIX . "tempspider WHERE level = -1";
$result_id = mysql_query($query, $id_connect);
} else {
//keep alive the ftp connection (if exists)
if (FTP_ENABLE) {
$ftp_id = phpdigFtpKeepAlive($ftp_id, $relative_script_path);
}
//indexing this page
$temp_path = $new_links['path'];
$temp_file = $new_links['file'];
$already_indexed = $new_links['indexed'];
$tempspider_id = $new_links['id'];
//reset variables
$spider_id = 0;
$nomodif = 0;
$ok_for_spider = 0;
$ok_for_index = 0;
$tag = '';
$revisit_after = '';
//Retrieve dates if page is already in database
$test_exists = phpdigGetSpiderRow($id_connect, $site_id, $temp_path, $temp_file);
function phpdigWriteText($relative_script_path, $spider_id, $text, $ftp_id = '')
{
global $br;
if (CONTENT_TEXT == 1) {
$file_text_path = $relative_script_path . '/' . TEXT_CONTENT_PATH . $spider_id . '.txt';
if ($f_handler = @fopen($file_text_path, 'w')) {
reset($text);
while (list($n_chunk, $text_to_store) = each($text)) {
fputs($f_handler, wordwrap($text_to_store) . " ");
}
fclose($f_handler);
@chmod($file_text_path, 0666);
//here the ftp case
if (FTP_ENABLE) {
$ftp_id = phpdigFtpKeepAlive($ftp_id);
@ftp_delete($ftp_id, $spider_id . '.txt');
$res_ftp = false;
$try_count = 0;
while (!$res_ftp && $try_count++ < 10) {
$res_ftp = @ftp_put($ftp_id, $spider_id . '.txt', $file_text_path, FTP_ASCII);
if (!$res_ftp) {
sleep(2);
}
}
if (!$res_ftp) {
print "Ftp_put error !" . $br;
}
}
} else {
print "Warning : Unable to create the content file {$file_text_path} ! {$br}";
}
}
return $ftp_id;
}
