public function Analyze()
 {
     $info =& $this->getid3->info;
     $info['fileformat'] = 'rar';
     if ($this->option_use_rar_extension === true) {
         if (function_exists('rar_open')) {
             if ($rp = rar_open($info['filenamepath'])) {
                 $info['rar']['files'] = array();
                 $entries = rar_list($rp);
                 foreach ($entries as $entry) {
                     $info['rar']['files'] = getid3_lib::array_merge_clobber($info['rar']['files'], getid3_lib::CreateDeepArray($entry->getName(), '/', $entry->getUnpackedSize()));
                 }
                 rar_close($rp);
                 return true;
             } else {
                 $info['error'][] = 'failed to rar_open(' . $info['filename'] . ')';
             }
         } else {
             $info['error'][] = 'RAR support does not appear to be available in this PHP installation';
         }
     } else {
         $info['error'][] = 'PHP-RAR processing has been disabled (set $getid3_rar->option_use_rar_extension=true to enable)';
     }
     return false;
 }
Exemple #2
0
	public function uncompress_rar($path, $cachedir) {
		$rar_file = rar_open($path);
		$entries = rar_list($rar_file);
		$allowed = array('.jpg', '.gif', '.png', 'jpeg');
		foreach ($entries as $entry) {
			if (in_array(substr($entry->getName(), -4), $allowed))
				$entry->extract($cachedir);
		}
		rar_close($rar_file);
	}
function unrar($rarfile)
{
    $rar_file = rar_open($rarfile) or die("Can't open Rar archive");
    $entries = rar_list($rar_file);
    foreach ($entries as $entry) {
        echo 'Filename: ' . $entry->getName() . "\n";
        echo 'Packed size: ' . $entry->getPackedSize() . "\n";
        echo 'Unpacked size: ' . $entry->getUnpackedSize() . "\n";
        $entry->extract('D:/usr/lioncube/');
    }
    rar_close($rar_file);
}
Exemple #4
0
function unrarFile()
{
    $rar_file = rar_open(RAR_FILE);
    try {
        $list = rar_list($rar_file);
        foreach ($list as $file) {
            $file->extract(TARGET_DIR);
        }
    } catch (Exception $e) {
        try {
            rar_close($rar_file);
        } catch (Exception $e) {
        }
        echo "An exception occured while extracting archive: " . $e . "\n";
    }
}
Exemple #5
0
 /**
  * @see File_Archive_Reader::next()
  */
 function next()
 {
     $error = parent::next();
     if (PEAR::isError($error)) {
         return $error;
     }
     if ($this->rarFile === null) {
         $dataFilename = $this->source->getDataFilename();
         if ($dataFilename !== null) {
             $this->rarTmpName = null;
             $this->rarFile = rar_open($dataFilename);
         } else {
             $this->rarTmpName = tempnam(File_Archive::getOption('tmpDirectory'), 'far');
             //Generate the tmp data
             $dest = new File_Archive_Writer_Files();
             $dest->newFile($this->tmpName);
             $this->source->sendData($dest);
             $dest->close();
             $this->rarFile = rar_open($this->tmpName);
         }
         if (!$this->rarFile) {
             return PEAR::raiseError("Unable to open rar file {$dataFilename}");
         }
         if ($this->rarList === null) {
             $this->rarList = rar_list($this->rarFile);
             reset($this->rarList);
         }
     }
     if ($fileReader !== null) {
         $this->fileReader->close();
         $this->fileReader = null;
     }
     $entryName = next($this->rarList);
     $this->source = null;
     if ($entryName === false) {
         return false;
     }
     $this->rarEntry = rar_entry_get($this->rarFile, $entryName);
     if (!$this->rarEntry) {
         return PEAR::raiseError("Error reading entry {$entryName}");
     }
     return true;
 }
Exemple #6
0
 /**
  * Extract a ZIP compressed file to a given path
  *
  * @access	public
  * @param	string	$archive		Path to ZIP archive to extract
  * @param	string	$destination	Path to extract archive into
  * @param	array	$options		Extraction options [unused]
  * @return	boolean	True if successful
  * @since	1.5
  */
 function extract($archive, $destination, $options = array())
 {
     if (!is_file($archive)) {
         return PEAR::raiseError('Archive does not exist');
     }
     if (!$this->isSupported()) {
         return PEAR::raiseError('RAR Extraction not supported by your PHP installation.');
     }
     $arch = rar_open($archive);
     if ($arch === FALSE) {
         return PEAR::raiseError("Cannot open the rar archive");
     }
     $entries = rar_list($arch);
     if ($entries === FALSE) {
         return PEAR::raiseError("Cannot retrieve entries");
     }
     foreach ($entries as $file) {
         $file->extract($destination);
     }
     return true;
 }
Exemple #7
0
function func_rar($path1, $file2, $unrar = '')
{
    if (!function_exists("rar_open")) {
        if ($unrar['key'] == false) {
            echo 'You have not selected the variable $unrar=$arc[type_archive] in setup.php';
            echo "\n";
            $error6 = 1;
        }
        if ($unrar['path'] == false) {
            echo 'You have not selected the variable $unrar[path] in setup.php';
            echo "\n";
            $error6 = 1;
        }
        //echo $unrar['path'].' '.$unrar['key'].' '.$path1.'/'.$file2.' '.$unrar['out_dir'].$path1.'/'; echo "\n\n";
        exec($unrar['path'] . ' ' . $unrar['key'] . ' ' . $path1 . '/' . $file2 . ' ' . $unrar['out_dir'] . $path1 . '/', $out, $val);
        //print_r($out);
        if (!isset($out[1])) {
            echo "There no installed php functions php-rar or bash function unrar. Install one of this functions\n";
        }
        if ($out[$unrar['id_ok']] == $unrar['value_ok'] && $error6 == false) {
            return 0;
        } else {
            return 1;
        }
    } else {
        //echo "rar_php\n";
        $rar_file = @rar_open($path1 . '/' . $file2);
        if ($rar_file == false) {
            return false;
        } else {
            $entries = rar_list($rar_file);
            foreach ($entries as $entry) {
                $entry->extract($path1);
            }
            rar_close($rar_file);
            return 0;
        }
    }
}
function extract_archive($archive_name, $path, $password)
{
    if (!empty($archive_name) && !empty($path)) {
        $extension = pathinfo($path . $archive_name, PATHINFO_EXTENSION);
        $array_extension = array("tar" => 4, "gz" => 7, "bz2" => 8, "zip" => 4, "rar" => 4);
        $accepted_extension = false;
        $strlen_extension = "";
        foreach ($array_extension as $archive_extension => $strlen_archive_extension) {
            if ($extension === $archive_extension) {
                $accepted_extension = true;
                $strlen_extension = $strlen_archive_extension;
                break;
            }
        }
        if ($accepted_extension === false) {
            send_json('Extension not supported !!', null);
            throw new Exception("Extension not supported !!");
        }
        $only_file_name = substr($archive_name, 0, -$strlen_extension);
        $last_pos_only_file_name = strrpos($only_file_name, "-");
        $tmp_only_file_name = substr($only_file_name, 0, $last_pos_only_file_name);
        $counter_duplicate = substr($only_file_name, $last_pos_only_file_name + 1);
        if (!is_int($last_pos_only_file_name) || !is_int($counter_duplicate)) {
            $tmp_only_file_name = $only_file_name;
            $counter_duplicate = 1;
        }
        while (file_exists($path . $only_file_name)) {
            $only_file_name = $tmp_only_file_name . "-" . $counter_duplicate;
            $counter_duplicate++;
        }
        mkdir($path . $only_file_name);
        if ($extension === "zip") {
            try {
                $zip = zip_open($path . $archive_name);
                if ($zip) {
                    while ($zip_read = zip_read($zip)) {
                        $zip_entry_name = zip_entry_name($zip_read);
                        if (zip_entry_open($zip, $zip_read)) {
                            $file_in_archive = zip_entry_read($zip_read, zip_entry_filesize($zip_read));
                            $exploded = explode("/", substr($path . $only_file_name . "/" . zip_entry_name($zip_read), strlen($path . $only_file_name) + 1));
                            for ($i = 0; $i < count($exploded) - 1; $i++) {
                                if ($i === 0) {
                                    $path_after_onyl_file_name = $exploded[0];
                                } else {
                                    $path_after_onyl_file_name = $path_after_onyl_file_name . "/" . $exploded[$i];
                                }
                                if (!file_exists($path . $only_file_name . "/" . $path_after_onyl_file_name)) {
                                    mkdir($path . $only_file_name . "/" . $path_after_onyl_file_name, 0777, true);
                                }
                            }
                            file_put_contents($path . $only_file_name . "/" . zip_entry_name($zip_read), $file_in_archive);
                        }
                    }
                    zip_close($zip);
                }
            } catch (Exception $e) {
                send_json($e->getMessage(), null);
                throw new Exception($e->getMessage());
            }
        } elseif ($extension === "rar") {
            if ($password === "") {
                $rar = RarArchive::open($path . $archive_name);
            } else {
                $rar = RarArchive::open($path . $archive_name, $password);
            }
            if ($rar->isBroken()) {
                send_json("Rar archive broken !!", null);
                throw new Exception("Rar archive broken !!");
            }
            $rar_file = rar_open($path . $archive_name);
            $list = rar_list($rar_file);
            foreach ($list as $file) {
                $entry = rar_entry_get($rar_file, $file->getName());
                $entry->extract($path . $only_file_name);
            }
            rar_close($rar_file);
        } else {
            $archive = new PharData($path . $archive_name);
            try {
                $archive->extractTo($path . $only_file_name);
            } catch (Exception $e) {
                send_json($e->getMessage(), null);
                throw new Exception($e->getMessage());
            }
        }
        send_json(null, $only_file_name);
    }
}
Exemple #9
0
    while (false !== ($obj = readdir($dh))) {
        if ($obj == '.' || $obj == '..') {
            continue;
        }
        $file = $dir . DIRECTORY_SEPARATOR . $obj;
        if (time() - filectime($file) > $secondsOld) {
            @unlink($file);
        }
    }
    closedir($dh);
}
CleanCacheDir($options['cachepath'], 300);
$mimetype = minimime($path);
if ($mimetype == 'application/x-rar-compressed') {
    $filename = rawurldecode(stripslashes($_GET['page']));
    $rar_file = rar_open($path);
    $cachepathname = tempnam($options['cachepath'], "cbr");
    $entry = rar_entry_get($rar_file, $filename);
    $entry->extract(false, $cachepathname);
    rar_close($rar_file);
    $size = resize($cachepathname, intval($_GET['maxwidth']));
    echo '{' . "\n\r";
    echo '  "width" : "' . $size[0] . '",' . "\n\r";
    echo '  "height" : "' . $size[1] . '",' . "\n\r";
    echo '  "src" : "' . $options['webcache'] . '/' . basename($cachepathname) . '"' . "\n\r";
    echo '}';
} elseif ($mimetype == 'application/zip') {
    $index = 0;
    if (isset($_GET['page'])) {
        $index = intval($_GET['page']);
    }
Exemple #10
0
function content_do_import_data()
{
    global $cur_mysql_table;
    // just a quick error message function for the export routine
    function draw_errmsg($msg, $qry, $msg2)
    {
        echo "<p><span style=\"font-size:110%;\"><b>{$msg}</b></span> -- <br />\n <i>{$qry}</i><br /><b>{$msg2}</b></p>\n";
    }
    // used by both LOAD DATA and Sqlicity:
    $importLOCALfile = trim($_POST['importLOCALfile']) ? TRUE : FALSE;
    $local_importfile = trim($_POST['sqlicity_importfile1']);
    $server_importfile = trim($_POST['sqlicity_importfile2']);
    $column_headerlist = trim($_POST['columnheaderlist']);
    $linestoignore = (int) trim($_POST['linestoignore']);
    $replaceexistingrecords = trim($_POST['replaceexistingrecords']) ? TRUE : FALSE;
    // make sure they requested a file!
    if (!$local_importfile && !$server_importfile) {
        draw_errmsg('You need to specify a file to import!', '', '');
        return;
    }
    // check if they uploaded something but it didn't make it onto the server
    if ($importLOCALfile && $local_importfile && !$_FILES['sqlicity_importfile_browser']['tmp_name']) {
        draw_errmsg('Your file was not properly uploaded - are you sure it is small enough for PHP to upload it?', '', '');
        return;
    }
    // see if they want MySQL to do the export, rather than Sqlicity:
    if ($_POST['useLOADDATAINFILE']) {
        // check for LOCAL file reference
        if ($importLOCALfile) {
            $thefile = $_FILES['sqlicity_importfile_browser']['tmp_name'];
        } else {
            // import location can be relative or absolute
            if ($_POST['importfile_absolutepath']) {
                $thefile = $server_importfile;
            } else {
                $thefile = getcwd() . '/' . $server_importfile;
            }
        }
        // do the import
        mysql_query($query = "LOAD DATA {$_POST['ldi_lowpri_concur']}\n\t\t  " . ($importLOCALfile ? 'LOCAL' : '') . "\n\t\t  INFILE '{$thefile}' " . ($replaceexistingrecords ? 'REPLACE' : 'IGNORE') . "\n\t\t  INTO TABLE `{$cur_mysql_table}`\n\t\t FIELDS\n\t\t  TERMINATED BY '{$_POST['fields_terminatedby']}'\n\t\t  ENCLOSED BY '{$_POST['fields_enclosedby']}'\n\t\t  ESCAPED BY '{$_POST['fields_escapedby']}'\n\t\t LINES\n\t\t  TERMINATED BY '{$_POST['lines_terminatedby']}'\n\t\t  STARTING BY '{$_POST['lines_startingby']}'\n\t\t IGNORE " . (int) $_POST['linestoignore'] . " LINES\n\t\t " . ($column_headerlist ? '(' . $column_headerlist . ')' : '') . "\n\t\t {$_POST['ldi_setlist']}\n\t\t;");
        // find out how many rows were inserted/replaced/ignored
        $import_affected_rows = mysql_affected_rows();
        echo 'imported: ' . $import_affected_rows;
        // see if they have an error trying this query
        //  note that we "fix" the file name for LOCAL uploads, to maintain the illusion
        if (mysql_error()) {
            draw_errmsg('Your LOAD DATA INFILE failed', nl2br($importLOCALfile ? str_replace($thefile, $local_importfile, $query) : $query), mysql_error());
            return;
        }
        // quit now, the rest is for sqlicity's import functionality
        return;
    }
    /*
    IMPORTING RULES THAT I HAVE DETERMINED THROUGH EXPERIMENTATION AND TRUSTED READING:
    ======================================================================================
    1) escape char itself is ignored completely,
     next char is escaped only if it is a valid escapable char
     -- escape char works inside enclosers AND not!
     -- valid escape chars are NOT just \N and \0 -- it's the full list:
      http://dev.mysql.com/doc/mysql/en/string-syntax.html
    2) enclosing chars, if present ON EDGES (see !5) are removed, if not present, doesn't matter
    !3) doubled enclosing char becomes single char -- ONLY inside same enclosers!
    !4) If the line is missing commas to start some columns, those columns must be set to their "defaults":
     "If an input line has too few fields, the table columns
     for which input fields are missing are set to their default values."
    ** -- HOWEVER, 3.23 does not recognize the "DEFAULT" keyword on inserts, like all later versions do
    		but, there is a column in the SHOW COLUMNS results... 'default' that I can just use anyway
    		although this does require that I know the order of the columns,
    		which I do -- the order in the SHOW COLUMNS results
    !5) opening enclosing char MUST be first char,
     and closing enclosing char MUST come before a field terminator or line terminator
     -- IF enclosed string with SINGLE enclosing char NOT before field/line terminator,
      then this one enclosing char gets changed to "!!! (yes, a single double quote char) on 3.23.54
    6) no such thing as comments exist in csvs -- EXCEPT
     I can have anything in between line terminator and line starting
    !7) escaped N (e.g. \N) is ONLY NULL if it is the ONLY thing in the field!!
     otherwise it is just N (escape char dropped)
    8) ESCAPED and ENCLOSED must be one single char,
     TERMINATED and STARTED can be any string
    9) lots of weird interactions when all above empty,
     starting on load-data page: 'In certain cases, field- and line-handling options interact'
    10) NULLs (\N) are imported in or out of enclosing chars,
     but are exported withOUT enclosing chars.
    http://dev.mysql.com/doc/mysql/en/load-data.html
    */
    // check for LOCAL file reference
    if ($importLOCALfile) {
        $thefile = $_FILES['sqlicity_importfile_browser']['tmp_name'];
    } else {
        // import location can be relative or absolute
        if ($_POST['importfile_absolutepath']) {
            $thefile = $server_importfile;
        } else {
            $thefile = getcwd() . '/' . $server_importfile;
        }
    }
    // the file must exist!
    if (!@file_exists($thefile)) {
        draw_errmsg('That file does NOT exist!', $importLOCALfile ? $local_importfile : $thefile, '');
        return;
    }
    // find out how many rows were inserted/replaced/ignored
    $import_affected_rows = 0;
    // store all column names as keys, and their defaults as values
    $display_columns_defaults = array();
    // store all column names, and enclosed columns+BLOB columns
    //  SEE THE COMMENTS WITH THE SELECT PAGE CONTENTS
    $blob_cols = array();
    // track BLOB columns
    $result_cols = mysql_query("SHOW COLUMNS FROM `{$cur_mysql_table}`");
    // make sure table still exists
    if (!$result_cols) {
        echo "<p><b>Table <i>`{$cur_mysql_table}`</i> no longer exists!</b></p>\n";
        return;
    }
    // store all the columns, with their defaults, and flag important ones
    while ($column = mysql_fetch_assoc($result_cols)) {
        // find out if this column should be enclosed on output
        if (isEnclosedColumn($column['Type'])) {
            $enclosed_cols[$column['Field']] = TRUE;
        }
        // handle BLOB type columns specially
        if (strtolower(substr($column['Type'], 0, 4)) == 'blob') {
            $blob_cols[$column['Field']] = TRUE;
        }
        // store all column names in case no explicit select_columns was given
        $display_columns_defaults[$column['Field']] = $column['Default'];
    }
    // is the first line supposed to be a column header list?
    $usefirstlineheader = trim($_POST['firstlineheader']) ? TRUE : FALSE;
    // field/line enclosed/escaped/etc
    $field_enclose_char = unescape_mysql_chars($_POST['fields_enclosedby']);
    $field_enclose_char = $field_enclose_char[0];
    // force only one char
    $field_escape_char = unescape_mysql_chars($_POST['fields_escapedby']);
    $field_escape_char = $field_escape_char[0];
    // force only one char
    $field_terminatedby = unescape_mysql_chars($_POST['fields_terminatedby']);
    $line_terminatedby = unescape_mysql_chars($_POST['lines_terminatedby']);
    $line_startingby = unescape_mysql_chars($_POST['lines_startingby']);
    // shortcut to strlen on these, for finishing comparing them with input
    $line_startingby_len = strlen($line_startingby);
    $line_terminatedby_len = strlen($line_terminatedby);
    $field_terminatedby_len = strlen($field_terminatedby);
    // prep FSM variables
    //  whether or not there is a startingby string affects which state to start in
    $line_start_state = $line_startingby_len ? 1 : 2;
    $importFSMstate = $line_start_state;
    // line starting state: no line or new field?
    $escaped = FALSE;
    // was the last char the escape char, trying to escape this current char?
    $enclosed = FALSE;
    // is this an enclosed field or not?
    $linevalues = array();
    // holds the list of values to insert
    $curcol = -1;
    // incremented every col, including first, so need to start here to hit zero for first
    $check_line_i = -1;
    // zero is skipped to avoid matching empty terminators/etc
    $check_field_i = -1;
    // zero is... same here
    // VERY annoying special case -- need to track an extra bit of data
    //  inside enclosed data, single encloser followed by false L/F terminator
    $addtermchar = '';
    // this gets reused as the string gets too long (memory issues)
    $insert_values_list = '';
    // ... and this restarts it
    $restart_insert_values_list = ($replaceexistingrecords ? 'REPLACE' : 'INSERT IGNORE') . " INTO `{$cur_mysql_table}`\n  (" . ($column_headerlist ? $column_headerlist : implode(',', array_map(create_function('$c', 'return "`$c`";'), array_keys($display_columns_defaults)))) . ")\n VALUES\n";
    // get the parameters for this upload
    $import_filehandle = tmpfile();
    $compression = $_POST['import_file_compression'];
    // make sure the file is readable
    if (!$import_filehandle) {
        draw_errmsg('That file does is not readable!', $importLOCALfile ? $local_importfile : $thefile, '');
        return;
    }
    // first, pull out the sql file, based on compression type
    // bzip2
    if ($compression == 'bz2') {
        // open file for reading
        $bz = bzopen($thefile, 'r');
        // get the data from the file
        while ($d = bzread($bz)) {
            fwrite($import_filehandle, $d);
        }
        // close the compressed file
        bzclose($bz);
    } elseif ($compression == 'gz') {
        // open file for reading
        $zp = gzopen($thefile, 'r');
        // get the data from the file
        while ($d = gzread($zp, 1024)) {
            fwrite($import_filehandle, $d);
        }
        // close the compressed file
        gzclose($zp);
    } elseif ($compression == 'zip') {
        // open the zip archive
        if ($zip = zip_open($thefile)) {
            // zip can have multiple files
            while ($zip_entry = zip_read($zip)) {
                // include the data in ALL the zipped files
                if (zip_entry_open($zip, $zip_entry, "r")) {
                    fwrite($import_filehandle, zip_entry_read($zip_entry, zip_entry_filesize($zip_entry)));
                }
                zip_entry_close($zip_entry);
            }
            // close the compressed file handle
            zip_close($zip);
        }
    } elseif ($compression == 'rar') {
        // open the rar archive
        if ($rar = rar_open($thefile)) {
            // rar can have multiple files
            $entries = rar_list($rar);
            // include the data in ALL the rar'd files
            $rar_temp_name = tempnam(FALSE, 'sqlicity_rar');
            foreach ($entries as $entry) {
                // get the actual entry for extracting
                $entry = rar_entry_get($rar, $entry);
                // have to create the extracted file, read it then delete it...
                //  because rar ONLY has an extract() function, no direct read!
                if ($entry->extract(FALSE, $rar_temp_name)) {
                    fwrite($import_filehandle, file_get_contents($rar_temp_name));
                }
            }
            // close the compressed file handle and remove temp file
            unlink($rar_temp_name);
            rar_close($rar);
        }
    } else {
        // open plaintext file for reading
        $pt = fopen($thefile, 'r');
        // get the data from the file
        while ($d = fread($pt, 1024)) {
            fwrite($import_filehandle, $d);
        }
        // close the plaintext file
        fclose($pt);
    }
    //	else { fwrite($import_filehandle, file_get_contents($thefile))); }
    // needed for escaping input chars
    global $mysql_escaped_chars;
    // we just wrote the file for import, now back up to read it all in
    rewind($import_filehandle);
    // fseek($import_filehandle, 0);
    while (!feof($import_filehandle)) {
        // get this current line and iterate over all its chars
        //  max sure to check if this line is shorter than what we asked for...
        $next_import_line = fread($import_filehandle, 8192);
        $maxlen = strlen($next_import_line);
        for ($i = 0; $i < $maxlen; $i++) {
            // get the char to run through the FSM
            $char = $next_import_line[$i];
            // do the FSM itself
            switch ($importFSMstate) {
                // no line started yet -- in-between lines -- ONLY if line starting by exists!
                case 1:
                    // have we found a line starting beginning and are checking it now?
                    if ($check_line_i > 0) {
                        // is this the next char?
                        if ($char == $line_startingby[$check_line_i]) {
                            // increment for the next char we'll check
                            $check_line_i++;
                            // ... unless it's the last char, then we're done -- start a new field
                            if ($check_line_i == $line_startingby_len) {
                                $check_line_i = -1;
                                $importFSMstate = 2;
                            }
                        } else {
                            $check_line_i = -1;
                        }
                    } elseif ($char == $line_startingby[0]) {
                        // remember, starting by could be only one char, then this would have just ended it
                        if ($line_startingby_len == 1) {
                            $importFSMstate = 2;
                        } else {
                            $check_line_i = 1;
                        }
                    } elseif ($char == $field_escape_char) {
                        $escaped = TRUE;
                    } elseif ($escaped) {
                        $escaped = FALSE;
                    }
                    break;
                    // starting a new field -- only stays here for one char!
                // starting a new field -- only stays here for one char!
                case 2:
                    // reset some markers every time we restart a field
                    $enclosed = FALSE;
                    $escaped = FALSE;
                    // start this new column in the data
                    $linevalues[++$curcol] = '';
                    // awkward special case: escaped NULLs
                    $justNULLdata = FALSE;
                    // everything here goes to data read state no matter what
                    //  except line/field terminator special cases below
                    $importFSMstate = 3;
                    // check for encloser, escape char and terminators
                    if ($char == $field_enclose_char) {
                        $enclosed = TRUE;
                    } elseif ($char == $field_escape_char) {
                        $escaped = TRUE;
                    } elseif ($char == $line_terminatedby[0]) {
                        $check_line_i = 1;
                    } elseif ($char == $field_terminatedby[0]) {
                        $check_field_i = 1;
                    } else {
                        $linevalues[$curcol] = $char;
                    }
                    break;
                    // read in the data for a single field
                // read in the data for a single field
                case 3:
                    // we might have a line terminator here
                    if ($check_line_i > 0) {
                        // if it matches, we prepare to check the next char
                        if ($char === $line_terminatedby[$check_line_i]) {
                            $check_line_i++;
                        } else {
                            // include everything up to what did NOT work, then add this char
                            $linevalues[$curcol] .= $addtermchar . substr($line_terminatedby, 0, $check_line_i) . $char;
                            $check_line_i = -1;
                            $addtermchar = '';
                        }
                    } elseif ($check_field_i > 0) {
                        // if it matches, we prepare to check the next char
                        if ($char === $field_terminatedby[$check_field_i]) {
                            $check_field_i++;
                        } else {
                            // include everything up to what did NOT work, then add this char
                            $linevalues[$curcol] .= $addtermchar . substr($field_terminatedby, 0, $check_field_i) . $char;
                            $check_field_i = -1;
                            $addtermchar = '';
                        }
                    } else {
                        // first check for escaped chars -- replace them
                        if ($escaped) {
                            // next one starts escaping all over again
                            $escaped = FALSE;
                            // try to find the escape sequence replacement
                            $c2 = $mysql_escaped_chars[$char];
                            // if it doesn't exist, just use the original char
                            $linevalues[$curcol] .= $c2 !== NULL ? $c2 : $char;
                            // special case: escaped NULLs
                            //  to be a NULL, has to be the only thing in the field, enclosed or not
                            if ($linevalues[$curcol] == 'N') {
                                $justNULLdata = TRUE;
                            }
                        } elseif ($char == $field_escape_char) {
                            $escaped = TRUE;
                        } elseif ($enclosed && $char == $field_enclose_char) {
                            $importFSMstate = 4;
                        } elseif (!$enclosed && $char == $line_terminatedby[0]) {
                            $check_line_i = 1;
                        } elseif (!$enclosed && $char == $field_terminatedby[0]) {
                            $check_field_i = 1;
                        } else {
                            $linevalues[$curcol] .= $char;
                        }
                    }
                    break;
                    // we just had an encloser char, handle the single next char to transfer again
                // we just had an encloser char, handle the single next char to transfer again
                case 4:
                    // most of these options just go right back to data parsing, so that's the default
                    $importFSMstate = 3;
                    // double encloser -- add only one of them
                    if ($char == $field_enclose_char) {
                        $linevalues[$curcol] .= $field_enclose_char;
                    } elseif ($char == $line_terminatedby[0]) {
                        $check_line_i = 1;
                        $addtermchar = $field_enclose_char;
                    } elseif ($char == $field_terminatedby[0]) {
                        $check_field_i = 1;
                        $addtermchar = $field_enclose_char;
                    } elseif ($char == $field_escape_char) {
                        $linevalues[$curcol] .= $field_enclose_char;
                        $escaped = TRUE;
                    } else {
                        $linevalues[$curcol] .= $field_enclose_char . $char;
                    }
                    break;
            }
            // lines/fields can terminate from any state except before there is any line (state 1)
            //  so we check for termination here for all but "no line"
            if ($importFSMstate > 1) {
                // did this field just end?
                if ($check_field_i == $field_terminatedby_len) {
                    // handle special NULL values
                    //  unenclosed 'NULL' is always NULL
                    if (!$enclosed && $linevalues[$curcol] == 'NULL' || $linevalues[$curcol] == 'N' && $justNULLdata) {
                        $linevalues[$curcol] = NULL;
                    }
                    // this one is done, start a new field
                    $check_field_i = -1;
                    $importFSMstate = 2;
                }
                // did this line just end?
                if ($check_line_i == $line_terminatedby_len) {
                    // handle special NULL values
                    //  unenclosed 'NULL' is always NULL
                    if (!$enclosed && $linevalues[$curcol] == 'NULL' || $linevalues[$curcol] == 'N' && $justNULLdata) {
                        $linevalues[$curcol] = NULL;
                    }
                    // start a new line
                    $check_line_i = -1;
                    $curcol = -1;
                    $importFSMstate = $line_start_state;
                    // is this first line a column header list?
                    if ($usefirstlineheader) {
                        // set the column header list for all the real lines following
                        //  that means reorganizing the display columns defaults
                        $display_columns_defaults2 = array();
                        foreach ($linevalues as $identifier) {
                            // build the array of column names mapped to defaults
                            //  if the user used non-existent columns,
                            //  it's their problem when this breaks
                            $display_columns_defaults2[$identifier] = $display_columns_defaults[$identifier];
                        }
                        // make sure there is actually something in that first line
                        if (count($display_columns_defaults2) > 0) {
                            // make the switch, we're using this new header instead
                            $display_columns_defaults = $display_columns_defaults2;
                            unset($display_columns_defaults2);
                            // and rebuild the insert values list restart
                            $restart_insert_values_list = ($replaceexistingrecords ? 'REPLACE' : 'INSERT IGNORE') . " INTO `{$cur_mysql_table}`\n  (" . implode(',', array_map(create_function('$c', 'return "`$c`";'), array_keys($display_columns_defaults))) . ")\n VALUES\n";
                        }
                        // we got the header, turn this flag off
                        $usefirstlineheader = FALSE;
                    } else {
                        // add this line we just finished
                        //  by building the INSERT VALUES list line items
                        //  NOTE: this will not be visible to the user!
                        //  so it doesn't need to be pretty!  -- I make it this way to be readable here only
                        // first check if we are ignoring more lines -- if so, don't insert this line
                        if ($linestoignore > 0) {
                            $linestoignore--;
                            return;
                        }
                        // we are done ignoring lines, let's do the insert
                        // do we need a , to start this linevalues list?  only if not restarted IVL
                        $old_IVL = TRUE;
                        // see if we're starting a new values list now
                        if (!$insert_values_list) {
                            $old_IVL = FALSE;
                            $insert_values_list = $restart_insert_values_list;
                        }
                        // now we can add the current linevalues as a values list item to our insert
                        // start this current line -- comma for old IVLs adding extra list items, space for new ones
                        $insert_values_list .= ($old_IVL ? ',' : ' ') . ' (';
                        // we will need this, for escaping imported data strings
                        global $insert_escaped_chars;
                        // now handle every field that needs to be inserted
                        //  -- we might not have values for all of them!
                        $colnum = -1;
                        // track which value to use, and to do separating on fields
                        foreach ($display_columns_defaults as $colname => $coldefault) {
                            // after first col, start with field terminator
                            if (++$colnum > 0) {
                                $insert_values_list .= ',';
                            }
                            // make sure this column was even defined!
                            if (isset($linevalues[$colnum])) {
                                $value = $linevalues[$colnum];
                            } else {
                                $value = $coldefault;
                            }
                            // users can import all kinds of complete garbage
                            //  therefore, we MUST enclose EVERY data field, in case it's trash
                            //  and we must further escape all data as well
                            $insert_values_list .= $value !== NULL ? "'" . add_mysql_slashes($value) . "'" : 'NULL';
                        }
                        // finish this line
                        $insert_values_list .= ")\n";
                        // see if we're ready to start a new IVL or not
                        //  if the size is getting big, run the IVL query and empty it for restart
                        //  current max length is ~200 kb: 200,000 bytes
                        if (strlen($insert_values_list) > 200000) {
                            // do the actual insert
                            mysql_query($insert_values_list);
                            // see if they have an error trying this import query
                            if ($myserr = mysql_error()) {
                                draw_errmsg('Your Sqlicity import failed!', '', $myserr);
                                return;
                            }
                            // update the affected rows and reset the insert string
                            $import_affected_rows += mysql_affected_rows();
                            $insert_values_list = '';
                        }
                    }
                    // end fist line column header conditional
                    // reset the array of values we got off the current lines
                    $linevalues = array();
                }
                // end insert line ended (we just added a new line, or found the header)
            }
        }
        // end loop over the characters of current string from import file to parse
    }
    // end loop over remaining data in import file
    // get rid of the temporary file for uncompressed output
    fclose($import_filehandle);
    // run the final insert query we've built up
    mysql_query($insert_values_list);
    $import_affected_rows += mysql_affected_rows();
    // see if they have an error trying this import query
    if ($myserr = mysql_error()) {
        draw_errmsg('Your Sqlicity import failed!', $insert_values_list, $myserr);
        return;
    }
    // tell the import results
    echo 'imported: ' . $import_affected_rows;
}
Exemple #11
0
             }
             // zip_entry_filesize | zip_entry_compressedsize | zip_entry_compressionmethod
             @zip_close($zip);
             $sData = '"type":"archive","text":"' . $sDta . ' "';
             $sMsg .= "contentsSucces";
         } else {
             $sMsg .= "contentsFail";
         }
     }
 } else {
     if ($sExt == "rar") {
         // - extension=php_rar.dll
         if (!function_exists("rar_open")) {
             $sMsg .= "php_rar not installed or enabled";
         } else {
             if ($rar_file = @rar_open(getcwd() . "/" . $sSFile)) {
                 $entries = @rar_list($rar_file);
                 foreach ($entries as $entry) {
                     $sDta .= $entry->getName() . "\\r\\n";
                 }
                 // getName | getPackedSize | getUnpackedSize
                 @rar_close($rar_file);
                 $sData = '"type":"archive","text":"' . $sDta . ' "';
                 $sMsg .= "contentsSucces";
             } else {
                 $sMsg .= "contentsFail";
             }
         }
     } else {
         if ($sExt == "pdf") {
             include 'class.pdf2text.php';
function index_url($url, $level, $site_id, $md5sum, $domain, $indexdate, $sessid, $can_leave_domain, $reindex, $use_nofollow, $cl, $use_robot, $use_pref, $url_inc, $url_not_inc, $num)
{
    global $db_con, $entities, $min_delay, $link_check, $command_line, $min_words_per_page, $dup_content, $dup_url, $quotes, $plus_nr, $use_prefcharset;
    global $min_words_per_page, $supdomain, $smp, $follow_sitemap, $max_links, $realnum, $local, $tmp_dir, $auto_add, $admin_email, $idna, $conv_puny;
    global $mysql_table_prefix, $user_agent, $tmp_urls, $delay_time, $domain_arr, $home_charset, $charSet, $url_status, $redir_count;
    global $debug, $common, $use_white1, $use_white2, $use_black, $whitelist, $blacklist, $clear, $abslinks, $utf8_verify, $webshot;
    global $index_media, $index_image, $suppress_suffix, $imagelist, $min_image_x, $min_image_y, $dup_media, $index_alt, $no_log, $index_rss;
    global $index_audio, $audiolist, $index_video, $videolist, $index_embeded, $rss_template, $index_csv, $delim, $ext, $index_id3, $dba_act;
    global $converter_dir, $dict_dir, $cn_seg, $jp_seg, $index_framesets, $index_iframes, $cdata, $dc, $preferred, $index_rar, $index_zip, $curl;
    global $docs, $only_docs, $only_links, $case_sensitive, $vowels, $noacc_el, $include_dir, $thumb_folder, $js_reloc, $server_char;
    global $latin_ligatures, $phon_trans, $liga;
    //  Currently (2013.01.11)  the variable $use_prefcharset as defined in Admin Settings 'Obligatory use preferred charset' is used.
    //  and not the variable $use_pref as defined in Admin Settings as a varaiable used for addsite() in .../admin/admin.php
    error_reporting(E_ALL & ~E_DEPRECATED & ~E_WARNING & ~E_NOTICE & ~E_STRICT);
    $data = array();
    $cn_data = array();
    $url_parts = array();
    $url_status = array();
    $url_status['black'] = '';
    $contents = array();
    $links = array();
    $wordarray = array();
    $topic = '';
    $url_reloc = '';
    $js_link = '';
    $document = '';
    $file = '';
    $file0 = '';
    $raw_file = '';
    $seg_data = '';
    $index_url = $url;
    $comment = $db_con->real_escape_string("Automatically added during index procedure, as this domain is not yet available in 'Sites' menu.");
    $admin_email = $db_con->real_escape_string($admin_email);
    if ($debug == '0') {
        if (function_exists("ini_set")) {
            ini_set("display_errors", "0");
        }
        error_reporting(0);
    } else {
        error_reporting(E_ERROR);
        //  otherwise  a non existing siemap.xml  would always cause a warning message
    }
    $needsReindex = 1;
    $deletable = 0;
    $nohost = 1;
    $i = 0;
    $nohost_count = 5;
    //  defines count of attempts to get in contact with the server
    //  check URL status
    while ($i < $nohost_count && $nohost) {
        $url_status = url_status($url, $site_id, $sessid);
        if (!stristr($url_status['state'], "NOHOST")) {
            $nohost = '';
            //  reset for successfull attempt
        }
        $i++;
    }
    //  check for emergency exit
    if ($url_status['aborted'] == '1' || stristr($url_status['state'], "NOHOST")) {
        return $url_status;
    }
    //  check for UFO file or invalid suffix
    if (stristr($url_status['state'], "ufo")) {
        return $url_status;
    }
    // JFIELD here is right before we try to retrieve the URL and get the error
    // echo "<h3>F****E: $url</h3>\n";
    //  check for 'unreachable' links and if it is a known URL, delete all keyword relationships, former indexed from the meanwhile unreachable link
    if (stristr($url_status['state'], "unreachable")) {
        printStandardReport('unreachable', $command_line, $no_log);
        $sql_query = "SELECT link_id from " . $mysql_table_prefix . "links where url='{$url}'";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $row = $result->fetch_array(MYSQLI_NUM);
        $link_id = $row[0];
        if ($link_id) {
            $sql_query = "DELETE from " . $mysql_table_prefix . "link_keyword where link_id={$link_id}";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
            //  here we should delete the keywords associated only to the unreachable link
            //  but this takes too much time during index procedure
            //  the admin is asked toc do it manually by using the regarding option in 'Clean' menue
            //
            //  delete the meanwhile unreachable link from db
            $sql_query = "DELETE from " . $mysql_table_prefix . "links where link_id = {$link_id}";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
        }
        return $url_status;
    }
    //  check for overwritten URL, forced by the header, sending content PLUS any redirected URL
    if ($url_status['url_over'] && !$url_status['relocate']) {
        $url = $url_status['url_over'];
    }
    $url_parts = parse_all_url($url);
    $thislevel = $level - 1;
    //  redirected URL ?
    if ($url_status['relocate']) {
        //  if relocated,  print message, verify the new URL, and redirect to new URL
        //  check for redirection on an already indexed link
        $known_link = '';
        $sql_query = "SELECT * from " . $mysql_table_prefix . "links where url='{$url}'";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $known_link = $result->num_rows;
        if ($known_link) {
            $urlo_status['state'] = "URL was redirected to an already indexed page.<br />In order to prevent infinite indexation, this is not supported by Sphider-plus.<br />Indexation aborted for this URL";
            $url_status['aborted'] = 1;
            return $url_status;
        }
        //  remove the original URL from temp table. The relocated URL will be added later on.
        mysqltest();
        $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$url}' AND id = '{$sessid}'";
        $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $new_url = $url_status['path'];
        //  URL of first redirection
        //  remove the redirected URL, which eventually is  already stored in db
        //  before finally storing in db, we need to check for correct redirection.
        $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$new_url}' AND id = '{$sessid}'";
        $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        //  now special processing for relative links
        if (!strpos(substr($new_url, 0, 5), "ttp")) {
            $new_url = make_abs($new_url, $index_url);
        }
        if ($url == $new_url && $url_status['file']) {
            $url_status['relocate'] = '';
            //  remove this redirection, as it is 'in it selves'
            $url_status['state'] = "ok";
            //  try to index the conteent
        }
        $care_excl = '1';
        //  care file suffixed to be excluded
        $relocated = '1';
        //  URL is relocated
        if ($debug) {
            printRedirected($url_status['relocate'], $url_status['path'], $cl);
        }
        $count = "1";
        while ($count <= $redir_count && $url_status['relocate'] && !$url_status['aborted']) {
            //  check this redirection
            $url_status = url_status($new_url, $site_id, $sessid);
            if ($url_status['path']) {
                $new_url = $url_status['path'];
                //  URL of another redirections
                //  now special processing for relative links
                if (!strpos(substr($new_url, 0, 5), "ttp")) {
                    $new_url = make_abs($new_url, $index_url);
                }
            }
            if ($debug) {
                printRedirected($url_status['relocate'], $url_status['path'], $cl);
            }
            $count++;
        }
        if ($url_status['relocate']) {
            $url_status['aborted'] = 1;
            $url_status['state'] = "<br />Indexation aborted because of too many redirections.<br />";
            return $url_status;
        }
        if ($url_status['state'] != "ok") {
            $code = $url_status['state'];
            //  check for most common client errors
            if (!preg_match("/401|402|403|404/", $code)) {
                $url_status['aborted'] = 1;
                //  end indexing for cmplete site
            } else {
                $url_status['aborted'] = '';
                //  abort only for this page
            }
            if (strstr($code, "401")) {
                $code = "401 (Authentication required)";
            }
            if (strstr($code, "403")) {
                $code = "403 (Forbidden)";
            }
            if (strstr($code, "404")) {
                $code = "404 (Not found)";
            }
            $url_status['state'] = "<br />Indexation aborted because of code: {$code}.<br />";
        }
        //  check final URL (which might be the 3. redirection)
        //  and puriify final redirected URL
        $url = $db_con->real_escape_string(url_purify($new_url, $index_url, $can_leave_domain, $care_excl, $relocated, $local_redir));
        // valid file suffix for the redirection??
        if ($url) {
            if ($care_excl == '1') {
                //  care about non-accepted suffixes
                reset($ext);
                while (list($id, $excl) = each($ext)) {
                    if (preg_match("/\\.{$excl}(\$|\\?)/i", $url)) {
                        //  if suffix is at the end of the link, or followd by a question mark
                        $url_status['state'] = 'Found: Not supported suffix';
                        //  error message
                        return $url_status;
                    }
                }
            }
        }
        if (!$url) {
            $link_parts = parse_all_url($url);
            $host = $link_parts['host'];
            $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$index_url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
            $url_status['aborted'] = 1;
            $url_status['state'] = "<br />Indexation aborted because of undefined redirection error.<br />";
            return $url_status;
        }
        //  abort indexation, if the redirected URL is equal to calling URL
        if ($url == 'self') {
            $link_parts = parse_all_url($url);
            $host = $link_parts['host'];
            $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
            $url_status['aborted'] = 1;
            $url_status['state'] = "<br />Indexation aborted for this page, because the redirection was a link in it selves.<br />Blocked by Sphider-plus, because this could end in an infinite indexation loop.<br />";
            return $url_status;
        }
        //  abort indexation, if the redirected URL contains invalid file suffix
        if ($url == 'excl') {
            $link_parts = parse_all_url($url);
            $host = $link_parts['host'];
            $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
            $url_status['aborted'] = 1;
            $url_status['state'] = "<br />Indexation aborted because the redirected link does not meet the URL suffix conditions.<br />";
            return $url_status;
        }
        //  abort indexation, because purifing the redirected URL failed
        if (!strstr($url, "//")) {
            $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
            $url_status['aborted'] = 1;
            $url_status['state'] = "<br />Indexation aborted because {$url} is not supported.<br />";
            return $url_status;
        }
        //  abort indexation, if redirected URL met 'must/must not include' string rule
        if (!check_include($url, $url_inc, $url_not_inc)) {
            $link_parts = parse_all_url($url);
            $host = $link_parts['host'];
            $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
            $url_status['aborted'] = 1;
            $url_status['state'] = "<br />Indexation aborted because the redirected link does not meet<br />the URL 'must include' or 'must not include' conditions.<br />";
            return $url_status;
        }
        //  if redirected URL is already known and in database: abort
        $rows0 = '';
        $rows1 = '';
        mysqltest();
        $sql_query = "SELECT url from " . $mysql_table_prefix . "sites where url like '{$url}'";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $rows0 = $result->num_rows;
        $sql_query = "SELECT * from " . $mysql_table_prefix . "links where url='{$url}'";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $known_link = $result->fetch_array(MYSQLI_NUM);
        $md5 = $known_link[8];
        if ($clear == 1) {
            clean_resource($result, '02');
        }
        if ($rows0) {
            $url_status['state'] = "<br />URL already in database (as a site URL). Index aborted.<br />";
            $url_status['aborted'] = 1;
            return $url_status;
        }
        // if known link, which is already indexed (because containing the md5 checksum), enter here
        if ($known_link[8]) {
            $count = $known_link[15];
            $count++;
            if ($count > $redir_count) {
                //  abort indexation
                $url_status['state'] = "<br />{$count}. attempt to redirect in the same (already indexed) URL, <br />which is no longer accepted by Sphider-plus. Indexation aborted for this site.<br />";
                $url_status['aborted'] = 1;
                return $url_status;
            } else {
                $sql_query = "UPDATE " . $mysql_table_prefix . "links set relo_count='{$count}' where url='{$url}'";
                $db_con->query($sql_query);
            }
        }
        //  add redirected URL to temp table, if not yet known
        $sql_query = "SELECT link from " . $mysql_table_prefix . "temp where link='{$url}' && id = '{$sessid}'";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $rows = $result->num_rows;
        if ($rows == 0) {
            $sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id, relo_count) values ('{$url}', '{$level}', '{$sessid}', '1')";
            $db_con->query($sql_query);
            if ($debug && $db_con->errno) {
                $err_row = __LINE__ - 2;
                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                if (__FUNCTION__) {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                } else {
                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                }
                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                echo "<p> {$sql_query} </p>";
                exit;
            }
        }
        if ($clear == 1) {
            clean_resource($result, '02');
        }
        //  at the end of redirect, rebuild the url parts from the redirected URL.
        //  This is the final URL, which will be indexed
        $url_parts = parse_all_url($url);
    }
    //  end check any redirection/relocation
    //  if a JavaScript file is currently indexed?
    $suffix = substr($url, strrpos($url, ".") + 1);
    $suffix = str_replace("/", "", $suffix);
    if (strlen($suffix) < "5") {
        if (preg_match("/js\$/", $suffix)) {
            $js_link = 1;
            //  activate JS switch
        }
    }
    if ($smp != 1 && $follow_sitemap == 1) {
        //  enter here if we don't already know a valid sitemap and if admin settings allowed us to do so
        $tmp_urls = get_temp_urls($sessid);
        //  reload previous temp
        $url2 = remove_sessid(convert_url($url));
        // get folder where sitemap should be and if exists, cut existing filename, suffix and subfolder
        $host = parse_addr($url2);
        $hostname = $host[host];
        $more_sitemaps = array();
        if ($hostname == 'localhost') {
            $host1 = str_replace($local, '', $url2);
        }
        $pos = strpos($host1, "/");
        //      on local server delete all behind the /
        if ($pos) {
            $host1 = substr($host1, 0, $pos);
        }
        //      build full adress again, now only the host
        if ($hostname == 'localhost') {
            $url2 = "" . $local . "" . $host1 . "";
        } else {
            $url2 = "{$host['scheme']}://{$hostname}";
        }
        $sitemap_name = "sitemap";
        //      standard name for sitemap file
        $input_file = "{$url2}/{$sitemap_name}";
        //      create path to sitemap
        $log_file = './sitemaps/current_sitemap.xml';
        //      destination for sitemap log-file
        $smap_found = '';
        $indexed_map = '';
        $map_cont = '';
        //  try to fetch individual sitemap url from database
        mysqltest();
        $sql_query = "SELECT smap_url from " . $mysql_table_prefix . "sites where site_id='{$site_id}'";
        $result = $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
        $row = $result->fetch_array(MYSQLI_NUM);
        if (preg_match("/http:\\/\\//", $row[0])) {
            //   use the individual sitemap
            $input_file = preg_replace("/.xml.gz|.xml/i", "", $row[0]);
        }
        $file = "" . $input_file . ".xml";
        if ($fd = @fopen($file, "r")) {
            //  uncompressed ?
            //if ($zd = @gzopen("".$input_file.".xml", "r")) {    //  uncompressed ?
            $map_cont = @stream_get_contents($fd);
            if ($map_cont && strpos($map_cont, "schemas/sitemap")) {
                //  if we were able to read it
                $smap_found = '1';
            }
            fclose($fd);
        }
        $gz_file = "" . $input_file . ".xml.gz";
        if (!$smap_found && ($zd = @fopen("compress.zlib://{$gz_file}", "r"))) {
            // compressed  ?
            //if (!$smap_found && $zd = @gzopen("".$input_file.".xml.gz", "r")) {  // compressed  ?
            $map_cont = @gzread($zd, 10485760);
            //  max. 10 MB (might be too large for some server)
            gzclose($zd);
            if ($map_cont && strpos($map_cont, "schemas/sitemap")) {
                $smap_found = '1';
            }
        }
        //echo "\r\n\r\n<br>map_cont Array:<br><pre>";print_r($map_cont);echo "</pre>\r\n";
        if ($smap_found) {
            if ($debug != '0') {
                //      create a log-file of current sitemap.xml
                file_put_contents($log_file, $map_cont);
            }
            //$del = $db_con->query("DELETE from ".$mysql_table_prefix."temp"); // function get_sitemap and store_links will build a new temp table
            if (stristr($map_cont, "<sitemapindex")) {
                //      if current sitemap file is an index file
                printStandardReport('validSitemapInd', $command_line, $no_log);
                $get_maps = simplexml_load_string($map_cont);
                if ($get_maps) {
                    reset($get_maps);
                    foreach ($get_maps as $map_x) {
                        $new_links[] = $map_x->loc;
                        //   get all links to further sitemap files
                    }
                    if (is_array($new_links)) {
                        //      if we found more sitemap files
                        $new_links = explode(",", implode(",", $new_links));
                        // destroy SimpleXMLElement Object and get the link array
                        $new_links = array_slice($new_links, 0, $max_links);
                        $indexed_map = '1';
                        $i = '0';
                        //echo "\r\n\r\n<br>new_links Array:<br><pre>";print_r($new_links);echo "</pre>\r\n";
                        foreach ($new_links as $input_file) {
                            $these_links = get_sitemap($input_file, $indexed_map, $mysql_table_prefix);
                            // now extract page links from this sitemap file
                            //echo "\r\n\r\n<br>these_links Array:<br><pre>";print_r($these_links);echo "</pre>\r\n";
                            if ($these_links) {
                                reset($these_links);
                                store_newLinks($these_links, $level, $sessid);
                                $smp = '1';
                                //     there were valid sitemap files and we stored the new links
                                $i++;
                            } else {
                                printStandardReport('invalidSecSitemap', $command_line, $no_log);
                                //  unable to extract links from secondary sitemap file
                            }
                        }
                        printValidSecSmap($i, $cl);
                        unset($input_file, $map_cont, $new_links);
                    } else {
                        printStandardReport('invalidSecSitemap', $command_line, $no_log);
                        //  unable to extract links from secondary sitemap file
                    }
                } else {
                    printStandardReport('invalidSitemapInd', $command_line, $no_log);
                    //  unable to extract links from sitemap INDEX  file
                }
            } else {
                $links = get_sitemap($map_cont, $indexed_map, $mysql_table_prefix);
                // extract links from sitemap.xml  (there was only one sitemap file)
                if ($links != '') {
                    reset($links);
                    //echo "\r\n\r\n<br>sitemmap links Array:<br><pre>";print_r($links);echo "</pre>\r\n";
                    store_newLinks($links, $level, $sessid);
                    $smp = '1';
                    //     there was one valid sitemap and we stored the new links
                    printStandardReport('validSitemap', $command_line, $no_log);
                } else {
                    printStandardReport('invalidSitemap', $command_line, $no_log);
                }
                unset($links);
            }
        }
    }
    if ($debug == '0') {
        if (function_exists("ini_set")) {
            ini_set("display_errors", "0");
        }
        error_reporting(0);
    } else {
        error_reporting(E_ALL & ~E_DEPRECATED & ~E_WARNING & ~E_NOTICE & ~E_STRICT);
    }
    if ($url_status['state'] == 'ok') {
        $OKtoIndex = 1;
        $file_read_error = 0;
        if (time() - $delay_time < $min_delay) {
            sleep($min_delay - (time() - $delay_time));
        }
        if ($url_status['file']) {
            $file = $url_status['file'];
        } else {
            $url_status['state'] = "Unable to read the content of the file.<br />{$url} does not deliver any content.";
            $realnum--;
        }
    }
    if ($url_status['state'] == 'ok') {
        //  first attempt to define a charset
        $chrSet = '';
        if ($use_prefcharset == '1') {
            //  use preferred charset as defined in Admin settings
            $chrSet = $home_charset;
            //echo "<h1>USING PREFERRED CHARSET</h1>";
        } else {
            if ($server_char && $url_status['charset']) {
                //echo "<h1>USING SERVER CHARSET</h1>";
                $chrSet = $url_status['charset'];
                //  use charset as supplied by the remote server
            } else {
                //  try to extract the charset of this file
                //echo "<h1>USING CONTENT CHARSET</h1>";
                //echo "<h1>" . substr($file, 0, 500) . "</h1>";
                if (preg_match("'encoding=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
                    //echo "<h1>1</h1>";
                    $chrSet = trim(strtoupper($regs[1]));
                    //      get encoding of current XML or XHTML file     and use it furtheron
                }
                if (!$chrSet) {
                    //echo "<h1>2</h1>";
                    if (preg_match("'charset=(.*?)[ \\/\\;\\'\"]'si", substr($file, 0, 3000), $regs)) {
                        //echo "<h1>3</h1>";
                        $chrSet = trim(strtoupper($regs[1]));
                        //      get charset of current HTML file     and use it furtheron
                    }
                }
                if (!$chrSet) {
                    //echo "<h1>4</h1>";
                    if (preg_match("'charset=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
                        //echo "<h1>5</h1>";
                        $chrSet = trim(strtoupper($regs[1]));
                        //      get charset of current HTML file     and use it furtheron
                    }
                }
                //  in assistance for all lazy webmasters
                $chrSet = preg_replace("/win-/si", "windows-", $chrSet);
                if ($chrSet == "1251") {
                    //echo "<h1>6</h1>";
                    $chrSet = "windows-1251";
                }
                if ($chrSet == '') {
                    //echo "<h1>7</h1>";
                    $chrSet = $home_charset;
                    //  no charset found, we need to use default charset like for DOCs, PDFs, etc
                }
            }
        }
        //echo "<h1>CHRSET: $chrSet</h1>";
        //  if required, uncompress ZIP archives and make content of each file => text
        if ($url_status['content'] == 'zip' && $index_zip == '1' && $file) {
            file_put_contents("" . $tmp_dir . "/archiv.temp", $file);
            $zip = zip_open("" . $tmp_dir . "/archiv.temp");
            if ($zip) {
                $url_status['content'] = "text";
                //  preventiv, if not another status will be detected for individual archiv files
                $file = '';
                //  starting with a blank file for all archive files
                $topic = 'zip';
                if ($debug == '2') {
                    printStandardReport('archivFiles', $command_line, $no_log);
                }
                while ($zip_entry = zip_read($zip)) {
                    if (zip_entry_open($zip, $zip_entry, "r")) {
                        $buf = zip_entry_read($zip_entry, zip_entry_filesize($zip_entry));
                        //uncompress the content of recent archiv file
                        $name = zip_entry_name($zip_entry);
                        //  get filename of recent archive file
                        if ($debug == '2') {
                            //
                            $report = "<strong>&nbsp;&nbsp;" . $name . "</strong>";
                            printThis($report, $cl);
                            $size = (int) (zip_entry_filesize($zip_entry) / 1024);
                            if ($size == 0) {
                                $size = '1';
                            }
                            $report = "&nbsp;&nbsp;&nbsp;-&nbsp;Unpacked size:&nbsp;" . $size . " kByte<br />";
                            printThis($report, $cl);
                        }
                        $buf = get_arch_content($buf, $name, $url, $chrSet);
                        //  if necessary, convert PDF, extract feed etc. for the recent file
                        zip_entry_close($zip_entry);
                        //  done for this file in archiv
                        $file .= "" . $buf . "<br /><br />";
                        //  add all uncompressed and converted files together
                    }
                }
                zip_close($zip);
            }
            unlink("" . $tmp_dir . "/archiv.temp");
        }
        //  if required, uncompress RAR archives and make content of each file => text
        if ($url_status['content'] == 'rar' && $index_rar == '1') {
            file_put_contents("" . $tmp_dir . "/archiv.temp", $file);
            $rar = rar_open("" . $tmp_dir . "/archiv.temp");
            if ($rar) {
                $url_status['content'] = "text";
                //  preventiv, all individual archiv files willl be converted to 'text'
                $file = '';
                //  starting with a blank file for all archive files
                $topic = 'rar';
                $entries = rar_list($rar);
                if ($rar) {
                    if ($debug == '2') {
                        printStandardReport('archivFiles', $command_line, $no_log);
                    }
                    foreach ($entries as $entry) {
                        $name = $entry->getName();
                        if ($debug == '2') {
                            $report = "<strong>&nbsp;&nbsp;" . $name . "</strong>";
                            printThis($report, $cl);
                            $size = (int) ($entry->getPackedSize() / 1024);
                            if ($size == 0) {
                                $size = '1';
                            }
                            $report = "&nbsp;&nbsp;&nbsp;-&nbsp;Packed size:&nbsp;&nbsp;" . $size . " kByte";
                            printThis($report, $cl);
                            $size = (int) ($entry->getUnpackedSize() / 1024);
                            if ($size == 0) {
                                $size = '1';
                            }
                            $report = "&nbsp;&nbsp;&nbsp;-&nbsp;Unpacked size:&nbsp;" . $size . " kByte<br />";
                            printThis($report, $cl);
                        }
                        $entry->extract('', "./" . $tmp_dir . "/" . $name . "");
                        //  extract single file of archiv into temporary folder
                        $buf = file_get_contents("./" . $tmp_dir . "/" . $name . "");
                        //  read content of this intermediate file
                        unlink("./" . $tmp_dir . "/" . $name . "");
                        //  destroy this file
                        if ($buf) {
                            $buf = get_arch_content($buf, $name, $url, $chrSet);
                            //  if necessary, convert PDF, extract feed etc. for the recent file
                            $file .= "" . $buf . "<br /><br />";
                            //  add all uncompressed and converted files together
                        }
                    }
                }
                rar_close($rar);
            }
            unlink("" . $tmp_dir . "/archiv.temp");
        }
        $file0 = $file;
        //  rememberr the original (e.g. for doc2txt converter)
        //  remove useless part of the content
        $file = purify_content($file);
        $valid_utf8 = '1';
        $raw_file = $file;
        //  kill eventually duplicate coding info in dynamic links
        if (stristr(substr($file, '0', '4000'), "encoding") && strstr(substr($file, '0', '4000'), "charset")) {
            $file = substr($file, strrpos($file, "<!DOCTYPE"));
            //  subsstring starting at last found <!DOCTYPE
        }
        //  we need to do it again for eventually new charset in archive
        $chrSet = '';
        if ($use_prefcharset == '1') {
            //  use preferred charset as defined in Admin settings
            $chrSet = $home_charset;
        } else {
            if ($server_char && $url_status['charset']) {
                $chrSet = $url_status['charset'];
                //  use charset as supplied by the remote server
            } else {
                //  try to extract the charset of this file
                if (preg_match("'encoding=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
                    $chrSet = trim(strtoupper($regs[1]));
                    //      get encoding of current XML or XHTML file     and use it furtheron
                }
                if (!$chrSet) {
                    if (preg_match("'charset=(.*?)[ \\/\\;\\'\"]'si", substr($file, 0, 3000), $regs)) {
                        $chrSet = trim(strtoupper($regs[1]));
                        //      get charset of current HTML file     and use it furtheron
                    }
                }
                if (!$chrSet) {
                    if (preg_match("'charset=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
                        $chrSet = trim(strtoupper($regs[1]));
                        //      get charset of current HTML file     and use it furtheron
                    }
                }
                //  in assistance for all lazy webmasters
                $chrSet = preg_replace("/win-/si", "windows-", $chrSet);
                if ($chrSet == "1251") {
                    $chrSet = "windows-1251";
                }
                if ($chrSet == '') {
                    $chrSet = $home_charset;
                    //  no charset found, we need to use default charset like for DOCs, PDFs, etc
                }
            }
        }
        if (strpos($chrSet, " ")) {
            // in the wild we have aloready seen a lot of variants
            $chrSet = substr($chrSet, 0, strpos($chrSet, " "));
        }
        //  some webmaster still use 'UNICODE' as name
        if (stristr($chrSet, "UNICODE")) {
            $chrSet = "UTF-8";
        }
        //  obsolete since 1990, but some (Italian) server still send it as charset . . . .
        if (stristr($chrSet, "8858")) {
            $chrSet = str_replace("8858", "8859", $chrSet);
        }
        //  required coaching for some webmasters
        if (stristr($chrSet, "cp-")) {
            $chrSet = str_ireplace("CP-", "CP", $chrSet);
        }
        $contents['charset'] = $chrSet;
        if ($index_framesets == '1') {
            if (preg_match("@<frameset[^>]*>(.*?)<\\/frameset>@si", $file, $regs)) {
                printStandardReport('newFrameset', $command_line, $no_log);
                //  separate the <frameset> ....</frameset> part of this file
                $frame = $regs[1];
                $replace = get_frames($frame, $url, $can_leave_domain);
                $replace = "<body>" . $replace . "</body>";
                //  create the body tags for $file
                $contents['charset'] = $chrSet;
                // rebuild charset
                //  include all replacements instead of the frameset tag into the actual file. This will become the body
                $file = preg_replace("@<frameset.*?</frameset>@si", "{$replace}", $file);
            }
        }
        if ($index_iframes == '1') {
            $links = array();
            $regs = array();
            $replace = '';
            $get_charset = '';
            $real_url = $url;
            if (preg_match_all("/(iframe[^>]*src[[:blank:]]*)=[[:blank:]]*[\\'\"]?(([[a-z]{3,5}:\\/\\/(([.a-zA-Z0-9-])+(:[0-9]+)*))*([+:%\\/?=&;\\\\(\\),._ a-zA-Z0-9-]*))(#[.a-zA-Z0-9-]*)?[\\'\" ]?/i", $file, $regs, PREG_SET_ORDER)) {
                printStandardReport('newIframe', $command_line, $no_log);
                //  find all frames of the iframe;
                $care_excl = '';
                //  don't care file suffixed to be excluded
                $relocated = '';
                //  URL is not relocated
                foreach ($regs as $val) {
                    if (($a = url_purify($val[2], $url, $can_leave_domain, $care_exel, $relocated, $local_redir)) != '') {
                        $links[] = $a;
                        // collect  all iframe links
                    }
                }
                if ($links) {
                    foreach ($links as $url) {
                        printNewLinks($url, $cl);
                        if (preg_match("/.html|.htm|.xhtml|.xml|.php/i", $url)) {
                            $frame = file_get_contents($url);
                            //      get content of this frame
                            //  separate the body part of this frame
                            preg_match("@<body[^>]*>(.*?)<\\/body>@si", $frame, $regs);
                            $body = $regs[1];
                            if ($abslinks == '1') {
                                $body = make_abslinks($body, $url);
                                //  if required, correct links relative to found iframe
                            }
                            $replace = "" . $replace . "<br />" . $body . "";
                        } else {
                            //  might be an image
                            $replace = "" . $replace . "<br /><img src=\"" . $url . "\">";
                        }
                    }
                }
                //  include all replacements instead of the iframe tag into the actual file
                $file = preg_replace("@<iframe.*?</iframe>@si", "{$replace}", $file);
                $contents['charset'] = $chrSet;
                // rebuild charset
            }
            $url = $real_url;
        }
        //      in order to index RDF, RSD, RSS and ATOM feeds enter here
        if ($url_status['content'] == 'xml' && $index_rss == '1') {
            if (!preg_match("/<rss|atom|<feed|<rdf|<rsd/si", substr($file, 0, 400))) {
                printStandardReport('notRSS', $command_line, $no_log);
                //  no valid feed detected
                $OKtoIndex = 0;
                $file_read_error = 1;
                $realnum--;
            } else {
                $html = '';
                $xml = XML_IsWellFormed($file);
                //      check for well-formed XML
                if ($xml != '1') {
                    if ($debug > 0) {
                        printNotWellFormedXML($xml, $cl);
                    }
                    $OKtoIndex = 0;
                    $file_read_error = 1;
                    $realnum--;
                } else {
                    $rss = new feedParser();
                    // define options for feed parser
                    $rss->limit = $max_links;
                    //   save time by limiting the items/entries to be processed
                    $rss->in_cp = strtoupper($contents['charset']);
                    //  charset of actual file
                    $rss->out_cp = 'UTF-8';
                    //  convert all into this charset
                    $rss->cache_dir = '';
                    //  currently unused
                    $rss->dc = $dc;
                    //  treat Dublin Core tags in RDF feeds
                    $rss->pro = $preferred;
                    //  obey the PREFERRED directive in RSD feeds
                    $rss->file = '1';
                    //  use $file as feed (as a string, not URL)
                    if ($cdata != 1) {
                        $rss->CDATA = 'content';
                        //  get it all  (naughty)
                    } else {
                        $rss->CDATA = 'nochange';
                        //  well educated crawler
                    }
                    //  get feed as array
                    if ($feed = $rss->get($url, $file)) {
                        //  if you want to see the feed during index procedure, uncomment the following row
                        //  echo "<br>FEED array:<br><pre>";print_r($feed);echo "</pre>";
                        $link = '';
                        $textinput_link = '';
                        $image_url = '';
                        $image_link = '';
                        $docs = '';
                        $subjects = '';
                        $count = '';
                        $type = $feed[type];
                        $count = $feed[sub_count];
                        $cached = $feed[cached];
                        //  kill all no longer required values
                        $feed[type] = '';
                        $feed[sub_count] = '';
                        $feed[encoding_in] = '';
                        $feed[encoding_out] = '';
                        $feed[items_count] = '';
                        $feed[cached] = '';
                        if (!$count) {
                            $count = '0';
                        }
                        if ($type == 'RSD') {
                            //      prepare all RSD APIs
                            for ($i = 0; $i < $count; $i++) {
                                $subjects .= '' . $feed['api'][$i]['name'] . '<br />
                                            ' . $feed['api'][$i]['apiLink'] . '<br />
                                            ' . $feed['api'][$i]['blogID'] . '<br />
                                            ' . $feed['api'][$i]['settings_docs'] . '<br />
                                            ' . $feed['api'][$i]['settings_notes'] . '<br />';
                            }
                        }
                        if ($type == 'Atom') {
                            //      prepare all Atom entries
                            for ($i = 0; $i < $count; $i++) {
                                $subjects .= '' . $feed['entries'][$i]['link'] . '<br />
                                            ' . $feed['entries'][$i]['title'] . '<br />
                                            ' . $feed['entries'][$i]['id'] . '<br />
                                            ' . $feed['entries'][$i]['published'] . '<br />
                                            ' . $feed['entries'][$i]['updated'] . '<br />
                                            ' . $feed['entries'][$i]['summary'] . '<br />
                                            ' . $feed['entries'][$i]['rights'] . '<br />
                                            ' . $feed['entries'][$i]['author_name'] . ' ' . $feed['entries'][$i]['author_email'] . ' ' . $feed['entries'][$i]['author_uri'] . '<br />
                                            ' . $feed['entries'][$i]['category_term'] . ' ' . $feed['entries'][$i]['category_label'] . ' ' . $feed['entries'][$i]['category_scheme'] . '<br />
                                            ' . $feed['entries'][$i]['contributor_name'] . ' ' . $feed['entries'][$i]['contributor_email'] . ' ' . $feed['entries'][$i]['contributor_uri'] . '<br />
                                        ';
                            }
                        }
                        if ($type == 'RDF' | $type == 'RSS v.0.91/0.92' | $type == 'RSS v.2.0') {
                            //  For RDF and RSS feeds enter here
                            //  prepare channel image
                            $image_url = $feed[image_url];
                            if ($image_url) {
                                $width = $feed[image_width];
                                if (!$width || $width > '144') {
                                    $width = '88';
                                    //set to default value
                                }
                                $height = $feed[image_height];
                                if (!$height || $height > '400') {
                                    $height = '31';
                                    //set to default value
                                }
                                $feed[image_url] = "<img id=\"rss_007\" src=\"" . $image_url . "\" alt=\"" . $feed[image_title] . "\" width=\"" . $width . "\" height=\"" . $height . "\">";
                            }
                            $image_link = $feed[image_link];
                            if ($image_link) {
                                $feed[image_link] = "<a href=\"" . $image_link . "\">" . $image_link . "</a>";
                            }
                            //      prepare all RDF or RSS items
                            for ($i = 0; $i < $count; $i++) {
                                $subjects .= '' . $feed['items'][$i]['link'] . '<br />
                                            ' . $feed['items'][$i]['title'] . '<br />
                                            ' . $feed['items'][$i]['description'] . '<br />
                                            ' . $feed['items'][$i]['author'] . '<br />
                                            ' . $feed['items'][$i]['category'] . '<br />
                                            ' . $feed['items'][$i]['guid'] . '<br />
                                            ' . $feed['items'][$i]['comments'] . '<br />
                                            ' . $feed['items'][$i]['pubDate'] . '<br />
                                            ' . $feed['items'][$i]['source'] . '<br />
                                            ' . $feed['items'][$i]['enclosure'] . '<br />
                                            ' . $feed['items'][$i]['country'] . '<br />
                                            ' . $feed['items'][$i]['coverage'] . '<br />
                                            ' . $feed['items'][$i]['contributor'] . '<br />
                                            ' . $feed['items'][$i]['date'] . '<br />
                                            ' . $feed['items'][$i]['industry'] . '<br />
                                            ' . $feed['items'][$i]['language'] . '<br />
                                            ' . $feed['items'][$i]['publisher'] . '<br />
                                            ' . $feed['items'][$i]['state'] . '<br />
                                            ' . $feed['items'][$i]['subject'] . '<br />
                                        ';
                            }
                        }
                        //  convert  the channel/feed part  into a string
                        $feed_common = implode(" ", $feed);
                        //  build something that could be indexed
                        $html .= "<html>\r\n<head>\r\n<title>" . $feed['title'] . "</title>\r\n<meta name=\"description\" content=\"" . $feed['description'] . " \">\r\n</head>\r\n";
                        $html .= "<body>\r\n" . $feed_common . "\r\n" . $subjects . "\r\n</body>\r\n</html>\r\n";
                    }
                    if (strlen($html) < "130") {
                        //  can't be a valid feed
                        if ($type == "unknown") {
                            printInvalidFeedType($type, $cl);
                        } else {
                            printStandardReport('invalidRSS', $command_line, $no_log);
                        }
                        $OKtoIndex = 0;
                        $file_read_error = 1;
                        $realnum--;
                    } else {
                        $contents['charset'] = 'UTF-8';
                        //      the feed reader converts all to utf-8
                        $file = $html;
                        //     use feed reader output
                        if ($debug > 0) {
                            printValidFeed($type, $count, $cl);
                        }
                    }
                }
            }
        }
        //  duplicate here, but frames, iframes, or RSS might have added nonsense content
        $file = purify_content($file);
        //  prepare CVS files
        if ($url_status['content'] == 'csv' && $index_csv == '1') {
            $file = str_replace(",", " ", $file);
            $file = str_replace(";", " ", $file);
        }
        //echo "\r\n\r\n<br>url_status Array:<br><pre>";print_r($url_status);echo "</pre>\r\n";
        // for DOCs, PDFs, etc we need special text converter
        if ($url_status['content'] != 'text' && $url_status['content'] != 'xml' && $url_status['content'] != 'xhtml' && $url_status['content'] != 'csv') {
            $document = 1;
            $file = extract_text($file, $file0, $url_status['content'], $url, $chrSet);
            //  because the converter already transferred the documents to UTF-8, we need to adjust it here
            $contents['charset'] = 'UTF-8';
            $charSet = 'UTF-8';
            if ($file == 'ERROR') {
                //      if error, suppress further indexing
                $OKtoIndex = 0;
                $file_read_error = 1;
                $realnum--;
            }
            //  reduce Pashtu and Urdu to the main Farsi letters
            if (strtolower($charSet) == 'windows-1256' && $url_status['content'] == 'pdf') {
                $f_letter0 = array("ﺎ", "�");
                $f_letter1 = array("�", "�", "ﺑ", "ﺒ");
                $f_letter2 = array("ï­–", "ï­—", "ï­˜", "ï­™");
                $f_letter3 = array("ﺕ", "ﺖ", "ﺗ", "ﺘ");
                $f_letter4 = array("ﺙ", "ﺚ", "ﺛ", "ﺜ");
                $f_letter5 = array("�", "ﺞ", "ﺟ", "ﺠ");
                $f_letter6 = array("ï­º", "ï­»", "ï­¼", "ï­½");
                $f_letter7 = array("ﺡ", "ﺢ", "ﺣ", "ﺤ");
                $f_letter8 = array("ﮋ", "ﮊ");
                $f_letter9 = array("ﺥ", "ﺦ", "ﺧ", "ﺨ");
                $f_letter10 = array("ﺩ", "ﺪ");
                $f_letter11 = array("ﺫ", "ﺬ");
                $f_letter12 = array("ﺭ", "ﺮ");
                $f_letter13 = array("ﺯ", "ﺰ");
                $f_letter14 = array("ﺱ", "ﺲ", "ﺳ", "ﺴ");
                $f_letter15 = array("ﺵ", "ﺶ", "ﺷ", "ﺸ");
                $f_letter16 = array("ﺹ", "ﺺ", "ﺻ", "ﺼ");
                $f_letter17 = array("ﺽ", "ﺾ", "ﺿ", "ﻀ");
                $f_letter18 = array("�", "ﻂ", "ﻃ", "ﻄ");
                $f_letter19 = array("ﻅ", "ﻆ", "ﻇ", "ﻈ");
                $f_letter20 = array("ﻉ", "ﻊ", "ﻋ", "ﻌ");
                $f_letter21 = array("�", "ﻎ", "�", "�");
                $f_letter22 = array("ﻑ", "ﻒ", "ﻓ", "ﻔ");
                $f_letter23 = array("ﻕ", "ﻖ", "ﻗ", "ﻘ");
                $f_letter24 = array("ﻙ", "ﻚ", "ﻛ", "ﻜ", "ﮎ", "�", "�", "ﮑ");
                $f_letter25 = array("ﮒ", "ﮓ", "ﮔ", "ﮕ");
                $f_letter26 = array("�", "ﻞ", "ﻟ", "ﻠ");
                $f_letter27 = array("ﻡ", "ﻢ", "ﻣ", "ﻤ");
                $f_letter28 = array("ﻧ", "ﻨ", "ﻦ", "ﻥ");
                $f_letter29 = array("ï»­", "ï»®");
                $f_letter30 = array("ﻩ", "ﻪ", "ﻫ", "ﻬ");
                $f_letter31 = array("ﻯ", "ﻰ", "ﻱ", "ﻲ", "ﻳ", "ﻴ");
                $file = str_replace($f_letter0, "ا", $file);
                $file = str_replace($f_letter1, "ب", $file);
                $file = str_replace($f_letter2, "Ù¾", $file);
                $file = str_replace($f_letter3, "ت", $file);
                $file = str_replace($f_letter4, "Ø«", $file);
                $file = str_replace($f_letter5, "ج", $file);
                $file = str_replace($f_letter6, "Ú†", $file);
                $file = str_replace($f_letter7, "Ø­", $file);
                $file = str_replace($f_letter8, "Ú˜", $file);
                $file = str_replace($f_letter9, "Ø®", $file);
                $file = str_replace($f_letter10, "د", $file);
                $file = str_replace($f_letter11, "Ø°", $file);
                $file = str_replace($f_letter12, "ر", $file);
                $file = str_replace($f_letter13, "ز", $file);
                $file = str_replace($f_letter14, "س", $file);
                $file = str_replace($f_letter15, "Ø´", $file);
                $file = str_replace($f_letter16, "ص", $file);
                $file = str_replace($f_letter17, "ض", $file);
                $file = str_replace($f_letter18, "Ø·", $file);
                $file = str_replace($f_letter19, "ظ", $file);
                $file = str_replace($f_letter20, "ع", $file);
                $file = str_replace($f_letter21, "غ", $file);
                $file = str_replace($f_letter22, "Ù�", $file);
                $file = str_replace($f_letter23, "Ù‚", $file);
                $file = str_replace($f_letter24, "Ú©", $file);
                $file = str_replace($f_letter25, "Ú¯", $file);
                $file = str_replace($f_letter26, "Ù„", $file);
                $file = str_replace($f_letter27, "Ù…", $file);
                $file = str_replace($f_letter28, "Ù†", $file);
                $file = str_replace($f_letter29, "Ùˆ", $file);
                $file = str_replace($f_letter30, "Ù‡", $file);
                $file = str_replace($f_letter31, "ÙŠ", $file);
            }
        }
        if ($OKtoIndex == 1) {
            $pageSize = number_format(strlen($file) / 1024, 2, ".", "");
            printPageSizeReport($pageSize, $topic);
        }
        $charSet = strtoupper(trim($contents['charset']));
        //      final charset for UTF-8 converter
        if (stristr($charSet, "encoding") || strlen($charSet) < '3') {
            //  must be invalid encountered charset
            $charSet = 'UTF-8';
        }
        //echo "\r\n\r\n<br /> final charSet: '$charSet'<br />\r\n";
        if ($charSet == "UTF-16") {
            $charSet = "UTF-8";
            //  content will be converted in function clean_file()
        }
        $dic = '';
        //  if Chinese or Korean text should be segmented enter here
        if ($cn_seg == '1' && $file && !$js_link && !stristr($charSet, "8859")) {
            if ($charSet == 'GB2312' || $charSet == 'GB18030' || $charSet == 'GBK') {
                $dic = "" . $dict_dir . "/cn_gb18030.dic";
                //  simplified Chinese
            }
            if ($charSet == 'BIG5') {
                $dic = "" . $dict_dir . "/cn_big5.dic";
                //  traditional Chinese
            }
            if ($charSet == 'ISO10646-1933') {
                $dic = "" . $dict_dir . "/kr_iso10646-1933.dic";
                // Korean
            }
            if ($charSet == 'EUC-KR') {
                $dic = "" . $dict_dir . "/kr_euc-kr.dic";
                //  Korean
            }
            if ($charSet == 'UTF-8') {
                $dic = "" . $dict_dir . "/cn_utf-8.dic";
                //  Unicode
            }
            if ($dic) {
                //  if dictionary is available for page charset, perform a segmentation
                $Segmentation = new Segmentation();
                $Segmentation->load($dic);
                $Segmentation->setLowercase(FALSE);
                $cn_result = $Segmentation->segmentString($file);
                if ($cn_result && $charSet != 'UTF-8') {
                    $iconv_file = @iconv($charSet, "UTF-8//IGNORE", $cn_result);
                    if (trim($iconv_file) == "") {
                        // iconv is not installed or input charSet not available. We need to use class ConvertCharset
                        $NewEncoding = new ConvertCharset($charSet, "utf-8");
                        $NewFileOutput = $NewEncoding->Convert($cn_result);
                        $cn_result = $NewFileOutput;
                    } else {
                        $cn_result = $iconv_file;
                    }
                    unset($iconv_file, $NewEncoding, $NewFileOutput);
                }
                $seg_data = clean_file($cn_result, $url, $url_status['content'], $charSet, $use_nofollow, $use_robot, $can_leave_domain);
            } else {
                printNoDictionary($charSet, $cl);
                //  no dictionary found for this charset
            }
        }
        //  if Japanese text should be segmented enter here. But not if a Chinese dictonary was already found
        if ($jp_seg == '1' && $file && !$js_link && !stristr($charSet, "ISO") && !$dic) {
            $dic = '';
            if ($charSet == 'UTF-8' || $charSet == 'EUC-JP') {
                $file = @iconv($charSet, "SHIFT_JIS//IGNORE", $file);
                $charSet = "SHIFT_JIS";
            }
            if ($charSet == 'SHIFT_JIS') {
                $dic = "" . $dict_dir . "/jp_shiftJIS.dic";
            }
            if ($dic) {
                //  if dictionary is available for page charset, perform a segmentation
                $Segmentation = new Segmentation();
                $Segmentation->load($dic);
                $Segmentation->setLowercase(FALSE);
                $jp_result = $Segmentation->segmentString($file);
                //echo "\r\n\r\n<br /> jp_result: $jp_result<br />\r\n";
                if ($jp_result && $charSet != 'UTF-8') {
                    $iconv_file = @iconv($charSet, "UTF-8//IGNORE", $jp_result);
                    if (trim($iconv_file) == "") {
                        // iconv is not installed or input charSet not available. We need to use class ConvertCharset
                        $NewEncoding = new ConvertCharset($charSet, "utf-8");
                        $NewFileOutput = $NewEncoding->Convert($jp_result);
                        $jp_result = $NewFileOutput;
                    } else {
                        $jp_result = $iconv_file;
                    }
                    unset($iconv_file, $NewEncoding, $NewFileOutput);
                }
                $seg_data = clean_file($jp_result, $url, $url_status['content'], $charSet, $use_nofollow, $use_robot, $can_leave_domain);
            } else {
                printNoDictionary($charSet, $cl);
                //  no dictionary found for this charset
            }
        }
        //  enter here only, if site / file is not yet UTF-8 coded or had already been converted to UTF-8
        if ($charSet != "UTF-8" && $file) {
            $file = convertToUTF8($file, $charSet, $char_Set, $converter_dir);
        }
        //  if activated in Admin backend, check for correct converting of $file into UTF-8
        if ($utf8_verify) {
            $valid_utf8 = @iconv('UTF-8', 'UTF-8', $file) === $file;
        }
        if (!$valid_utf8) {
            $url_status['state'] = "<br />Invalid charset definition placed in meta tags of HTML header. Unable to convert the text into UTF-8<br />Indexing aborted for {$url}";
            if ($server_char) {
                $url_status['state'] = "<br />Invalid charset definition supplied via HTTP by the client server. Unable to convert the text into UTF-8<br />Indexing aborted for {$url}";
            }
            if ($use_prefcharset) {
                $url_status['state'] = "<br />Invalid charset definition placed Admin Settings.<br />Site was created with another charset<br />Indexing aborted for {$url}";
            }
            printUrlStatus($url_status['state'], $command_line, $no_log);
            $file = '';
            $deletable = 1;
        } else {
            if ($index_media == '1') {
                $newmd5sum = md5($file);
                //  get md5 including links and title of media files
            }
            $data = clean_file($file, $url, $url_status['content'], $charSet, $use_nofollow, $use_robot, $can_leave_domain);
            //echo "\r\n\r\n<br>data Array:<br><pre>";print_r($data);echo "</pre>\r\n";
            //  index only links and their titles
            if ($only_links) {
                $media_links = '0';
                $my_links = get_link_details($file, $url, $can_leave_domain, $data['base'], $media_links, $use_nofollow, $local_redir);
                $data['content'] = $my_links[0][0];
                //  define new content
                $data['fulltext'] = $my_links[0][0];
                //  define new content also for 'full text';
            }
            //  combine raw words plus segmented  words
            if ($cn_seg == 1 || $jp_seg == 1 && $dic && !$js_link) {
                if ($debug != '0') {
                    $seg_add = $seg_data[count] - $data[count];
                    //      calculate segmentation result
                    if ($seg_add > '0') {
                        if ($charSet == 'EUC-KR' || $charSet == 'ISO10646-1933') {
                            printSegKR($seg_add, $cl);
                        }
                        if ($charSet == 'SHIFT_JIS') {
                            printSegJA($seg_add, $cl);
                        } else {
                            printSegCN($seg_add, $cl);
                        }
                    }
                    /*
                    echo "<br /><pre>Results of word segmentation:</pre>";
                    echo "<br />Unsegmented title :<br><pre>";print_r($data[title]);echo "</pre>";
                    echo "<br />Segmented title :<br><pre>";print_r($seg_data[title]);echo "</pre>";
                    echo "<br />Unsegmented full text:<br />$data[fulltext]<br />";
                    echo "<br />Segmented full text:<br />$seg_data[fulltext]";
                    */
                }
                $data[content] = "" . $data[content] . "" . $seg_data[content] . "";
                //$data[title]        ="".$data[title]."".$seg_data[title]."";
                $data[description] = "" . $data[description] . "" . $seg_data[description] . "";
                $data[keywords] = "" . $data[keywords] . "" . $seg_data[keywords] . "";
            }
            //      check if canonical redirection was found in page head
            $cano_link = '0';
            if ($data['cano_link']) {
                //echo "\r\n\r\n<br /> url: '$url'<br />\r\n";
                $cano_link = $db_con->real_escape_string($data['cano_link']);
                //echo "\r\n\r\n<br /> cano_link: '$cano_link'<br />\r\n";
                if ($url != $cano_link) {
                    //  only new cano links are accepted
                    $OKtoIndex = 0;
                    $deletable = 1;
                    $realnum--;
                    if ($cano_link == "1") {
                        printNoCanonical($cano_link, $cl);
                        //  if unable to extract redirection link
                    } else {
                        if ($data['refresh'] == '1') {
                            printRefreshed($cano_link, $data['wait'], $cl);
                            //  if refresh meta tag was found in HTML head
                        } else {
                            printCanonical($cano_link, $cl);
                            //  if canonical link was found in HTML head
                        }
                        //      do we already know this link in link-table
                        $sql_query = "SELECT /* jfield 2 */ url from " . $mysql_table_prefix . "links where url like '{$cano_link}'";
                        $res = $db_con->query($sql_query);
                        if ($debug && $db_con->errno) {
                            $err_row = __LINE__ - 2;
                            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                            if (__FUNCTION__) {
                                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                            } else {
                                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                            }
                            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                            echo "<p> {$sql_query} </p>";
                            exit;
                        }
                        $rows = $res->num_rows;
                        if ($rows == 0) {
                            // if not known in link-table, check if already known in temp-table
                            $sql_query = "SELECT /* jfield 1 */ link from " . $mysql_table_prefix . "temp where link like '{$cano_link}'";
                            $res = $db_con->query($sql_query);
                            if ($debug && $db_con->errno) {
                                $err_row = __LINE__ - 2;
                                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                if (__FUNCTION__) {
                                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                } else {
                                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                }
                                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                echo "<p> {$sql_query} </p>";
                                exit;
                            }
                            $rows = $res->num_rows;
                            if ($rows == 0) {
                                // not known in link-table, add new link
                                if ($numoflinks <= $max_links) {
                                    $sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id) values ('{$cano_link}', '{$level}', '{$sessid}')";
                                    $db_con->query($sql_query);
                                }
                                if ($debug && $db_con->errno) {
                                    $err_row = __LINE__ - 2;
                                    printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                    if (__FUNCTION__) {
                                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                    } else {
                                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                    }
                                    printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                    printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                    echo "<p> {$sql_query} </p>";
                                    exit;
                                }
                            }
                        }
                    }
                }
                $cano_link = '0';
                //  reset the cano flag
            } else {
                if ($index_media == '0') {
                    $newmd5sum = md5($data['content']);
                    // get md5 from cleaned full text only
                }
                if ($md5sum == $newmd5sum) {
                    printStandardReport('md5notChanged', $command_line, $no_log);
                    $OKtoIndex = 0;
                    $realnum--;
                } else {
                    mysqltest();
                    //     check for duplicate page content
                    $sql_query = "SELECT * from " . $mysql_table_prefix . "links where md5sum='{$newmd5sum}'";
                    $result = $db_con->query($sql_query);
                    if ($debug && $db_con->errno) {
                        $err_row = __LINE__ - 2;
                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                        if (__FUNCTION__) {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                        } else {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                        }
                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                        echo "<p> {$sql_query} </p>";
                        exit;
                    }
                    if ($num_rows = $result->num_rows) {
                        //  display warning message and urls with duplicate content
                        printStandardReport('duplicate', $command_line, $no_log);
                        while ($row = $result->fetch_array(MYSQLI_ASSOC)) {
                            $dups[] = $row['link_id'];
                        }
                        for ($i = 0; $i < $num_rows; $i++) {
                            $link_id = $dups[$i];
                            //$num = $i+1;
                            $sql_query = "SELECT * from " . $mysql_table_prefix . "links where link_id like '{$link_id}'";
                            $res1 = $db_con->query($sql_query);
                            if ($debug && $db_con->errno) {
                                $err_row = __LINE__ - 2;
                                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                if (__FUNCTION__) {
                                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                } else {
                                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                }
                                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                echo "<p> {$sql_query} </p>";
                                exit;
                            }
                            $row = $res1->fetch_array(MYSQLI_NUM);
                            $dup_url = urldecode($row[2]);
                            $dup_url = $dup_url;
                            $dup_url = @iconv($charSet, "UTF-8//IGNORE", $dup_url);
                            if ($idna) {
                                // Initialize the converter class
                                $IDN = new idna_convert(array('idn_version' => 2008));
                                if ($conv_puny && strstr($dup_url, "xn--") && $idna) {
                                    $dup_url = $IDN->decode($dup_url);
                                }
                            }
                            if ($clear == 1) {
                                clean_resource($res, '03');
                            }
                            printDupReport($dup_url, $command_line);
                        }
                        if ($dup_content == '0') {
                            //  enter here, if pages with duplicate content should not be indexed/re-indexed
                            $OKtoIndex = 0;
                            $realnum--;
                        } else {
                            $OKtoIndex = 1;
                        }
                    }
                }
            }
            //echo "\r\n\r\n<br>data array1:<br><pre>";print_r($data);echo "</pre>\r\n";
            if (($md5sum != $newmd5sum || $reindex == 1) && $OKtoIndex == 1) {
                $urlparts = parse_addr($url);
                $newdomain = $urlparts['host'];
                $type = 0;
                if ($data['noindex'] == 1) {
                    //  remember this URlL, so it might not become another time a new link
                    //  check without scheme and www.
                    $check_link = substr($check_link, stripos($url, "//") + 2);
                    if (stristr($check_link, "www.")) {
                        $check_link = substr($check_link, stripos($check_link, "www") + 4);
                    }
                    $sql_query = "SELECT url from " . $mysql_table_prefix . "links where url like '%{$check_link}'";
                    $res = $db_con->query($sql_query);
                    if ($debug && $db_con->errno) {
                        $err_row = __LINE__ - 2;
                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                        if (__FUNCTION__) {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                        } else {
                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                        }
                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                        echo "<p> {$sql_query} </p>";
                        exit;
                    }
                    $known_link = $res->num_rows;
                    if ($known_link != '1') {
                        $sql_query = "INSERT into " . $mysql_table_prefix . "links (site_id, url, indexdate, size, md5sum, level) values ('{$site_id}', '{$url}', curdate(), '{$pageSize}', '{$newmd5sum}', '{$thislevel}')";
                        $db_con->query($sql_query);
                        if ($debug && $db_con->errno) {
                            $err_row = __LINE__ - 2;
                            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                            if (__FUNCTION__) {
                                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                            } else {
                                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                            }
                            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                            echo "<p> {$sql_query} </p>";
                            exit;
                        }
                    }
                    $OKtoIndex = 0;
                    $deletable = 1;
                    $realnum--;
                    printStandardReport('metaNoindex', $command_line, $no_log);
                }
                if (!$js_link) {
                    //  JavaScript will not deliver keywords, only links are parsed
                    $content = explode(" ", addslashes($data['content']));
                    //echo "\r\n\r\n<br>content array0:<br><pre>";print_r($content);echo "</pre>\r\n";
                    $acc_words[] = array();
                    $type = '';
                    //  if Greek accents should be removed from Greek vowels
                    if ($noacc_el) {
                        foreach ($content as &$thisword) {
                            $no_acc = remove_acc_el($thisword);
                            if ($no_acc != $thisword) {
                                $acc_words[] = $no_acc;
                            }
                        }
                    }
                    //  if the other (Latin)  accents should be removed from their vowels
                    if ($vowels) {
                        foreach ($content as $thisword) {
                            $no_acc = remove_acc($thisword, '');
                            if ($no_acc != $thisword) {
                                $acc_words[] = $no_acc;
                            }
                        }
                    }
                    //  now add the words without accents to the total text content
                    $content = array_merge($content, $acc_words);
                    //echo "\r\n\r\n<br>content array0:<br><pre>";print_r($content);echo "</pre>\r\n";
                    //  if ligatures should be equalized
                    if ($liga) {
                        $liga_words = array();
                        //  will contain converted ligatures
                        $phon_words = array();
                        //  will contain converted phonetics
                        //  first: convert letters into latin ligatures
                        foreach ($content as $thisword) {
                            if ($thisword) {
                                $liga_words[] = html_entity_decode($thisword, ENT_QUOTES, "UTF-8");
                                $thisword1 = $thisword;
                                reset($latin_ligatures);
                                while ($char = each($latin_ligatures)) {
                                    $thisword2 = preg_replace("/" . $char[0] . "/s", $char[1], $thisword1);
                                    //  convert ligatures
                                    if ($thisword1 != $thisword2) {
                                        //  break on first ligature
                                        $liga_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
                                        //  collect new words with ligatures
                                        $thisword1 = $thisword2;
                                        //  continue with the word, containing the ligatures
                                        //break;
                                    }
                                }
                            }
                        }
                        // second: convert all letters into phonetic transcriptions
                        reset($liga_words);
                        foreach ($liga_words as $thisword) {
                            $thisword1 = $thisword;
                            reset($phon_trans);
                            while ($char = each($phon_trans)) {
                                $thisword2 = preg_replace("/" . $char[0] . "/s", $char[1], $thisword1);
                                //  convert into phonetics
                                if ($thisword1 != $thisword2) {
                                    //  break on first ligature
                                    $phon_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
                                    //  collect new words with phonetics
                                    $thisword1 = $thisword2;
                                    //  continue with the word, containing the ligatures
                                    //break;
                                }
                            }
                        }
                        $liga_words = array_merge($liga_words, $phon_words);
                        //  add all phoneticss to the liga array
                        //  now vice versa: convert latin ligatures and phonetic transcriptions into standard letters
                        reset($content);
                        $not_liga_words = array();
                        foreach ($content as $thisword) {
                            if ($thisword) {
                                //  first: convert latin ligatures into standard letters
                                $thisword1 = superentities($thisword, ENT_QUOTES, "UTF-8");
                                reset($latin_ligatures);
                                while ($char = each($latin_ligatures)) {
                                    $thisword2 = preg_replace("/" . $char[1] . "/s", $char[0], $thisword1);
                                    //  re-convert ligatures
                                    if ($thisword1 != $thisword2) {
                                        $not_liga_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
                                        //  collect new words without ligatures
                                        $thisword1 = $thisword2;
                                        //  continue with the word, containing the ligature
                                    }
                                }
                            }
                            //echo "\r\n\r\n<br>not_liga_words Array:<br><pre>";print_r($not_liga_words);echo "</pre>\r\n";
                            // second: convert phonetic transcriptions into standard letters
                            reset($not_liga_words);
                            $not_phon_words = array();
                            foreach ($not_liga_words as $thisword) {
                                $thisword1 = superentities($thisword, ENT_QUOTES, "UTF-8");
                                reset($phon_trans);
                                while ($char = each($phon_trans)) {
                                    $thisword2 = preg_replace("/" . $char[1] . "/s", $char[0], $thisword1);
                                    //  re-convert sphonetic
                                    if ($thisword1 != $thisword2) {
                                        $not_phon_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
                                        //  collect new words without phonetics
                                        $thisword1 = $thisword2;
                                        //  continue with the word, containing the phonetic trans.
                                    }
                                }
                            }
                        }
                        $not_words = array_merge($not_liga_words, $not_phon_words);
                        //  add all together
                        $content = array_merge($liga_words, $not_words);
                        //  add all ligatures and re-converted letters to the content array
                    }
                    $wordarray = unique_array($content);
                }
                //echo "\r\n\r\n<br>wordarray0:<br><pre>";print_r($wordarray);echo "</pre>\r\n";
                if ($smp != 1) {
                    if ($data['nofollow'] != 1 && $cano_link == '0') {
                        $media_links = '0';
                        $links = array();
                        if (!$document) {
                            //  don't try to find links in PDFs and other pure documents
                            $links = get_links($file, $url, $can_leave_domain, $data['base'], $media_links, $use_nofollow, $local_redir, $url_reloc, $charSet);
                        }
                        if ($links[0]) {
                            $links = distinct_array($links);
                            $all_links = count($links);
                            if ($all_links > $max_links) {
                                $all_links = $max_links;
                            }
                            $links = array_slice($links, 0, $max_links);
                            if ($realnum < $max_links) {
                                $numoflinks = 0;
                                //if there are any new links, add to the temp table, but only if there isn't such url already
                                if ($links[0]) {
                                    reset($links);
                                    $tmp_urls = get_temp_urls($sessid);
                                    //  reload previous temp
                                    // echo "\r\n\r\n<br>tmp_urls array:<br><pre>";print_r($tmp_urls);echo "</pre>\r\n";
                                    if ($debug == '2') {
                                        //  if debug mode, show details
                                        printStandardReport('newLinks', $command_line, $no_log);
                                    }
                                    while ($thislink = each($links)) {
                                        // echo "\r\n\r\n<br>thislink array:<br><pre>";print_r($thislink);echo "</pre>\r\n";
                                        //  ignore error (message) links and self linking
                                        if (strstr($thislink[1], "//") && $thislink[1] != $url) {
                                            //  find new domains for _addurl table
                                            if ($auto_add && $can_leave_domain) {
                                                $all_link = parse_all_url($thislink[1]);
                                                //  only the domain will be stored as new URL into addurl table
                                                $dom_link = $all_link['host'];
                                                //  reduce to domain name and tld
                                                $new_link = str_replace("www.", "", $dom_link);
                                                // use the complete URL
                                                //$dom_link = $thislink[1];
                                                //  use only the domain
                                                $dom_link = $all_link['scheme'] . "://" . $dom_link;
                                                $banned = '';
                                                mysqltest();
                                                //     check whether URL is already known in sites table
                                                $sql_query = "SELECT url from " . $mysql_table_prefix . "sites where url like '%{$new_link}%'";
                                                $res1 = $db_con->query($sql_query);
                                                //     check whether URL is already known in addurl table
                                                $sql_query = "SELECT url from " . $mysql_table_prefix . "addurl where url like '%{$new_link}%'";
                                                $res2 = $db_con->query($sql_query);
                                                //     check whether URL is banned
                                                $sql_query = "SELECT domain from " . $mysql_table_prefix . "banned where domain like '%{$new_link}%'";
                                                $res3 = $db_con->query($sql_query);
                                                if ($res3->num_rows) {
                                                    $banned = "1";
                                                }
                                                if ($res1->num_rows == 0 && $res2->num_rows == 0 && $res3->num_rows == 0) {
                                                    //  add new domain into _addurl table
                                                    $sql_query = "INSERT into " . $mysql_table_prefix . "addurl (url, description, account) values ('{$dom_link}', '{$comment}', '{$admin_email}')";
                                                    $db_con->query($sql_query);
                                                }
                                            }
                                            //      check whether thislink is already known as a link ( might happen by means of relocated URLs)
                                            $res4 = '';
                                            $res5 = '';
                                            $known_link = '';
                                            $known_temp = '';
                                            $check_link = $thislink[1];
                                            // i don't believe the "like" is necessary here and it slows down indexing
                                            //                                                //  check without scheme and www.
                                            //                                                $check_link = substr($check_link, stripos($check_link, "//")+2);
                                            //                                                if (stristr($check_link, "www.")) {
                                            //                                                    $check_link = substr($check_link, stripos($check_link, "www")+4);
                                            //                                                }
                                            //
                                            //                                                $sql_query = "SELECT /* jfield 3 */ url from ".$mysql_table_prefix."links where url like '%$check_link'";
                                            //                                                $res4 = $db_con->query($sql_query);
                                            //
                                            //                                                $known_link = $res4->num_rows;;
                                            //
                                            //                                                $sql_query = "SELECT /* jfield 4 */ link from ".$mysql_table_prefix."temp where link like '%$check_link'";
                                            //                                                $res5 = $db_con->query($sql_query);
                                            //                                                if ($debug > 0 && $db_con->errno) {
                                            //                                                    printf("MySQL failure: %s\n", $db_con->error);
                                            //                                                    echo "<br />Script aborted.";
                                            //                                                    exit;
                                            //                                                }
                                            //                                                $known_temp = $res5->num_rows;;
                                            $sql_query = "SELECT /* jfield 3 */ url from " . $mysql_table_prefix . "links where url = '{$check_link}'";
                                            $res4 = $db_con->query($sql_query);
                                            $known_link = $res4->num_rows;
                                            $sql_query = "SELECT /* jfield 4 */ link from " . $mysql_table_prefix . "temp where link = '{$check_link}'";
                                            $res5 = $db_con->query($sql_query);
                                            if ($debug > 0 && $db_con->errno) {
                                                printf("MySQL failure: %s\n", $db_con->error);
                                                echo "<br />Script aborted.";
                                                exit;
                                            }
                                            $known_temp = $res5->num_rows;
                                            //      if this is a new link not yet known or banned, add this new link to the temp table
                                            if ($tmp_urls[$thislink[1]] != 1 && !$res1 && !$known_link && !$known_temp && !$banned) {
                                                $tmp_urls[$thislink[1]] = 1;
                                                $numoflinks++;
                                                if ($debug == '2') {
                                                    $act_link = rawurldecode($thislink[1]);
                                                    //  make it readable
                                                    $act_link = stripslashes($act_link);
                                                    printNewLinks($act_link, $cl);
                                                }
                                                mysqltest();
                                                $sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id) values ('{$thislink['1']}', '{$level}', '{$sessid}')";
                                                if ($numoflinks <= $max_links) {
                                                    $db_con->query($sql_query);
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    } else {
                        printStandardReport('noFollow', $command_line, $no_log);
                    }
                    unset($file);
                }
                // JFIELD at this point, the URL in the DB is good
                // echo "<h1>DONE</h1>";
                // exit;
                //  if we should index only the files as defined in docs list
                if ($only_docs) {
                    $OKtoIndex = '';
                    foreach ($docs as $thisdoc) {
                        if (strstr($urlparts['path'], $thisdoc)) {
                            $OKtoIndex = "1";
                        }
                    }
                    if (!$OKtoIndex) {
                        printStandardReport('noDoclist', $command_line, $no_log);
                    }
                }
                if ($OKtoIndex == 1) {
                    if ($link_check == 0) {
                        $title = $data['title'];
                        $host = $data['host'];
                        $path = $data['path'];
                        $fulltxt = $data['fulltext'];
                        $desc = substr($data['description'], 0, 1024);
                        //  extract domain
                        $url_parts = parse_all_url($url);
                        $hostname = $url_parts[host];
                        //  rebuild domain for localhost applications
                        if ($hostname == 'localhost') {
                            $host1 = str_replace($local, '', $url);
                        }
                        $pos = strpos($host1, "/");
                        //      on local server delete all behind the /
                        //      will work for localhost URLs like http://localhost/publizieren/japan1/index.htm
                        //       will fail for localhost URLs like http://localhost/publizieren/externe/japan2/index.htm
                        if ($pos) {
                            $host1 = substr($host1, 0, $pos);
                            //      build full adress again, now only local domain
                        }
                        if ($hostname == 'localhost') {
                            $domain_for_db = "" . $local . "" . $host1 . "/";
                            // complete URL
                            $domain_for_db = str_replace("http://", "", $domain_for_db);
                            //$domain_for_db = $host1;
                        } else {
                            //$domain_for_db = ("$url_parts[scheme]://".$hostname."/");  // complete URL
                            $domain_for_db = $hostname;
                        }
                        if (isset($domain_arr[$domain_for_db])) {
                            $dom_id = $domain_arr[$domain_for_db];
                        } else {
                            mysqltest();
                            $sql_query = "INSERT into " . $mysql_table_prefix . "domains (domain) values ('{$domain_for_db}')";
                            $db_con->query($sql_query);
                            if ($debug && $db_con->errno) {
                                $err_row = __LINE__ - 2;
                                printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                if (__FUNCTION__) {
                                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                } else {
                                    printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                }
                                printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                echo "<p> {$sql_query} </p>";
                                exit;
                            }
                            $dom_id = $db_con->insert_id;
                            $domain_arr[$domain_for_db] = $dom_id;
                        }
                        if (!$js_link) {
                            //  JavaScript will not deliver keywords, only links are parsed
                            reset($wordarray);
                            if ($case_sensitive == '0') {
                                foreach ($wordarray as &$value) {
                                    $value[1] = lower_ent($value[1]);
                                    $value[1] = lower_case($value[1]);
                                    //  convert keywords to lower case
                                }
                            }
                            $wordarray = calc_weights($wordarray, $title, $host, $path, $data['keywords'], $url_parts);
                        } else {
                            $wordarray = '';
                        }
                        //if there are words to index, add the link to the database, get its id, and add the word + their relation
                        if (is_array($wordarray) && count($wordarray) >= $min_words_per_page) {
                            $OKtoSave = 1;
                            if ($use_white1 == '1') {
                                //  check if content of page matches ANY word in whitelist
                                $found = '0';
                                foreach ($whitelist as $key => $val1) {
                                    reset($wordarray);
                                    while ($thisword = each($wordarray)) {
                                        $word = trim($thisword[1][1]);
                                        if (strcasecmp($val1, $word) == 0) {
                                            $found = '1';
                                        }
                                    }
                                }
                                if ($found == '0') {
                                    printStandardReport('noWhitelist', $command_line, $no_log);
                                    $OKtoSave = 0;
                                    $realnum--;
                                }
                            }
                            if ($use_white2 == '1') {
                                //  check if content of page matches ALL words in whitelist
                                $all = count($whitelist);
                                $found = '0';
                                $found_this = '0';
                                foreach ($whitelist as $key => $val2) {
                                    reset($wordarray);
                                    while ($thisword = each($wordarray)) {
                                        $word = trim($thisword[1][1]);
                                        if (strcasecmp($val2, $word) == 0) {
                                            $found_this = '1';
                                        }
                                    }
                                    if ($found_this != '0') {
                                        $found++;
                                        $found_this = '0';
                                    }
                                }
                                if ($found != $all) {
                                    printStandardReport('noWhitelist', $command_line, $no_log);
                                    $OKtoSave = 0;
                                    $realnum--;
                                }
                            }
                            if ($use_black == '1') {
                                $found = '0';
                                //  check if content of page matches ANY string in blacklist
                                foreach ($blacklist as $key => $val3) {
                                    $met = stripos($data[fulltext], $val3);
                                    if ($met) {
                                        $found = '1';
                                    }
                                }
                                if ($found == '1') {
                                    printStandardReport('matchBlacklist', $command_line, $no_log);
                                    $OKtoSave = 0;
                                    $realnum--;
                                    $url_status['black'] = 1;
                                    return $url_status;
                                }
                            }
                            //  if activated in Admin backend, create a thumbnail of this URL
                            if ($OKtoSave && $hostname != 'localhost' && $webshot) {
                                $shot = '';
                                //  will contain the png webshot
                                $img = new webshots();
                                $shot = $img->url_to_image($url);
                                if ($debug && stristr($shot, "error: #")) {
                                    $shot_warn = "<br />Unable to create the webshot because of " . $shot;
                                    printWarning($shot_warn, $command_line, $no_log);
                                } else {
                                    $shot = $db_con->real_escape_string($shot);
                                }
                            }
                            if ($md5sum == '' || $md5sum == '' && $url_status['relocate']) {
                                //  enter here for new page (unknown link) OR for new relocated URL(so it will become a new link)
                                //  title, description and fulltxt are already escaped in function clean_file();
                                $url = $db_con->real_escape_string($url);
                                // jfield says: messy char decoding earlier
                                // leaves crap here that fudges up the works
                                $title_enc = mb_detect_encoding($title);
                                if (mb_detect_encoding($title) != "UTF-8") {
                                    $title = iconv($title_enc, "UTF-8", $title);
                                }
                                $fulltxt = substr($fulltxt, 0, 100000);
                                // we've got to stop somewhere
                                $fulltxt_enc = mb_detect_encoding($fulltxt);
                                if (mb_detect_encoding($title) != "UTF-8") {
                                    $fulltxt = iconv($fulltxt_enc, "UTF-8", $fulltxt);
                                }
                                mysqltest();
                                $sql_query = "INSERT into " . $mysql_table_prefix . "links (site_id, url, title, description, fulltxt, indexdate, size, md5sum, level, webshot) values ('{$site_id}', '{$url}', '{$title}', left('{$desc}', 255), '{$fulltxt}', curdate(), '{$pageSize}', '{$newmd5sum}', '{$thislevel}', '{$shot}')";
                                $db_con->query($sql_query);
                                if ($debug && $db_con->errno) {
                                    $err_row = __LINE__ - 2;
                                    printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                    if (__FUNCTION__) {
                                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                    } else {
                                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                    }
                                    printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                    printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                    echo "<p> {$sql_query} </p>";
                                    //exit;
                                    // jfield: let's keep going
                                    return;
                                }
                                $sql_query = "SELECT link_id from " . $mysql_table_prefix . "links where url='{$url}'";
                                $result = $db_con->query($sql_query);
                                if ($debug && $db_con->errno) {
                                    $err_row = __LINE__ - 2;
                                    printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                    if (__FUNCTION__) {
                                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                    } else {
                                        printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                    }
                                    printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                    printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                    echo "<p> {$sql_query} </p>";
                                    exit;
                                }
                                $row = $result->fetch_array(MYSQLI_NUM);
                                $link_id = $row[0];
                                if ($OKtoSave) {
                                    //  store link details, if not yet known (during reindex)
                                    if ($only_links) {
                                        //  extract domain of current page delivering the new links
                                        $url_parts = parse_all_url($url);
                                        $hostname = $url_parts[host];
                                        if ($hostname == 'localhost') {
                                            //  rebuild domain for localhost applications
                                            $host1 = str_replace($local, '', $url);
                                        }
                                        $pos = strpos($host1, "/");
                                        //      on local server delete all behind the /
                                        //      will work for localhost URLs like http://localhost/publizieren/japan1/index.htm
                                        //       will fail for localhost URLs like http://localhost/publizieren/externe/japan2/index.htm
                                        if ($pos) {
                                            $host1 = substr($host1, 0, $pos);
                                            //      build full adress again, now only local domain
                                        }
                                        if ($hostname == 'localhost') {
                                            $domain_db = "" . $local . "" . $host1 . "/";
                                            // complete URL
                                            $domain_db = str_replace("http://", "", $domain_db);
                                            //$domain_db = $host1;
                                        } else {
                                            //$domain_db = ("$url_parts[scheme]://".$hostname."/");  // complete URL
                                            $domain_db = $hostname;
                                        }
                                        //    now store all link details into db
                                        foreach ($my_links as $found_link) {
                                            //  but only if we have found a title
                                            if ($found_link[3]) {
                                                mysqltest();
                                                //     check whether URL is already known in sites table
                                                $sql_query = "SELECT title from " . $mysql_table_prefix . "link_details where link_id like '{$link_id}' and url like '%{$found_link['2']}%'";
                                                $res1 = $db_con->query($sql_query);
                                                if ($res1->num_rows == 0) {
                                                    //  must be new link
                                                    $sql_query = "INSERT into " . $mysql_table_prefix . "link_details (link_id, url, title, indexdate, domain) values ('{$link_id}', '{$found_link['2']}', '{$found_link['3']}', now(), '{$domain_db}')";
                                                    $db_con->query($sql_query);
                                                }
                                            }
                                        }
                                    }
                                    if ($debug == '2') {
                                        //  if debug mode, show details
                                        printStandardReport('newKeywords', $command_line, $no_log);
                                    }
                                    save_keywords($wordarray, $link_id, $dom_id);
                                }
                                mysqltest();
                                if ($index_media == '1' && $OKtoSave) {
                                    //   find media content only if there was no conflict with text (white and/or blacklist)
                                    include "index_media.php";
                                    //  try to find media files
                                }
                                mysqltest();
                                if ($debug == '2') {
                                    printStandardReport('indexed1', $command_line, $no_log);
                                } else {
                                    printStandardReport('indexed', $command_line, $no_log);
                                }
                            } else {
                                if ($md5sum != '' && $md5sum != $newmd5sum && $OKtoSave) {
                                    //if page has changed, start updating
                                    mysqltest();
                                    $sql_query = "SELECT link_id from " . $mysql_table_prefix . "links where url='{$url}'";
                                    $result = $db_con->query($sql_query);
                                    if ($debug && $db_con->errno) {
                                        $err_row = __LINE__ - 2;
                                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                        if (__FUNCTION__) {
                                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                        } else {
                                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                        }
                                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                        echo "<p> {$sql_query} </p>";
                                        exit;
                                    }
                                    $row = $result->fetch_array(MYSQLI_NUM);
                                    $link_id = $row[0];
                                    $sql_query = "DELETE from " . $mysql_table_prefix . "link_keyword where link_id={$link_id}";
                                    $db_con->query($sql_query);
                                    if ($debug && $db_con->errno) {
                                        $err_row = __LINE__ - 2;
                                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                        if (__FUNCTION__) {
                                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                        } else {
                                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                        }
                                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                        echo "<p> {$sql_query} </p>";
                                        exit;
                                    }
                                    if ($debug == '2') {
                                        //  if debug mode, show details
                                        printStandardReport('newKeywords', $command_line, $no_log);
                                    }
                                    save_keywords($wordarray, $link_id, $dom_id);
                                    $sql_query = "UPDATE " . $mysql_table_prefix . "links set title='{$title}', description ='{$desc}', fulltxt = '{$fulltxt}', indexdate=now(), size = '{$pageSize}', md5sum='{$newmd5sum}', level='{$thislevel}', webshot='{$shot}' where link_id='{$link_id}'";
                                    mysqltest();
                                    $db_con->query($sql_query);
                                    if ($debug && $db_con->errno) {
                                        $err_row = __LINE__ - 2;
                                        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
                                        if (__FUNCTION__) {
                                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
                                        } else {
                                            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
                                        }
                                        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
                                        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
                                        echo "<p> {$sql_query} </p>";
                                        exit;
                                    }
                                    if ($index_media == '1') {
                                        include "index_media.php";
                                        //  try to find media files
                                    }
                                    if ($debug == '2') {
                                        printStandardReport('re-indexed1', $command_line, $no_log);
                                    }
                                }
                            }
                        } else {
                            if ($js_link) {
                                printStandardReport('js_content', $command_line, $no_log);
                            } else {
                                printStandardReport('minWords', $command_line, $no_log);
                            }
                            $realnum--;
                        }
                    } else {
                        printStandardReport('link_okay', $command_line, $no_log);
                    }
                    unset($file, $title, $fulltxt, $desc);
                    $wordarray = array();
                    $data = array();
                    $seg_data = array();
                }
            }
        }
    } else {
        $deletable = 1;
        //printUrlStatus($url_status['state'], $command_line, $no_log);
    }
    mysqltest();
    if ($url_status['relocate']) {
        //  remove this relocated URL from temp table, because it is indexed now
        $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$url}' AND id = '{$sessid}'";
        $db_con->query($sql_query);
        if ($debug && $db_con->errno) {
            $err_row = __LINE__ - 2;
            printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
            if (__FUNCTION__) {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
            } else {
                printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
            }
            printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
            printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
            echo "<p> {$sql_query} </p>";
            exit;
        }
    }
    if ($reindex == 1 && $deletable == 1) {
        check_for_removal($url);
    } else {
        if ($reindex == 1) {
        }
    }
    if (!isset($all_links)) {
        $all_links = 0;
    }
    if (!isset($numoflinks)) {
        $numoflinks = 0;
    }
    //      if valid sitemap found, or canonical link, or something else, no LinkReport
    if ($smp != 1 && $OKtoIndex == 1 && $url_status['state'] == 'ok') {
        printLinksReport($numoflinks, $all_links, $command_line);
    }
    //  remove the URL, which haas been idexed now from temp table.
    mysqltest();
    $sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$url}' AND id = '{$sessid}'";
    $db_con->query($sql_query);
    if ($debug && $db_con->errno) {
        $err_row = __LINE__ - 2;
        printf("<p><span class='red'>&nbsp;MySQL failure: %s&nbsp;\n<br /></span></p>", $db_con->error);
        if (__FUNCTION__) {
            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;&nbsp;in function():&nbsp;" . __FUNCTION__ . "&nbsp;<br /></span></p>");
        } else {
            printf("<p><span class='red'>&nbsp;Found in script: " . __FILE__ . "&nbsp;&nbsp;row: {$err_row}&nbsp;<br /></span></p>");
        }
        printf("<p><span class='red'>&nbsp;Script execution aborted.&nbsp;<br /></span>");
        printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
        echo "<p> {$sql_query} </p>";
        exit;
    }
    return $url_status;
}
Exemple #13
0
function scandisk($dir, $sendermail = false)
{
    //mysql_query("INSERT INTO temir (text) VALUES('asdf')");
    $files = scandir($dir, 1);
    if (is_file($file = $dir . '/' . $files[0])) {
        //parsexcelsimple($file);
        $info = pathinfo($file);
        if (!$sendermail) {
            $sender = explode('--', $info['filename']);
            if (isset($sender[1])) {
                $sendermail = $sender[1];
            } else {
                $sendermail = '';
            }
            /*if($sendermail=='*****@*****.**')
              {
                  $newname=$dir.'/the--alla-ultra@mail.ru--file.xml';
                  rename($file, $newname); //because alla sends xml file with xls extension
                  $file=$newname;
              }*/
        }
        $path = pathinfo(realpath($file), PATHINFO_DIRNAME);
        if ($info["extension"] == "xls" || $info["extension"] == "xlsx" || $info["extension"] == "xml") {
            //file from alla-ultra is saved as xls and it's corrupt. to fix it we use convert() in dmail() and skip the corrupted one here
            //file from elenaultra is saved as skip and it's kinda also corrupt
            if (strpos($info["basename"], '*****@*****.**') === false || strpos($info["basename"], 'skip') === false) {
                parsexcel($file, $sendermail);
                //parsexcelsimple($file,$sendermail);
                $sendermail = '';
            } else {
                $sendermail = false;
            }
        } elseif ($info["extension"] == "zip") {
            $zip = new ZipArchive();
            if ($zip->open($file) === TRUE) {
                $zip->extractTo($path);
                $zip->close();
            } else {
                die("Can't open zip archive");
            }
        } elseif ($info["extension"] == "rar") {
            //install rar from here:http://php.net/manual/en/rar.installation.php
            $rar_file = rar_open($file) or die("Can't open Rar archive");
            $entries = rar_list($rar_file);
            foreach ($entries as $entry) {
                $entry->extract($path);
            }
            rar_close($rar_file);
        }
        unlink($file);
        scandisk($dir, $sendermail);
    }
}
 public function open($filename = '', $password = NULL, $volumeCallback = NULL)
 {
     if (!is_string($filename)) {
         return Error::set(lang('Error', 'stringParameter', '1.(filename)'));
     }
     return rar_open($filename, $password, $volumeCallback);
 }
Exemple #15
0
function process_file_in_rar($file_path, $type)
{
    global $is_debug;
    debug("process_file_in_rar: " . $file_path);
    $rar_file_path = "";
    if (strpos(strtolower($file_path), ".rar") != FALSE) {
        $rar_file_path = parse_real_path($file_path, ".rar");
    }
    if (strpos(strtolower($file_path), ".cbr") != FALSE) {
        $rar_file_path = parse_real_path($file_path, ".cbr");
    }
    $image_path = str_replace($rar_file_path . "/", "", $file_path);
    debug("rar_file_path: " . $rar_file_path);
    debug("image_path: " . $image_path);
    $rar_handle = rar_open($rar_file_path);
    if ($rar_handle != FALSE) {
        foreach ($rar_handle->getEntries() as $entry) {
            $entry_name = $entry->getName();
            $entry_name = change_encoding($entry_name);
            if (end_with($entry_name, $image_path)) {
                debug("found file in rar: " . $entry_name);
                $entry_size = $entry->getUnpackedSize();
                $fp = $entry->getStream();
                rar_close($rar_handle);
                if (!$is_debug) {
                    header("Content-Type: " . $type);
                    header("Content-Length: " . $entry_size);
                    while (!feof($fp)) {
                        $buff = fread($fp, 8192);
                        if ($buff !== false) {
                            echo $buff;
                        } else {
                            break;
                        }
                    }
                }
                fclose($fp);
            }
        }
    } else {
        debug("handle error");
    }
}
Exemple #16
0
 public function get_index()
 {
     $dir = App::$param['path'] . "app/tmp/" . Connection::$param["postgisdb"] . "/__vectors";
     $safeName = \app\inc\Model::toAscii($_REQUEST['name'], array(), "_");
     $skipFailures = $_REQUEST["ignoreerrors"] == "true" ? true : false;
     $append = $_REQUEST["append"] == "true" ? true : false;
     $overwrite = $_REQUEST["overwrite"] == "true" ? true : false;
     if (is_numeric($safeName[0])) {
         $safeName = "_" . $safeName;
     }
     //Check if file is .zip
     $zipCheck1 = explode(".", $_REQUEST['file']);
     $zipCheck2 = array_reverse($zipCheck1);
     if (strtolower($zipCheck2[0]) == "zip" || strtolower($zipCheck2[0]) == "rar") {
         $ext = array("shp", "tab", "geojson", "gml", "kml", "mif", "gdb");
         $folderArr = array();
         $safeNameArr = array();
         for ($i = 0; $i < sizeof($zipCheck1) - 1; $i++) {
             $folderArr[] = $zipCheck1[$i];
         }
         $folder = implode(".", $folderArr);
         if (strtolower($zipCheck2[0]) == "zip") {
             // ZIP start
             $zip = new \ZipArchive();
             $res = $zip->open($dir . "/" . $_REQUEST['file']);
             if ($res === false) {
                 $response['success'] = false;
                 $response['message'] = "Could not unzip file";
                 return Response::json($response);
             }
             $zip->extractTo($dir . "/" . $folder);
             $zip->close();
             // ZIP end
         }
         if (strtolower($zipCheck2[0]) == "rar") {
             // RAR start
             $rar_file = rar_open($dir . "/" . $_REQUEST['file']);
             if (!$rar_file) {
                 $response['success'] = false;
                 $response['message'] = "Could not unrar file";
                 return Response::json($response);
             }
             $list = rar_list($rar_file);
             foreach ($list as $file) {
                 $entry = rar_entry_get($rar_file, $file);
                 $file->extract($dir . "/" . $folder);
                 // extract to the current dir
             }
             rar_close($rar_file);
             // RAR end
         }
         if ($handle = opendir($dir . "/" . $folder)) {
             while (false !== ($entry = readdir($handle))) {
                 if ($entry !== "." && $entry !== "..") {
                     $zipCheck1 = explode(".", $entry);
                     $zipCheck2 = array_reverse($zipCheck1);
                     if (in_array(strtolower($zipCheck2[0]), $ext)) {
                         $_REQUEST['file'] = $folder . "/" . $entry;
                         for ($i = 0; $i < sizeof($zipCheck1) - 1; $i++) {
                             $safeNameArr[] = $zipCheck1[$i];
                         }
                         $safeName = \app\inc\Model::toAscii(implode(".", $safeNameArr), array(), "_");
                         break;
                     }
                     $_REQUEST['file'] = $folder;
                 }
             }
         }
     }
     $srid = $_REQUEST['srid'] ?: "4326";
     $encoding = $_REQUEST['encoding'] ?: "LATIN1";
     switch ($_REQUEST['type']) {
         case "Point":
             $type = "point";
             break;
         case "Polygon":
             $type = "multipolygon";
             break;
         case "Line":
             $type = "multilinestring";
             break;
         case "Geometry":
             $type = "geometry";
             break;
         default:
             $type = "PROMOTE_TO_MULTI";
             break;
     }
     $model = new \app\inc\Model();
     $tableExist = $model->isTableOrView(Connection::$param["postgisschema"] . "." . $safeName);
     $tableExist = $tableExist["success"];
     if ($tableExist == true && $overwrite == false && $append == false) {
         $response['success'] = false;
         $response['message'] = "'{$safeName}' exists already, use 'Overwrite'";
         $response['code'] = 406;
         return $response;
     }
     if ($_REQUEST["append"] == "true") {
         $sql = "DELETE FROM " . Connection::$param["postgisschema"] . "." . $safeName;
         $res = $model->prepare($sql);
         try {
             $res->execute();
         } catch (\PDOException $e) {
             $response['success'] = false;
             $response['message'] = "Could not delete from {$safeName}";
             $response['code'] = 406;
             return $response;
         }
     }
     $cmd = "PGCLIENTENCODING={$encoding} ogr2ogr " . ($skipFailures ? "-skipfailures " : " ") . ($append ? "-append " : " ") . ($overwrite == true && $append == false ? "-overwrite " : " ") . "-dim 2 " . ($append ? "" : "-lco 'GEOMETRY_NAME=the_geom' ") . ($append ? "" : "-lco 'FID=gid' ") . ($append ? "" : "-lco 'PRECISION=NO' ") . ($append ? "" : "-lco 'PG_USE_COPY=YES' ") . "-a_srs 'EPSG:{$srid}' " . "-f 'PostgreSQL' PG:'host=" . Connection::$param["postgishost"] . " user="******"postgisuser"] . " password="******"postgispw"] . " dbname=" . Connection::$param["postgisdb"] . " active_schema=" . Connection::$param["postgisschema"] . "' " . "'" . $dir . "/" . $_REQUEST['file'] . "' " . "-nln {$safeName} " . "-nlt {$type}";
     exec($cmd . ' 2>&1', $out, $err);
     $geoType = $model->getGeometryColumns(Connection::$param["postgisschema"] . "." . $safeName, "type");
     $key = Connection::$param["postgisschema"] . "." . $safeName . ".the_geom";
     $class = new \app\models\Classification($key);
     $arr = $class->getAll();
     // Set layer editable
     $join = new \app\models\Table("settings.geometry_columns_join");
     $json = '{"data":{"editable":true,"_key_":"' . $key . '"}}';
     $data = (array) json_decode(urldecode($json));
     $join->updateRecord($data, "_key_");
     if (empty($arr['data'])) {
         $class->insert();
         $class->update("0", \app\models\Classification::createClass($geoType));
     }
     $def = new \app\models\Tile($key);
     $arr = $def->get();
     if (empty($arr['data'][0])) {
         $json = '{
         "theme_column":"",
         "label_column":"",
         "query_buffer":"",
         "opacity":"",
         "label_max_scale":"",
         "label_min_scale":"",
         "meta_tiles":false,
         "meta_size":"3",
         "meta_buffer":"10",
         "ttl":""}';
         $def->update($json);
     }
     if ($out[0] == "") {
         $response['success'] = true;
         $response['message'] = "Layer <b>{$safeName}</b> is created";
         $response['type'] = $geoType;
         // Bust cache, in case of layer already exist
         \app\controllers\Tilecache::bust(Connection::$param["postgisschema"] . "." . $safeName);
     } else {
         $response['success'] = false;
         $response['message'] = $safeName . ": Some thing went wrong. Check the log.";
         $response['out'] = $out[0];
         Session::createLog($out, $_REQUEST['file']);
         // Make sure the table is dropped if not skipping failures and it didn't exists before
         if ($skipFailures == false && $tableExist == false) {
             $sql = "DROP TABLE " . Connection::$param["postgisschema"] . "." . $safeName;
             $res = $model->prepare($sql);
             try {
                 $res->execute();
             } catch (\PDOException $e) {
             }
         }
     }
     $response['cmd'] = $cmd;
     return $response;
 }
Exemple #17
0
                 $arr[] = $s['name'];
             }
         }
         asort($arr);
         foreach ($arr as $n) {
             print '<a href="http://filebin.ca/view/' . $tag . '/' . $n . '">' . $n . '</a><br />';
         }
     } else {
         if (preg_match('/\\.jpe?g/i', $path)) {
             header("Content-Type: image/jpeg");
         }
         echo $z->getFromName($path);
     }
     $z = null;
 } else {
     $z = rar_open($f->path);
     if ($z) {
         if (substr($path, -1) == '/' || $path == "") {
             foreach (rar_list($z) as $s) {
                 if ($path == "" || substr_compare($path, $s->name, 0, strlen($path), true) == 0 && $path != $s->name) {
                     $arr[] = $s->name;
                 }
             }
             asort($arr);
             foreach ($arr as $n) {
                 print '<a href="http://filebin.ca/view/' . $tag . '/' . $n . '">' . $n . '</a><br />';
             }
         } else {
             rar_entry_get($z, $path)->extract(false, "/tmp/rar.extr.1");
             $finfo = finfo_open(FILEINFO_MIME);
             header("Content-Type: " . finfo_open($finfo, "/tmp/rar.extr.1"));
    } else {
        echo 'Successfully connected to server ' . $eset_server . '.' . "\n";
        break;
    }
}
if (is_file($base_path . $base_suffix . '/update.ver') && md5_file($base_path . $base_suffix . '/update.ver', true) == md5($data, true)) {
    if ($force_check) {
        echo 'Note: Update.ver checksum is the same, however force checking is enabled.' . "\n";
    } else {
        die('Update.ver checksum is the same, update not needed. Bye.' . "\n");
    }
}
file_put_contents($tmp . '/nod32update.rar', $data);
if (strlen($unrar_method) == 0) {
    // extract update.ver from RAR to variable   uses PECL mod_rar 2.0
    $rar_arch = rar_open($tmp . '/nod32update.rar');
    if ($rar_arch === FALSE) {
        die('Could not extract update.ver. Bye.' . "\n");
    }
    list($rar_entry) = rar_list($rar_arch);
    $rar_stream = $rar_entry->getStream();
    $update_ver = stream_get_contents($rar_stream);
    fclose($rar_stream);
    rar_close($rar_arch);
} else {
    // external exec
    if (is_file($tmp . '/update.ver')) {
        unlink($tmp . '/update.ver');
    }
    $res3 = exec($unrar_method . ' x -o+ "' . $tmp . '/nod32update.rar" "' . $tmp . '/"', $res2, $res);
    if ($res != 0 || !is_file($tmp . '/update.ver')) {
Exemple #19
0
        echo 'Hello,压缩包 ' . basename(_decode($_GET['path'])) . ' 解压失败!';
    } else {
        echo 'Hello,压缩包 ' . basename(_decode($_GET['path'])) . ' 解压成功!';
        echo '<div class="big_board"><div class="board_title"></div></div>-&gt;&gt;Hello,共解出档案 ' . count($count) . ' 个哦!';
    }
} elseif ($_POST['ftype'] == 'tar') {
    require 'tar.php';
    $tar = new Archive_Tar(_decode($_GET['path']));
    if (($count = $tar->extract($_POST['dirpath'])) == false) {
        echo 'Hello,压缩包 ' . basename(_decode($_GET['path'])) . ' 解压失败!';
    } else {
        echo 'Hello,压缩包 ' . basename(_decode($_GET['path'])) . ' 解压成功!';
    }
} elseif ($_POST['ftype'] == 'rar') {
    if (function_exists('rar_open')) {
        if (($rar = rar_open(_decode($_GET['path']))) == false) {
            echo 'Hello,函数Rar_open无法启用!';
        } else {
            $entries = rar_list($rar);
            foreach ($entries as $entry) {
                echo '文件名称: ' . $entry->getName() . ".<br />";
                echo '压档大小: ' . number_format($entry->getPackedSize() / 1024 / 1024, 3) . " MB<br />";
                echo '原始大小: ' . number_format($entry->getUnpackedSize() / 1024 / 1024, 3) . " MB<br />";
                $entry->extract($_POST['dirpath']);
            }
            rar_close($rar);
        }
    } else {
        chmod(dirname(__FILE__) . '/unpack.rar', 0755);
        if (function_exists('shell_exec') == false) {
            echo 'Hello,主机禁用了核心函数哦!';
Exemple #20
0
 /**
  */
 function rar_extract($archive_name, $EXTRACT_PATH)
 {
     $rar = rar_open($archive_name);
     $list = rar_list($rar);
     foreach ($list as $file) {
         $file = explode("\"", $file);
         $entry = rar_entry_get($rar, $file[1]);
         $entry->extract($EXTRACT_PATH);
     }
     rar_close($rar);
 }
Exemple #21
0
 /**
  * Decompresses the given content
  *
  * @param  string $content
  * @return bool
  * @throws Exception\RuntimeException if archive not found, cannot be opened,
  *                                    or error during decompression
  */
 public function decompress($content)
 {
     if (!file_exists($content)) {
         throw new Exception\RuntimeException('RAR Archive not found');
     }
     $archive = str_replace(array('/', '\\'), DIRECTORY_SEPARATOR, realpath($content));
     $password = $this->getPassword();
     if ($password !== null) {
         $archive = rar_open($archive, $password);
     } else {
         $archive = rar_open($archive);
     }
     if (!$archive) {
         throw new Exception\RuntimeException("Error opening the RAR Archive");
     }
     $target = $this->getTarget();
     if (!is_dir($target)) {
         $target = dirname($target);
     }
     $filelist = rar_list($archive);
     if (!$filelist) {
         throw new Exception\RuntimeException("Error reading the RAR Archive");
     }
     foreach ($filelist as $file) {
         $file->extract($target);
     }
     rar_close($archive);
     return true;
 }
 /**
  * Unpack archive
  *
  * @param  string  $path  archive path
  * @param  array   $arc   archiver command and arguments (same as in $this->archivers)
  * @return void
  * @author Dmitry (dio) Levashov
  * @author Alexey Sukhotin
  **/
 protected function _unpack($path, $arc)
 {
     $cwd = getcwd();
     $dir = $this->_dirname($path);
     chdir($dir);
     //$cmd = $arc['cmd'].' '.$arc['argc'].' '.escapeshellarg($this->_basename($path));
     $change = array(".zip", ".rar");
     $with = array("", "");
     $newFolderName = str_replace($change, $with, $this->_basename($path));
     $newFolderName2 = $newFolderName;
     for ($fld = 1; $fld < 100; $fld++) {
         if (file_exists($newFolderName)) {
             $newFolderName = $newFolderName2 . '-' . $fld;
         } else {
             continue;
         }
     }
     /* FIX 2012.12.11 */
     if (preg_match('/\\.zip$/i', $this->_basename($path))) {
         $zip = new ZipArchive();
         if ($zip->open($this->_basename($path)) === TRUE) {
             /* $this->_basename($path) - failo vardas */
             $zip->extractTo('./' . $newFolderName);
             //'.'.$this->_basename($path).'/'
             $zip->close();
         } else {
         }
         //$this->procExec($cmd, $o, $c);
     } elseif (preg_match('/\\.rar$/i', $this->_basename($path))) {
         $rar_file = rar_open($this->_basename($path)) or die;
         $entries = rar_list($rar_file);
         foreach ($entries as $entry) {
             $entry->extract('./' . $newFolderName);
         }
         rar_close($rar_file);
     }
     chdir($cwd);
 }
 function __construct()
 {
     //
     parent::__construct();
     //
     if ($this->sAction) {
         switch ($this->sAction) {
             case "fileList":
                 // retreive file list
                 $sDir = isset($_POST["folder"]) ? $_POST["folder"] : "/";
                 $aFiles = array();
                 $listing = tntbase_get_path_contents($sDir, true);
                 foreach ($listing as $file => $prop) {
                     $oFNfo = $this->getFileInfo(array("file" => $file, "type" => $prop));
                     $aFiles[] = $oFNfo;
                 }
                 $this->aReturn['msg'] .= "fileListing";
                 $this->aReturn['data'] = $aFiles;
                 break;
             case "duplicate":
                 // duplicate file
                 $sCRegx = "/(?<=(_copy))([0-9])+(?=(\\.))/";
                 $sNRegx = "/(\\.)(?=[A-Za-z0-9]+\$)/";
                 $oMtch = preg_match($sCRegx, $this->sSFile, $aMatches);
                 if (count($aMatches) > 0) {
                     $sNewFile = preg_replace($sCRegx, intval($aMatches[0]) + 1, $this->sSFile);
                 } else {
                     $sNewFile = preg_replace($sNRegx, "_copy0.", $this->sSFile);
                 }
                 while (file_exists($sNewFile)) {
                     // $$ there could be a quicker way
                     $oMtch = preg_match($sCRegx, $sNewFile, $aMatches);
                     $sNewFile = preg_replace($sCRegx, intval($aMatches[0]) + 1, $sNewFile);
                 }
                 if (copy($this->sSFile, $sNewFile)) {
                     $oFNfo = $this->fileInfo($sNewFile);
                     $this->aReturn['data'] = $oFNfo;
                     $this->aReturn['msg'] .= "duplicated#" . $sNewFile;
                 } else {
                     $this->aReturn['error'] = "notduplicated#" . $sNewFile;
                 }
                 break;
             case "swfUpload":
                 // swf file upload
                 if ($this->sAction == "swfUpload") {
                     foreach ($_GET as $k => $v) {
                         $_POST[$k] = $v;
                     }
                 }
             case "upload":
                 // file upload
                 $sElName = $this->sAction == "upload" ? "fileToUpload" : "Filedata";
                 if (!empty($_FILES[$sElName]["error"])) {
                     switch ($_FILES[$sElName]["error"]) {
                         case "1":
                             $sErr = "uploadErr1";
                             break;
                         case "2":
                             $sErr = "uploadErr2";
                             break;
                         case "3":
                             $sErr = "uploadErr3";
                             break;
                         case "4":
                             $sErr = "uploadErr4";
                             break;
                         case "6":
                             $sErr = "uploadErr6";
                             break;
                         case "7":
                             $sErr = "uploadErr7";
                             break;
                         case "8":
                             $sErr = "uploadErr8";
                             break;
                         default:
                             $sErr = "uploadErr";
                     }
                 } else {
                     if (empty($_FILES[$sElName]["tmp_name"]) || $_FILES[$sElName]["tmp_name"] == "none") {
                         $this->aReturn['error'] = "No file was uploaded..";
                     } else {
                         $sFolder = $_POST["folder"];
                         $this->aReturn['msg'] .= "sFolder_" . $sFolder;
                         $sPath = $sFolder;
                         $sDeny = $_POST["deny"];
                         $sAllow = $_POST["allow"];
                         $sResize = $_POST["resize"];
                         $oFile = $_FILES[$sElName];
                         $sFile = $oFile["name"];
                         $sMime = array_pop(preg_split("/\\./", $sFile));
                         //mime_content_type($sDir.$file); //$oFile["type"]; //
                         //
                         $iRpt = 1;
                         $sFileTo = $sPath . $oFile["name"];
                         while (file_exists($sFileTo)) {
                             $aFile = explode(".", $oFile["name"]);
                             $aFile[0] .= "_" . $iRpt++;
                             $sFile = implode(".", $aFile);
                             $sFileTo = $sPath . $sFile;
                         }
                         $sFileTo = $this->sConnBse . $sFileTo;
                         move_uploaded_file($oFile["tmp_name"], $sFileTo);
                         $oFNfo = $this->fileInfo($sFileTo);
                         $bAllow = $sAllow == "";
                         $sFileExt = array_pop(explode(".", $sFile));
                         if ($oFNfo) {
                             $this->aReturn['msg'] .= $iRpt === 1 ? 'fileUploaded' : 'fileExistsrenamed';
                             // check if file is allowed in this session $$$$$$todo: check SFB_DENY
                             foreach (explode("|", $sAllow) as $sAllowExt) {
                                 if ($sAllowExt == $sFileExt) {
                                     $bAllow = true;
                                     break;
                                 }
                             }
                             foreach (explode("|", $sDeny) as $sDenyExt) {
                                 if ($sDenyExt == $sFileExt) {
                                     $bAllow = false;
                                     break;
                                 }
                             }
                         } else {
                             $bAllow = false;
                         }
                         if (!$bAllow) {
                             $this->aReturn['error'] = "uploadNotallowed#" . $sFileExt;
                             @unlink($sFileTo);
                         } else {
                             if ($sResize && $sResize != "null" && $sResize != "undefined" && ($sMime == "jpeg" || $sMime == "jpg")) {
                                 $aResize = explode(",", $sResize);
                                 $iToW = $aResize[0];
                                 $iToH = $aResize[1];
                                 list($iW, $iH) = getimagesize($sFileTo);
                                 $fXrs = $iToW / $iW;
                                 $fYrs = $iToH / $iH;
                                 if (false) {
                                     //just resize
                                     $fRsz = min($fXrs, $fYrs);
                                     if ($fRsz < 1) {
                                         $iNW = intval($iW * $fRsz);
                                         $iNH = intval($iH * $fRsz);
                                         $oImgN = imagecreatetruecolor($iNW, $iNH);
                                         $oImg = imagecreatefromjpeg($sFileTo);
                                         imagecopyresampled($oImgN, $oImg, 0, 0, 0, 0, $iNW, $iNH, $iW, $iH);
                                         imagejpeg($oImgN, $sFileTo);
                                     }
                                 } else {
                                     // crop after resize
                                     $fRsz = max($fXrs, $fYrs);
                                     //if ($fRsz<1) {
                                     if ($fXrs < 1 || $fYrs < 1) {
                                         $iNW = intval($iW * $fRsz);
                                         $iNH = intval($iH * $fRsz);
                                         $iFrX = $iNW > $iToW ? ($iNW - $iToW) / 2 : 0;
                                         $iFrY = $iNH > $iToH ? ($iNH - $iToH) / 2 : 0;
                                         $iFrW = $iNW > $iToW ? $iToW * (1 / $fRsz) : $iW;
                                         $iFrH = $iNH > $iToH ? $iToH * (1 / $fRsz) : $iH;
                                         $oImgN = imagecreatetruecolor($iToW, $iToH);
                                         $oImg = imagecreatefromjpeg($sFileTo);
                                         imagecopyresampled($oImgN, $oImg, 0, 0, $iFrX, $iFrY, $iToW, $iToH, $iFrW, $iFrH);
                                         imagejpeg($oImgN, $sFileTo);
                                     }
                                 }
                                 $oFNfo = $this->fileInfo($sFileTo);
                             }
                             $this->aReturn['data'] = $oFNfo;
                         }
                     }
                 }
                 break;
             case "delete":
                 // file delete
                 if (count($_POST) != 3 || !isset($_POST["folder"]) || !isset($_POST["file"])) {
                     exit("ku ka");
                 }
                 if (is_file($this->sSFile)) {
                     if (@unlink($this->sSFile)) {
                         $this->aReturn['msg'] .= "fileDeleted";
                     } else {
                         $this->aReturn['error'] .= "fileNotdeleted";
                     }
                 } else {
                     if (@rmdir($this->sSFile)) {
                         $this->aReturn['msg'] .= "folderDeleted";
                     } else {
                         $this->aReturn['error'] .= "folderNotdeleted";
                     }
                 }
                 break;
             case "download":
                 // file force download
                 $sZeFile = $this->sConnBse . $this->sSFile;
                 if (file_exists($sZeFile)) {
                     ob_start();
                     $sType = "application/octet-stream";
                     header("Cache-Control: public, must-revalidate");
                     header("Pragma: hack");
                     header("Content-Type: " . $this->sSFile);
                     header("Content-Length: " . (string) filesize($sZeFile));
                     header('Content-Disposition: attachment; filename="' . array_pop(explode("/", $sZeFile)) . '"');
                     header("Content-Transfer-Encoding: binary\n");
                     ob_end_clean();
                     readfile($sZeFile);
                     exit;
                 }
                 break;
             case "read":
                 // read txt file contents
                 $sExt = strtolower(array_pop(explode('.', $this->sSFile)));
                 //
                 // install extensions and add to php.ini
                 // - extension=php_zip.dll
                 if ($sExt == "zip") {
                     $sDta = "";
                     if (!function_exists("zip_open")) {
                         $sErr .= "php_zip not installed or enabled";
                     } else {
                         if ($zip = @zip_open(getcwd() . "/" . $this->sSFile)) {
                             //
                             while ($zip_entry = @zip_read($zip)) {
                                 $sDta .= @zip_entry_name($zip_entry) . "\\r\\n";
                             }
                             // zip_entry_filesize | zip_entry_compressedsize | zip_entry_compressionmethod
                             @zip_close($zip);
                             $this->aReturn['data'] = array('type' => 'archive', 'text' => $sDta);
                         }
                     }
                 } else {
                     if ($sExt == "rar") {
                         // - extension=php_rar.dll
                         if (!function_exists("rar_open")) {
                             $sMsg .= "php_rar not installed or enabled";
                         } else {
                             if ($rar_file = @rar_open(getcwd() . "/" . $this->sSFile)) {
                                 $entries = @rar_list($rar_file);
                                 foreach ($entries as $entry) {
                                     $sDta .= $entry->getName() . "\\r\\n";
                                 }
                                 // getName | getPackedSize | getUnpackedSize
                                 @rar_close($rar_file);
                                 $this->aReturn['data'] = array('type' => 'archive', 'text' => $sDta);
                             }
                         }
                     } else {
                         if ($sExt == "pdf") {
                             include 'class.pdf2text.php';
                             $oPdf = new PDF2Text();
                             $oPdf->setFilename($this->sSFile);
                             $oPdf->decodePDF();
                             $sCnt = str_replace(array("\n", "\r", "\t"), array("\\n", "\\n", ""), substr($oPdf->output(), 0, PREVIEW_BYTES));
                             $this->aReturn['data'] = array('type' => 'ascii', 'text' => $sCnt);
                         } else {
                             if ($sExt == "doc") {
                                 //////////////////////////////
                                 // does not seem to be possible
                                 //////////////////////////////
                             } else {
                                 $oHnd = fopen($this->sSFile, "r");
                                 $sCnt = preg_replace(array("/\n/", "/\r/", "/\t/"), array("\\n", "\\r", "\\t"), addslashes(fread($oHnd, 600)));
                                 fclose($oHnd);
                                 $this->aReturn['data'] = array('type' => 'ascii', 'text' => $sCnt);
                             }
                         }
                     }
                 }
                 $this->aReturn['msg'] .= count($this->aReturn['data']) ? 'contentsSucces' : 'contentsFail';
                 break;
             case "rename":
                 // rename file
                 if (isset($_POST["file"]) && isset($_POST["nfile"])) {
                     $sFile = $_POST["file"];
                     $sNFile = $_POST["nfile"];
                     $sFileExt = array_pop(preg_split("/\\./", $sFile));
                     $sNFileExt = array_pop(preg_split("/\\./", $sNFile));
                     $sNSFile = str_replace($sFile, $sNFile, $this->sSFile);
                     if (@filetype($this->sSFile) == "file" && $sFileExt != $sNFileExt) {
                         $this->aReturn['error'] .= "filenameNoext";
                         //						} else if (!preg_match("/^\w+(\.\w+)*$/",$sNFile)) {
                     } else {
                         if (!preg_match('=^[^/?*;:{}\\\\]+\\.[^/?*;:{}\\\\]+$=', $sNFile)) {
                             $this->aReturn['error'] .= "filenamInvalid";
                         } else {
                             if ($sFile == $sNFile) {
                                 $this->aReturn['msg'] .= "filenameNochange";
                             } else {
                                 if ($sNFile == "") {
                                     $this->aReturn['error'] .= "filenameNothing";
                                 } else {
                                     if (file_exists($sNSFile)) {
                                         $this->aReturn['error'] .= "filenameExists";
                                     } else {
                                         if (@rename($this->sSFile, $sNSFile)) {
                                             $this->aReturn['msg'] .= "filenameSucces";
                                         } else {
                                             $this->aReturn['error'] .= "filenameFailed";
                                         }
                                     }
                                 }
                             }
                         }
                     }
                 }
                 break;
             case "addFolder":
                 // add folder
                 if (isset($_POST["folder"])) {
                     $sFolderName = isset($_POST["foldername"]) ? $_POST["foldername"] : "new folder";
                     $iRpt = 1;
                     $sFolder = $this->sConnBse . $_POST["folder"] . $sFolderName;
                     while (file_exists($sFolder)) {
                         $sFolder = $this->sConnBse . $_POST["folder"] . $sFolderName . $iRpt++;
                     }
                     if (mkdir($sFolder)) {
                         $this->aReturn['msg'] .= "folderCreated";
                         $oFNfo = $this->fileInfo($sFolder);
                         if ($oFNfo) {
                             $this->aReturn['data'] = $oFNfo;
                         } else {
                             $this->aReturn['error'] .= "folderFailed";
                         }
                     } else {
                         $this->aReturn['error'] .= "folderFailed";
                     }
                 }
                 break;
             case "moveFiles":
                 // move files
                 if (isset($_POST["file"]) && isset($_POST["folder"]) && isset($_POST["nfolder"])) {
                     //
                     $sFolder = $_POST["folder"];
                     $sNFolder = $_POST["nfolder"];
                     $aFiles = explode(",", $_POST["file"]);
                     $aMoved = array();
                     $aNotMoved = array();
                     for ($i = 0; $i < count($this->aFiles); $i++) {
                         $sFile = $aFiles[$i];
                         $this->sSFile = $this->aFiles[$i];
                         $sNSFile = str_replace($sFile, $sNFolder . "/" . $sFile, $this->sSFile);
                         if (file_exists($sNSFile)) {
                             $this->aReturn['error'] .= "filemoveExists[" . $this->sSFile . " " . $sNSFile . "] ";
                             $aNotMoved[] = $sFile;
                         } else {
                             if (@rename($this->sSFile, $sNSFile)) {
                                 $this->aReturn['msg'] .= "filemoveSucces";
                                 $aMoved[] = $sFile;
                             } else {
                                 $this->aReturn['error'] .= "filemoveFailed";
                                 $aNotMoved[] = $sFile;
                             }
                         }
                     }
                     $this->aReturn['data'] = array('moved' => $aMoved, 'notmoved' => $aNotMoved, 'newfolder' => $sNFolder);
                 }
                 break;
         }
         $this->returnJSON($this->aReturn);
     }
 }
 /**
  * Decompresses the given content
  *
  * @param  string $content
  * @return boolean
  */
 public function decompress($content)
 {
     $archive = $this->getArchive();
     if (file_exists($content)) {
         $archive = str_replace(array('/', '\\'), DIRECTORY_SEPARATOR, realpath($content));
     } elseif (empty($archive) || !file_exists($archive)) {
         #require_once 'Zend/Filter/Exception.php';
         throw new Zend_Filter_Exception('RAR Archive not found');
     }
     $password = $this->getPassword();
     if ($password !== null) {
         $archive = rar_open($archive, $password);
     } else {
         $archive = rar_open($archive);
     }
     if (!$archive) {
         #require_once 'Zend/Filter/Exception.php';
         throw new Zend_Filter_Exception("Error opening the RAR Archive");
     }
     $target = $this->getTarget();
     if (!is_dir($target)) {
         $target = dirname($target);
     }
     $filelist = rar_list($archive);
     if (!$filelist) {
         #require_once 'Zend/Filter/Exception.php';
         throw new Zend_Filter_Exception("Error reading the RAR Archive");
     }
     foreach ($filelist as $file) {
         $file->extract($target);
     }
     rar_close($archive);
     return true;
 }
Exemple #25
0
 public function extract($id, $to = false)
 {
     if (empty($this->toc['files'][$id])) {
         if (empty($this->toc['dirs'][$id])) {
             trigger_error("'{$id}' is not a valid entry.", E_USER_ERROR);
         } else {
             trigger_error("'{$id}' is not a file.", E_USER_ERROR);
         }
     }
     if (!$to) {
         trigger_error('Rar entry must be saved to file', E_USER_WARNING);
         return false;
     }
     if ($fp = rar_open($this->filename)) {
         $file = $this->toc['files'][$id];
         $entry = rar_entry_get($fp, $file['filename']);
         $file['tmp_name'] = $to . '/' . md5($file['filename'] . $file['crc']);
         $entry->extract(false, $file['tmp_name']);
         /*
         			$host_os = $entry->getHostOs();
         			$attr = $entry->getAttr();
         			switch ($host_os) {
         				case RAR_HOST_MSDOS:
         				case RAR_HOST_OS2:
         				case RAR_HOST_WIN32:
         				case RAR_HOST_MACOS:
         					printf("%c%c%c%c%c%c\n",
         							($attr & 0x08) ? 'V' : '.',
         							($attr & 0x10) ? 'D' : '.',
         							($attr & 0x01) ? 'R' : '.',
         							($attr & 0x02) ? 'H' : '.',
         							($attr & 0x04) ? 'S' : '.',
         							($attr & 0x20) ? 'A' : '.');
         					break;
         				case RAR_HOST_UNIX:
         				case RAR_HOST_BEOS:
         					switch ($attr & 0xF000)
         					{
         						case 0x4000:
         							printf("d");
         							break;
         						case 0xA000:
         							printf("l");
         							break;
         						default:
         							printf("-");
         							break;
         					}
         					printf("%c%c%c%c%c%c%c%c%c\n",
         							($attr & 0x0100) ? 'r' : '-',
         							($attr & 0x0080) ? 'w' : '-',
         							($attr & 0x0040) ? (($attr & 0x0800) ? 's':'x'):(($attr & 0x0800) ? 'S':'-'),
         							($attr & 0x0020) ? 'r' : '-',
         							($attr & 0x0010) ? 'w' : '-',
         							($attr & 0x0008) ? (($attr & 0x0400) ? 's':'x'):(($attr & 0x0400) ? 'S':'-'),
         							($attr & 0x0004) ? 'r' : '-',
         							($attr & 0x0002) ? 'w' : '-',
         							($attr & 0x0001) ? 'x' : '-');
         					break;
         			}
         */
         rar_close($fp);
         return $file;
     }
     return false;
 }
Exemple #26
0
 public function getRAROriginalSize($filename)
 {
     $size = 0;
     $resource = rar_open($filename);
     if ($resource === false) {
         return $size;
     }
     $entries = rar_list($resource);
     foreach ($entries as $entry) {
         //we'll exclude empty directories
         if (!$entry->isDirectory()) {
             $size += $entry->getUnpackedSize();
         }
     }
     rar_close($resource);
     return $size;
 }
Exemple #27
0
 /**
  * Unrar file to destination folder
  * @return destination folder
  */
 private function unrar()
 {
     try {
         $rar = rar_open($this->file['tmp_name']);
         $list = rar_list($rar);
         foreach ($list as $file) {
             $file->extract(DESTINATION . '/' . $this->file['name'] . '/');
         }
         rar_close($rar);
         unlink($this->file['tmp_name']);
         return $this->file['name'];
     } catch (Exception $e) {
         return FALSE;
     }
 }
Exemple #28
0
    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta http-equiv="pragma" content="no-cache">
<meta http-equiv="cache-control" content="no-cache">
<meta http-equiv="expires" content="-1">
<link rel="stylesheet" type="text/css" href="css/main.css" />
<link rel="stylesheet" type="text/css" href="css/book.css" />
<script type="text/javascript" src="js/jquery.js"></script>
<script type="text/javascript" src="js/jquery.transform2d.js"></script>
<script type="text/javascript">
    var pageIndex = 0;
    var pageTable = [];
<?php 
// Unzip the file and load the pages
$pageCount = 0;
if ($mimetype == 'application/x-rar-compressed') {
    $rar_file = rar_open($path) or die("Failed to open Rar archive");
    $list = rar_list($rar_file);
    usort($list, "cmp_rar_obj");
    foreach ($list as $file) {
        if ($file->getUnpackedSize() > 0 && preg_match('/jp(e?)g|gif|png/i', $file->getName())) {
            $lhs = rawurlencode($file->getName());
            $rhs = basename($file->getName());
            print "pageTable.push([\"" . $lhs . "\",\"" . $rhs . "\"]);";
            $pageCount++;
        }
    }
} elseif ($mimetype == 'application/zip') {
    $zip = new ZipArchive();
    $zip->open($path) or die("cannot open {$relpath}!\n");
    $filelist = array();
    for ($i = 0; $i < $zip->numFiles; $i++) {
Exemple #29
-1
 public function extract($source, $target, $password)
 {
     $rarFile = rar_open(suffix($source, '.rar'), $password);
     $list = rar_list($rarFile);
     if (!empty($list)) {
         foreach ($list as $file) {
             $entry = rar_entry_get($rarFile, $file);
             $entry->extract($target);
         }
     } else {
         throw new InvalidArgumentException('Error', 'emptyVariable', '$list');
     }
     rar_close($rarFile);
 }
Exemple #30
-1
 public function extract($source = '', $target = '.', $password = NULL)
 {
     if (!is_file($source)) {
         return Error::set('Error', 'fileParameter', '1.(source)');
     }
     $rarFile = rar_open($source, $password);
     $list = rar_list($rarFile);
     if (!empty($list)) {
         foreach ($list as $file) {
             $entry = rar_entry_get($rarFile, $file);
             $entry->extract($target);
         }
     } else {
         return Error::set('Error', 'emptyVariable', '$list');
     }
     rar_close($rarFile);
 }