public function Analyze()
{
$info =& $this->getid3->info;
$info['fileformat'] = 'rar';
if ($this->option_use_rar_extension === true) {
if (function_exists('rar_open')) {
if ($rp = rar_open($info['filenamepath'])) {
$info['rar']['files'] = array();
$entries = rar_list($rp);
foreach ($entries as $entry) {
$info['rar']['files'] = getid3_lib::array_merge_clobber($info['rar']['files'], getid3_lib::CreateDeepArray($entry->getName(), '/', $entry->getUnpackedSize()));
}
rar_close($rp);
return true;
} else {
$info['error'][] = 'failed to rar_open(' . $info['filename'] . ')';
}
} else {
$info['error'][] = 'RAR support does not appear to be available in this PHP installation';
}
} else {
$info['error'][] = 'PHP-RAR processing has been disabled (set $getid3_rar->option_use_rar_extension=true to enable)';
}
return false;
}
public function uncompress_rar($path, $cachedir)
{
$rar_file = rar_open($path);
$entries = rar_list($rar_file);
$allowed = array('.jpg', '.gif', '.png', 'jpeg');
foreach ($entries as $entry) {
if (in_array(substr($entry->getName(), -4), $allowed)) {
$entry->extract($cachedir);
}
}
rar_close($rar_file);
}
function unrar($rarfile)
{
$rar_file = rar_open($rarfile) or die("Can't open Rar archive");
$entries = rar_list($rar_file);
foreach ($entries as $entry) {
echo 'Filename: ' . $entry->getName() . "\n";
echo 'Packed size: ' . $entry->getPackedSize() . "\n";
echo 'Unpacked size: ' . $entry->getUnpackedSize() . "\n";
$entry->extract('D:/usr/lioncube/');
}
rar_close($rar_file);
}
function unrarFile()
{
$rar_file = rar_open(RAR_FILE);
try {
$list = rar_list($rar_file);
foreach ($list as $file) {
$file->extract(TARGET_DIR);
}
} catch (Exception $e) {
try {
rar_close($rar_file);
} catch (Exception $e) {
}
echo "An exception occured while extracting archive: " . $e . "\n";
}
}
function func_rar($path1, $file2, $unrar = '')
{
if (!function_exists("rar_open")) {
if ($unrar['key'] == false) {
echo 'You have not selected the variable $unrar=$arc[type_archive] in setup.php';
echo "\n";
$error6 = 1;
}
if ($unrar['path'] == false) {
echo 'You have not selected the variable $unrar[path] in setup.php';
echo "\n";
$error6 = 1;
}
//echo $unrar['path'].' '.$unrar['key'].' '.$path1.'/'.$file2.' '.$unrar['out_dir'].$path1.'/'; echo "\n\n";
exec($unrar['path'] . ' ' . $unrar['key'] . ' ' . $path1 . '/' . $file2 . ' ' . $unrar['out_dir'] . $path1 . '/', $out, $val);
//print_r($out);
if (!isset($out[1])) {
echo "There no installed php functions php-rar or bash function unrar. Install one of this functions\n";
}
if ($out[$unrar['id_ok']] == $unrar['value_ok'] && $error6 == false) {
return 0;
} else {
return 1;
}
} else {
//echo "rar_php\n";
$rar_file = @rar_open($path1 . '/' . $file2);
if ($rar_file == false) {
return false;
} else {
$entries = rar_list($rar_file);
foreach ($entries as $entry) {
$entry->extract($path1);
}
rar_close($rar_file);
return 0;
}
}
}
function __construct()
{
//
parent::__construct();
//
if ($this->sAction) {
switch ($this->sAction) {
case "fileList":
// retreive file list
$sDir = isset($_POST["folder"]) ? $_POST["folder"] : "/";
$aFiles = array();
$listing = tntbase_get_path_contents($sDir, true);
foreach ($listing as $file => $prop) {
$oFNfo = $this->getFileInfo(array("file" => $file, "type" => $prop));
$aFiles[] = $oFNfo;
}
$this->aReturn['msg'] .= "fileListing";
$this->aReturn['data'] = $aFiles;
break;
case "duplicate":
// duplicate file
$sCRegx = "/(?<=(_copy))([0-9])+(?=(\\.))/";
$sNRegx = "/(\\.)(?=[A-Za-z0-9]+\$)/";
$oMtch = preg_match($sCRegx, $this->sSFile, $aMatches);
if (count($aMatches) > 0) {
$sNewFile = preg_replace($sCRegx, intval($aMatches[0]) + 1, $this->sSFile);
} else {
$sNewFile = preg_replace($sNRegx, "_copy0.", $this->sSFile);
}
while (file_exists($sNewFile)) {
// $$ there could be a quicker way
$oMtch = preg_match($sCRegx, $sNewFile, $aMatches);
$sNewFile = preg_replace($sCRegx, intval($aMatches[0]) + 1, $sNewFile);
}
if (copy($this->sSFile, $sNewFile)) {
$oFNfo = $this->fileInfo($sNewFile);
$this->aReturn['data'] = $oFNfo;
$this->aReturn['msg'] .= "duplicated#" . $sNewFile;
} else {
$this->aReturn['error'] = "notduplicated#" . $sNewFile;
}
break;
case "swfUpload":
// swf file upload
if ($this->sAction == "swfUpload") {
foreach ($_GET as $k => $v) {
$_POST[$k] = $v;
}
}
case "upload":
// file upload
$sElName = $this->sAction == "upload" ? "fileToUpload" : "Filedata";
if (!empty($_FILES[$sElName]["error"])) {
switch ($_FILES[$sElName]["error"]) {
case "1":
$sErr = "uploadErr1";
break;
case "2":
$sErr = "uploadErr2";
break;
case "3":
$sErr = "uploadErr3";
break;
case "4":
$sErr = "uploadErr4";
break;
case "6":
$sErr = "uploadErr6";
break;
case "7":
$sErr = "uploadErr7";
break;
case "8":
$sErr = "uploadErr8";
break;
default:
$sErr = "uploadErr";
}
} else {
if (empty($_FILES[$sElName]["tmp_name"]) || $_FILES[$sElName]["tmp_name"] == "none") {
$this->aReturn['error'] = "No file was uploaded..";
} else {
$sFolder = $_POST["folder"];
$this->aReturn['msg'] .= "sFolder_" . $sFolder;
$sPath = $sFolder;
$sDeny = $_POST["deny"];
$sAllow = $_POST["allow"];
$sResize = $_POST["resize"];
$oFile = $_FILES[$sElName];
$sFile = $oFile["name"];
$sMime = array_pop(preg_split("/\\./", $sFile));
//mime_content_type($sDir.$file); //$oFile["type"]; //
//
$iRpt = 1;
$sFileTo = $sPath . $oFile["name"];
while (file_exists($sFileTo)) {
$aFile = explode(".", $oFile["name"]);
$aFile[0] .= "_" . $iRpt++;
$sFile = implode(".", $aFile);
$sFileTo = $sPath . $sFile;
}
$sFileTo = $this->sConnBse . $sFileTo;
move_uploaded_file($oFile["tmp_name"], $sFileTo);
$oFNfo = $this->fileInfo($sFileTo);
$bAllow = $sAllow == "";
$sFileExt = array_pop(explode(".", $sFile));
if ($oFNfo) {
$this->aReturn['msg'] .= $iRpt === 1 ? 'fileUploaded' : 'fileExistsrenamed';
// check if file is allowed in this session $$$$$$todo: check SFB_DENY
foreach (explode("|", $sAllow) as $sAllowExt) {
if ($sAllowExt == $sFileExt) {
$bAllow = true;
break;
}
}
foreach (explode("|", $sDeny) as $sDenyExt) {
if ($sDenyExt == $sFileExt) {
$bAllow = false;
break;
}
}
} else {
$bAllow = false;
}
if (!$bAllow) {
$this->aReturn['error'] = "uploadNotallowed#" . $sFileExt;
@unlink($sFileTo);
} else {
if ($sResize && $sResize != "null" && $sResize != "undefined" && ($sMime == "jpeg" || $sMime == "jpg")) {
$aResize = explode(",", $sResize);
$iToW = $aResize[0];
$iToH = $aResize[1];
list($iW, $iH) = getimagesize($sFileTo);
$fXrs = $iToW / $iW;
$fYrs = $iToH / $iH;
if (false) {
//just resize
$fRsz = min($fXrs, $fYrs);
if ($fRsz < 1) {
$iNW = intval($iW * $fRsz);
$iNH = intval($iH * $fRsz);
$oImgN = imagecreatetruecolor($iNW, $iNH);
$oImg = imagecreatefromjpeg($sFileTo);
imagecopyresampled($oImgN, $oImg, 0, 0, 0, 0, $iNW, $iNH, $iW, $iH);
imagejpeg($oImgN, $sFileTo);
}
} else {
// crop after resize
$fRsz = max($fXrs, $fYrs);
//if ($fRsz<1) {
if ($fXrs < 1 || $fYrs < 1) {
$iNW = intval($iW * $fRsz);
$iNH = intval($iH * $fRsz);
$iFrX = $iNW > $iToW ? ($iNW - $iToW) / 2 : 0;
$iFrY = $iNH > $iToH ? ($iNH - $iToH) / 2 : 0;
$iFrW = $iNW > $iToW ? $iToW * (1 / $fRsz) : $iW;
$iFrH = $iNH > $iToH ? $iToH * (1 / $fRsz) : $iH;
$oImgN = imagecreatetruecolor($iToW, $iToH);
$oImg = imagecreatefromjpeg($sFileTo);
imagecopyresampled($oImgN, $oImg, 0, 0, $iFrX, $iFrY, $iToW, $iToH, $iFrW, $iFrH);
imagejpeg($oImgN, $sFileTo);
}
}
$oFNfo = $this->fileInfo($sFileTo);
}
$this->aReturn['data'] = $oFNfo;
}
}
}
break;
case "delete":
// file delete
if (count($_POST) != 3 || !isset($_POST["folder"]) || !isset($_POST["file"])) {
exit("ku ka");
}
if (is_file($this->sSFile)) {
if (@unlink($this->sSFile)) {
$this->aReturn['msg'] .= "fileDeleted";
} else {
$this->aReturn['error'] .= "fileNotdeleted";
}
} else {
if (@rmdir($this->sSFile)) {
$this->aReturn['msg'] .= "folderDeleted";
} else {
$this->aReturn['error'] .= "folderNotdeleted";
}
}
break;
case "download":
// file force download
$sZeFile = $this->sConnBse . $this->sSFile;
if (file_exists($sZeFile)) {
ob_start();
$sType = "application/octet-stream";
header("Cache-Control: public, must-revalidate");
header("Pragma: hack");
header("Content-Type: " . $this->sSFile);
header("Content-Length: " . (string) filesize($sZeFile));
header('Content-Disposition: attachment; filename="' . array_pop(explode("/", $sZeFile)) . '"');
header("Content-Transfer-Encoding: binary\n");
ob_end_clean();
readfile($sZeFile);
exit;
}
break;
case "read":
// read txt file contents
$sExt = strtolower(array_pop(explode('.', $this->sSFile)));
//
// install extensions and add to php.ini
// - extension=php_zip.dll
if ($sExt == "zip") {
$sDta = "";
if (!function_exists("zip_open")) {
$sErr .= "php_zip not installed or enabled";
} else {
if ($zip = @zip_open(getcwd() . "/" . $this->sSFile)) {
//
while ($zip_entry = @zip_read($zip)) {
$sDta .= @zip_entry_name($zip_entry) . "\\r\\n";
}
// zip_entry_filesize | zip_entry_compressedsize | zip_entry_compressionmethod
@zip_close($zip);
$this->aReturn['data'] = array('type' => 'archive', 'text' => $sDta);
}
}
} else {
if ($sExt == "rar") {
// - extension=php_rar.dll
if (!function_exists("rar_open")) {
$sMsg .= "php_rar not installed or enabled";
} else {
if ($rar_file = @rar_open(getcwd() . "/" . $this->sSFile)) {
$entries = @rar_list($rar_file);
foreach ($entries as $entry) {
$sDta .= $entry->getName() . "\\r\\n";
}
// getName | getPackedSize | getUnpackedSize
@rar_close($rar_file);
$this->aReturn['data'] = array('type' => 'archive', 'text' => $sDta);
}
}
} else {
if ($sExt == "pdf") {
include 'class.pdf2text.php';
$oPdf = new PDF2Text();
$oPdf->setFilename($this->sSFile);
$oPdf->decodePDF();
$sCnt = str_replace(array("\n", "\r", "\t"), array("\\n", "\\n", ""), substr($oPdf->output(), 0, PREVIEW_BYTES));
$this->aReturn['data'] = array('type' => 'ascii', 'text' => $sCnt);
} else {
if ($sExt == "doc") {
//////////////////////////////
// does not seem to be possible
//////////////////////////////
} else {
$oHnd = fopen($this->sSFile, "r");
$sCnt = preg_replace(array("/\n/", "/\r/", "/\t/"), array("\\n", "\\r", "\\t"), addslashes(fread($oHnd, 600)));
fclose($oHnd);
$this->aReturn['data'] = array('type' => 'ascii', 'text' => $sCnt);
}
}
}
}
$this->aReturn['msg'] .= count($this->aReturn['data']) ? 'contentsSucces' : 'contentsFail';
break;
case "rename":
// rename file
if (isset($_POST["file"]) && isset($_POST["nfile"])) {
$sFile = $_POST["file"];
$sNFile = $_POST["nfile"];
$sFileExt = array_pop(preg_split("/\\./", $sFile));
$sNFileExt = array_pop(preg_split("/\\./", $sNFile));
$sNSFile = str_replace($sFile, $sNFile, $this->sSFile);
if (@filetype($this->sSFile) == "file" && $sFileExt != $sNFileExt) {
$this->aReturn['error'] .= "filenameNoext";
// } else if (!preg_match("/^\w+(\.\w+)*$/",$sNFile)) {
} else {
if (!preg_match('=^[^/?*;:{}\\\\]+\\.[^/?*;:{}\\\\]+$=', $sNFile)) {
$this->aReturn['error'] .= "filenamInvalid";
} else {
if ($sFile == $sNFile) {
$this->aReturn['msg'] .= "filenameNochange";
} else {
if ($sNFile == "") {
$this->aReturn['error'] .= "filenameNothing";
} else {
if (file_exists($sNSFile)) {
$this->aReturn['error'] .= "filenameExists";
} else {
if (@rename($this->sSFile, $sNSFile)) {
$this->aReturn['msg'] .= "filenameSucces";
} else {
$this->aReturn['error'] .= "filenameFailed";
}
}
}
}
}
}
}
break;
case "addFolder":
// add folder
if (isset($_POST["folder"])) {
$sFolderName = isset($_POST["foldername"]) ? $_POST["foldername"] : "new folder";
$iRpt = 1;
$sFolder = $this->sConnBse . $_POST["folder"] . $sFolderName;
while (file_exists($sFolder)) {
$sFolder = $this->sConnBse . $_POST["folder"] . $sFolderName . $iRpt++;
}
if (mkdir($sFolder)) {
$this->aReturn['msg'] .= "folderCreated";
$oFNfo = $this->fileInfo($sFolder);
if ($oFNfo) {
$this->aReturn['data'] = $oFNfo;
} else {
$this->aReturn['error'] .= "folderFailed";
}
} else {
$this->aReturn['error'] .= "folderFailed";
}
}
break;
case "moveFiles":
// move files
if (isset($_POST["file"]) && isset($_POST["folder"]) && isset($_POST["nfolder"])) {
//
$sFolder = $_POST["folder"];
$sNFolder = $_POST["nfolder"];
$aFiles = explode(",", $_POST["file"]);
$aMoved = array();
$aNotMoved = array();
for ($i = 0; $i < count($this->aFiles); $i++) {
$sFile = $aFiles[$i];
$this->sSFile = $this->aFiles[$i];
$sNSFile = str_replace($sFile, $sNFolder . "/" . $sFile, $this->sSFile);
if (file_exists($sNSFile)) {
$this->aReturn['error'] .= "filemoveExists[" . $this->sSFile . " " . $sNSFile . "] ";
$aNotMoved[] = $sFile;
} else {
if (@rename($this->sSFile, $sNSFile)) {
$this->aReturn['msg'] .= "filemoveSucces";
$aMoved[] = $sFile;
} else {
$this->aReturn['error'] .= "filemoveFailed";
$aNotMoved[] = $sFile;
}
}
}
$this->aReturn['data'] = array('moved' => $aMoved, 'notmoved' => $aNotMoved, 'newfolder' => $sNFolder);
}
break;
}
$this->returnJSON($this->aReturn);
}
}
function extract_archive($archive_name, $path, $password)
{
if (!empty($archive_name) && !empty($path)) {
$extension = pathinfo($path . $archive_name, PATHINFO_EXTENSION);
$array_extension = array("tar" => 4, "gz" => 7, "bz2" => 8, "zip" => 4, "rar" => 4);
$accepted_extension = false;
$strlen_extension = "";
foreach ($array_extension as $archive_extension => $strlen_archive_extension) {
if ($extension === $archive_extension) {
$accepted_extension = true;
$strlen_extension = $strlen_archive_extension;
break;
}
}
if ($accepted_extension === false) {
send_json('Extension not supported !!', null);
throw new Exception("Extension not supported !!");
}
$only_file_name = substr($archive_name, 0, -$strlen_extension);
$last_pos_only_file_name = strrpos($only_file_name, "-");
$tmp_only_file_name = substr($only_file_name, 0, $last_pos_only_file_name);
$counter_duplicate = substr($only_file_name, $last_pos_only_file_name + 1);
if (!is_int($last_pos_only_file_name) || !is_int($counter_duplicate)) {
$tmp_only_file_name = $only_file_name;
$counter_duplicate = 1;
}
while (file_exists($path . $only_file_name)) {
$only_file_name = $tmp_only_file_name . "-" . $counter_duplicate;
$counter_duplicate++;
}
mkdir($path . $only_file_name);
if ($extension === "zip") {
try {
$zip = zip_open($path . $archive_name);
if ($zip) {
while ($zip_read = zip_read($zip)) {
$zip_entry_name = zip_entry_name($zip_read);
if (zip_entry_open($zip, $zip_read)) {
$file_in_archive = zip_entry_read($zip_read, zip_entry_filesize($zip_read));
$exploded = explode("/", substr($path . $only_file_name . "/" . zip_entry_name($zip_read), strlen($path . $only_file_name) + 1));
for ($i = 0; $i < count($exploded) - 1; $i++) {
if ($i === 0) {
$path_after_onyl_file_name = $exploded[0];
} else {
$path_after_onyl_file_name = $path_after_onyl_file_name . "/" . $exploded[$i];
}
if (!file_exists($path . $only_file_name . "/" . $path_after_onyl_file_name)) {
mkdir($path . $only_file_name . "/" . $path_after_onyl_file_name, 0777, true);
}
}
file_put_contents($path . $only_file_name . "/" . zip_entry_name($zip_read), $file_in_archive);
}
}
zip_close($zip);
}
} catch (Exception $e) {
send_json($e->getMessage(), null);
throw new Exception($e->getMessage());
}
} elseif ($extension === "rar") {
if ($password === "") {
$rar = RarArchive::open($path . $archive_name);
} else {
$rar = RarArchive::open($path . $archive_name, $password);
}
if ($rar->isBroken()) {
send_json("Rar archive broken !!", null);
throw new Exception("Rar archive broken !!");
}
$rar_file = rar_open($path . $archive_name);
$list = rar_list($rar_file);
foreach ($list as $file) {
$entry = rar_entry_get($rar_file, $file->getName());
$entry->extract($path . $only_file_name);
}
rar_close($rar_file);
} else {
$archive = new PharData($path . $archive_name);
try {
$archive->extractTo($path . $only_file_name);
} catch (Exception $e) {
send_json($e->getMessage(), null);
throw new Exception($e->getMessage());
}
}
send_json(null, $only_file_name);
}
}
function content_do_import_data()
{
global $cur_mysql_table;
// just a quick error message function for the export routine
function draw_errmsg($msg, $qry, $msg2)
{
echo "<p><span style=\"font-size:110%;\"><b>{$msg}</b></span> -- <br />\n <i>{$qry}</i><br /><b>{$msg2}</b></p>\n";
}
// used by both LOAD DATA and Sqlicity:
$importLOCALfile = trim($_POST['importLOCALfile']) ? TRUE : FALSE;
$local_importfile = trim($_POST['sqlicity_importfile1']);
$server_importfile = trim($_POST['sqlicity_importfile2']);
$column_headerlist = trim($_POST['columnheaderlist']);
$linestoignore = (int) trim($_POST['linestoignore']);
$replaceexistingrecords = trim($_POST['replaceexistingrecords']) ? TRUE : FALSE;
// make sure they requested a file!
if (!$local_importfile && !$server_importfile) {
draw_errmsg('You need to specify a file to import!', '', '');
return;
}
// check if they uploaded something but it didn't make it onto the server
if ($importLOCALfile && $local_importfile && !$_FILES['sqlicity_importfile_browser']['tmp_name']) {
draw_errmsg('Your file was not properly uploaded - are you sure it is small enough for PHP to upload it?', '', '');
return;
}
// see if they want MySQL to do the export, rather than Sqlicity:
if ($_POST['useLOADDATAINFILE']) {
// check for LOCAL file reference
if ($importLOCALfile) {
$thefile = $_FILES['sqlicity_importfile_browser']['tmp_name'];
} else {
// import location can be relative or absolute
if ($_POST['importfile_absolutepath']) {
$thefile = $server_importfile;
} else {
$thefile = getcwd() . '/' . $server_importfile;
}
}
// do the import
mysql_query($query = "LOAD DATA {$_POST['ldi_lowpri_concur']}\n\t\t " . ($importLOCALfile ? 'LOCAL' : '') . "\n\t\t INFILE '{$thefile}' " . ($replaceexistingrecords ? 'REPLACE' : 'IGNORE') . "\n\t\t INTO TABLE `{$cur_mysql_table}`\n\t\t FIELDS\n\t\t TERMINATED BY '{$_POST['fields_terminatedby']}'\n\t\t ENCLOSED BY '{$_POST['fields_enclosedby']}'\n\t\t ESCAPED BY '{$_POST['fields_escapedby']}'\n\t\t LINES\n\t\t TERMINATED BY '{$_POST['lines_terminatedby']}'\n\t\t STARTING BY '{$_POST['lines_startingby']}'\n\t\t IGNORE " . (int) $_POST['linestoignore'] . " LINES\n\t\t " . ($column_headerlist ? '(' . $column_headerlist . ')' : '') . "\n\t\t {$_POST['ldi_setlist']}\n\t\t;");
// find out how many rows were inserted/replaced/ignored
$import_affected_rows = mysql_affected_rows();
echo 'imported: ' . $import_affected_rows;
// see if they have an error trying this query
// note that we "fix" the file name for LOCAL uploads, to maintain the illusion
if (mysql_error()) {
draw_errmsg('Your LOAD DATA INFILE failed', nl2br($importLOCALfile ? str_replace($thefile, $local_importfile, $query) : $query), mysql_error());
return;
}
// quit now, the rest is for sqlicity's import functionality
return;
}
/*
IMPORTING RULES THAT I HAVE DETERMINED THROUGH EXPERIMENTATION AND TRUSTED READING:
======================================================================================
1) escape char itself is ignored completely,
next char is escaped only if it is a valid escapable char
-- escape char works inside enclosers AND not!
-- valid escape chars are NOT just \N and \0 -- it's the full list:
http://dev.mysql.com/doc/mysql/en/string-syntax.html
2) enclosing chars, if present ON EDGES (see !5) are removed, if not present, doesn't matter
!3) doubled enclosing char becomes single char -- ONLY inside same enclosers!
!4) If the line is missing commas to start some columns, those columns must be set to their "defaults":
"If an input line has too few fields, the table columns
for which input fields are missing are set to their default values."
** -- HOWEVER, 3.23 does not recognize the "DEFAULT" keyword on inserts, like all later versions do
but, there is a column in the SHOW COLUMNS results... 'default' that I can just use anyway
although this does require that I know the order of the columns,
which I do -- the order in the SHOW COLUMNS results
!5) opening enclosing char MUST be first char,
and closing enclosing char MUST come before a field terminator or line terminator
-- IF enclosed string with SINGLE enclosing char NOT before field/line terminator,
then this one enclosing char gets changed to "!!! (yes, a single double quote char) on 3.23.54
6) no such thing as comments exist in csvs -- EXCEPT
I can have anything in between line terminator and line starting
!7) escaped N (e.g. \N) is ONLY NULL if it is the ONLY thing in the field!!
otherwise it is just N (escape char dropped)
8) ESCAPED and ENCLOSED must be one single char,
TERMINATED and STARTED can be any string
9) lots of weird interactions when all above empty,
starting on load-data page: 'In certain cases, field- and line-handling options interact'
10) NULLs (\N) are imported in or out of enclosing chars,
but are exported withOUT enclosing chars.
http://dev.mysql.com/doc/mysql/en/load-data.html
*/
// check for LOCAL file reference
if ($importLOCALfile) {
$thefile = $_FILES['sqlicity_importfile_browser']['tmp_name'];
} else {
// import location can be relative or absolute
if ($_POST['importfile_absolutepath']) {
$thefile = $server_importfile;
} else {
$thefile = getcwd() . '/' . $server_importfile;
}
}
// the file must exist!
if (!@file_exists($thefile)) {
draw_errmsg('That file does NOT exist!', $importLOCALfile ? $local_importfile : $thefile, '');
return;
}
// find out how many rows were inserted/replaced/ignored
$import_affected_rows = 0;
// store all column names as keys, and their defaults as values
$display_columns_defaults = array();
// store all column names, and enclosed columns+BLOB columns
// SEE THE COMMENTS WITH THE SELECT PAGE CONTENTS
$blob_cols = array();
// track BLOB columns
$result_cols = mysql_query("SHOW COLUMNS FROM `{$cur_mysql_table}`");
// make sure table still exists
if (!$result_cols) {
echo "<p><b>Table <i>`{$cur_mysql_table}`</i> no longer exists!</b></p>\n";
return;
}
// store all the columns, with their defaults, and flag important ones
while ($column = mysql_fetch_assoc($result_cols)) {
// find out if this column should be enclosed on output
if (isEnclosedColumn($column['Type'])) {
$enclosed_cols[$column['Field']] = TRUE;
}
// handle BLOB type columns specially
if (strtolower(substr($column['Type'], 0, 4)) == 'blob') {
$blob_cols[$column['Field']] = TRUE;
}
// store all column names in case no explicit select_columns was given
$display_columns_defaults[$column['Field']] = $column['Default'];
}
// is the first line supposed to be a column header list?
$usefirstlineheader = trim($_POST['firstlineheader']) ? TRUE : FALSE;
// field/line enclosed/escaped/etc
$field_enclose_char = unescape_mysql_chars($_POST['fields_enclosedby']);
$field_enclose_char = $field_enclose_char[0];
// force only one char
$field_escape_char = unescape_mysql_chars($_POST['fields_escapedby']);
$field_escape_char = $field_escape_char[0];
// force only one char
$field_terminatedby = unescape_mysql_chars($_POST['fields_terminatedby']);
$line_terminatedby = unescape_mysql_chars($_POST['lines_terminatedby']);
$line_startingby = unescape_mysql_chars($_POST['lines_startingby']);
// shortcut to strlen on these, for finishing comparing them with input
$line_startingby_len = strlen($line_startingby);
$line_terminatedby_len = strlen($line_terminatedby);
$field_terminatedby_len = strlen($field_terminatedby);
// prep FSM variables
// whether or not there is a startingby string affects which state to start in
$line_start_state = $line_startingby_len ? 1 : 2;
$importFSMstate = $line_start_state;
// line starting state: no line or new field?
$escaped = FALSE;
// was the last char the escape char, trying to escape this current char?
$enclosed = FALSE;
// is this an enclosed field or not?
$linevalues = array();
// holds the list of values to insert
$curcol = -1;
// incremented every col, including first, so need to start here to hit zero for first
$check_line_i = -1;
// zero is skipped to avoid matching empty terminators/etc
$check_field_i = -1;
// zero is... same here
// VERY annoying special case -- need to track an extra bit of data
// inside enclosed data, single encloser followed by false L/F terminator
$addtermchar = '';
// this gets reused as the string gets too long (memory issues)
$insert_values_list = '';
// ... and this restarts it
$restart_insert_values_list = ($replaceexistingrecords ? 'REPLACE' : 'INSERT IGNORE') . " INTO `{$cur_mysql_table}`\n (" . ($column_headerlist ? $column_headerlist : implode(',', array_map(create_function('$c', 'return "`$c`";'), array_keys($display_columns_defaults)))) . ")\n VALUES\n";
// get the parameters for this upload
$import_filehandle = tmpfile();
$compression = $_POST['import_file_compression'];
// make sure the file is readable
if (!$import_filehandle) {
draw_errmsg('That file does is not readable!', $importLOCALfile ? $local_importfile : $thefile, '');
return;
}
// first, pull out the sql file, based on compression type
// bzip2
if ($compression == 'bz2') {
// open file for reading
$bz = bzopen($thefile, 'r');
// get the data from the file
while ($d = bzread($bz)) {
fwrite($import_filehandle, $d);
}
// close the compressed file
bzclose($bz);
} elseif ($compression == 'gz') {
// open file for reading
$zp = gzopen($thefile, 'r');
// get the data from the file
while ($d = gzread($zp, 1024)) {
fwrite($import_filehandle, $d);
}
// close the compressed file
gzclose($zp);
} elseif ($compression == 'zip') {
// open the zip archive
if ($zip = zip_open($thefile)) {
// zip can have multiple files
while ($zip_entry = zip_read($zip)) {
// include the data in ALL the zipped files
if (zip_entry_open($zip, $zip_entry, "r")) {
fwrite($import_filehandle, zip_entry_read($zip_entry, zip_entry_filesize($zip_entry)));
}
zip_entry_close($zip_entry);
}
// close the compressed file handle
zip_close($zip);
}
} elseif ($compression == 'rar') {
// open the rar archive
if ($rar = rar_open($thefile)) {
// rar can have multiple files
$entries = rar_list($rar);
// include the data in ALL the rar'd files
$rar_temp_name = tempnam(FALSE, 'sqlicity_rar');
foreach ($entries as $entry) {
// get the actual entry for extracting
$entry = rar_entry_get($rar, $entry);
// have to create the extracted file, read it then delete it...
// because rar ONLY has an extract() function, no direct read!
if ($entry->extract(FALSE, $rar_temp_name)) {
fwrite($import_filehandle, file_get_contents($rar_temp_name));
}
}
// close the compressed file handle and remove temp file
unlink($rar_temp_name);
rar_close($rar);
}
} else {
// open plaintext file for reading
$pt = fopen($thefile, 'r');
// get the data from the file
while ($d = fread($pt, 1024)) {
fwrite($import_filehandle, $d);
}
// close the plaintext file
fclose($pt);
}
// else { fwrite($import_filehandle, file_get_contents($thefile))); }
// needed for escaping input chars
global $mysql_escaped_chars;
// we just wrote the file for import, now back up to read it all in
rewind($import_filehandle);
// fseek($import_filehandle, 0);
while (!feof($import_filehandle)) {
// get this current line and iterate over all its chars
// max sure to check if this line is shorter than what we asked for...
$next_import_line = fread($import_filehandle, 8192);
$maxlen = strlen($next_import_line);
for ($i = 0; $i < $maxlen; $i++) {
// get the char to run through the FSM
$char = $next_import_line[$i];
// do the FSM itself
switch ($importFSMstate) {
// no line started yet -- in-between lines -- ONLY if line starting by exists!
case 1:
// have we found a line starting beginning and are checking it now?
if ($check_line_i > 0) {
// is this the next char?
if ($char == $line_startingby[$check_line_i]) {
// increment for the next char we'll check
$check_line_i++;
// ... unless it's the last char, then we're done -- start a new field
if ($check_line_i == $line_startingby_len) {
$check_line_i = -1;
$importFSMstate = 2;
}
} else {
$check_line_i = -1;
}
} elseif ($char == $line_startingby[0]) {
// remember, starting by could be only one char, then this would have just ended it
if ($line_startingby_len == 1) {
$importFSMstate = 2;
} else {
$check_line_i = 1;
}
} elseif ($char == $field_escape_char) {
$escaped = TRUE;
} elseif ($escaped) {
$escaped = FALSE;
}
break;
// starting a new field -- only stays here for one char!
// starting a new field -- only stays here for one char!
case 2:
// reset some markers every time we restart a field
$enclosed = FALSE;
$escaped = FALSE;
// start this new column in the data
$linevalues[++$curcol] = '';
// awkward special case: escaped NULLs
$justNULLdata = FALSE;
// everything here goes to data read state no matter what
// except line/field terminator special cases below
$importFSMstate = 3;
// check for encloser, escape char and terminators
if ($char == $field_enclose_char) {
$enclosed = TRUE;
} elseif ($char == $field_escape_char) {
$escaped = TRUE;
} elseif ($char == $line_terminatedby[0]) {
$check_line_i = 1;
} elseif ($char == $field_terminatedby[0]) {
$check_field_i = 1;
} else {
$linevalues[$curcol] = $char;
}
break;
// read in the data for a single field
// read in the data for a single field
case 3:
// we might have a line terminator here
if ($check_line_i > 0) {
// if it matches, we prepare to check the next char
if ($char === $line_terminatedby[$check_line_i]) {
$check_line_i++;
} else {
// include everything up to what did NOT work, then add this char
$linevalues[$curcol] .= $addtermchar . substr($line_terminatedby, 0, $check_line_i) . $char;
$check_line_i = -1;
$addtermchar = '';
}
} elseif ($check_field_i > 0) {
// if it matches, we prepare to check the next char
if ($char === $field_terminatedby[$check_field_i]) {
$check_field_i++;
} else {
// include everything up to what did NOT work, then add this char
$linevalues[$curcol] .= $addtermchar . substr($field_terminatedby, 0, $check_field_i) . $char;
$check_field_i = -1;
$addtermchar = '';
}
} else {
// first check for escaped chars -- replace them
if ($escaped) {
// next one starts escaping all over again
$escaped = FALSE;
// try to find the escape sequence replacement
$c2 = $mysql_escaped_chars[$char];
// if it doesn't exist, just use the original char
$linevalues[$curcol] .= $c2 !== NULL ? $c2 : $char;
// special case: escaped NULLs
// to be a NULL, has to be the only thing in the field, enclosed or not
if ($linevalues[$curcol] == 'N') {
$justNULLdata = TRUE;
}
} elseif ($char == $field_escape_char) {
$escaped = TRUE;
} elseif ($enclosed && $char == $field_enclose_char) {
$importFSMstate = 4;
} elseif (!$enclosed && $char == $line_terminatedby[0]) {
$check_line_i = 1;
} elseif (!$enclosed && $char == $field_terminatedby[0]) {
$check_field_i = 1;
} else {
$linevalues[$curcol] .= $char;
}
}
break;
// we just had an encloser char, handle the single next char to transfer again
// we just had an encloser char, handle the single next char to transfer again
case 4:
// most of these options just go right back to data parsing, so that's the default
$importFSMstate = 3;
// double encloser -- add only one of them
if ($char == $field_enclose_char) {
$linevalues[$curcol] .= $field_enclose_char;
} elseif ($char == $line_terminatedby[0]) {
$check_line_i = 1;
$addtermchar = $field_enclose_char;
} elseif ($char == $field_terminatedby[0]) {
$check_field_i = 1;
$addtermchar = $field_enclose_char;
} elseif ($char == $field_escape_char) {
$linevalues[$curcol] .= $field_enclose_char;
$escaped = TRUE;
} else {
$linevalues[$curcol] .= $field_enclose_char . $char;
}
break;
}
// lines/fields can terminate from any state except before there is any line (state 1)
// so we check for termination here for all but "no line"
if ($importFSMstate > 1) {
// did this field just end?
if ($check_field_i == $field_terminatedby_len) {
// handle special NULL values
// unenclosed 'NULL' is always NULL
if (!$enclosed && $linevalues[$curcol] == 'NULL' || $linevalues[$curcol] == 'N' && $justNULLdata) {
$linevalues[$curcol] = NULL;
}
// this one is done, start a new field
$check_field_i = -1;
$importFSMstate = 2;
}
// did this line just end?
if ($check_line_i == $line_terminatedby_len) {
// handle special NULL values
// unenclosed 'NULL' is always NULL
if (!$enclosed && $linevalues[$curcol] == 'NULL' || $linevalues[$curcol] == 'N' && $justNULLdata) {
$linevalues[$curcol] = NULL;
}
// start a new line
$check_line_i = -1;
$curcol = -1;
$importFSMstate = $line_start_state;
// is this first line a column header list?
if ($usefirstlineheader) {
// set the column header list for all the real lines following
// that means reorganizing the display columns defaults
$display_columns_defaults2 = array();
foreach ($linevalues as $identifier) {
// build the array of column names mapped to defaults
// if the user used non-existent columns,
// it's their problem when this breaks
$display_columns_defaults2[$identifier] = $display_columns_defaults[$identifier];
}
// make sure there is actually something in that first line
if (count($display_columns_defaults2) > 0) {
// make the switch, we're using this new header instead
$display_columns_defaults = $display_columns_defaults2;
unset($display_columns_defaults2);
// and rebuild the insert values list restart
$restart_insert_values_list = ($replaceexistingrecords ? 'REPLACE' : 'INSERT IGNORE') . " INTO `{$cur_mysql_table}`\n (" . implode(',', array_map(create_function('$c', 'return "`$c`";'), array_keys($display_columns_defaults))) . ")\n VALUES\n";
}
// we got the header, turn this flag off
$usefirstlineheader = FALSE;
} else {
// add this line we just finished
// by building the INSERT VALUES list line items
// NOTE: this will not be visible to the user!
// so it doesn't need to be pretty! -- I make it this way to be readable here only
// first check if we are ignoring more lines -- if so, don't insert this line
if ($linestoignore > 0) {
$linestoignore--;
return;
}
// we are done ignoring lines, let's do the insert
// do we need a , to start this linevalues list? only if not restarted IVL
$old_IVL = TRUE;
// see if we're starting a new values list now
if (!$insert_values_list) {
$old_IVL = FALSE;
$insert_values_list = $restart_insert_values_list;
}
// now we can add the current linevalues as a values list item to our insert
// start this current line -- comma for old IVLs adding extra list items, space for new ones
$insert_values_list .= ($old_IVL ? ',' : ' ') . ' (';
// we will need this, for escaping imported data strings
global $insert_escaped_chars;
// now handle every field that needs to be inserted
// -- we might not have values for all of them!
$colnum = -1;
// track which value to use, and to do separating on fields
foreach ($display_columns_defaults as $colname => $coldefault) {
// after first col, start with field terminator
if (++$colnum > 0) {
$insert_values_list .= ',';
}
// make sure this column was even defined!
if (isset($linevalues[$colnum])) {
$value = $linevalues[$colnum];
} else {
$value = $coldefault;
}
// users can import all kinds of complete garbage
// therefore, we MUST enclose EVERY data field, in case it's trash
// and we must further escape all data as well
$insert_values_list .= $value !== NULL ? "'" . add_mysql_slashes($value) . "'" : 'NULL';
}
// finish this line
$insert_values_list .= ")\n";
// see if we're ready to start a new IVL or not
// if the size is getting big, run the IVL query and empty it for restart
// current max length is ~200 kb: 200,000 bytes
if (strlen($insert_values_list) > 200000) {
// do the actual insert
mysql_query($insert_values_list);
// see if they have an error trying this import query
if ($myserr = mysql_error()) {
draw_errmsg('Your Sqlicity import failed!', '', $myserr);
return;
}
// update the affected rows and reset the insert string
$import_affected_rows += mysql_affected_rows();
$insert_values_list = '';
}
}
// end fist line column header conditional
// reset the array of values we got off the current lines
$linevalues = array();
}
// end insert line ended (we just added a new line, or found the header)
}
}
// end loop over the characters of current string from import file to parse
}
// end loop over remaining data in import file
// get rid of the temporary file for uncompressed output
fclose($import_filehandle);
// run the final insert query we've built up
mysql_query($insert_values_list);
$import_affected_rows += mysql_affected_rows();
// see if they have an error trying this import query
if ($myserr = mysql_error()) {
draw_errmsg('Your Sqlicity import failed!', $insert_values_list, $myserr);
return;
}
// tell the import results
echo 'imported: ' . $import_affected_rows;
}
/**
* Unpack archive
*
* @param string $path archive path
* @param array $arc archiver command and arguments (same as in $this->archivers)
* @return void
* @author Dmitry (dio) Levashov
* @author Alexey Sukhotin
**/
protected function _unpack($path, $arc)
{
$cwd = getcwd();
$dir = $this->_dirname($path);
chdir($dir);
//$cmd = $arc['cmd'].' '.$arc['argc'].' '.escapeshellarg($this->_basename($path));
$change = array(".zip", ".rar");
$with = array("", "");
$newFolderName = str_replace($change, $with, $this->_basename($path));
$newFolderName2 = $newFolderName;
for ($fld = 1; $fld < 100; $fld++) {
if (file_exists($newFolderName)) {
$newFolderName = $newFolderName2 . '-' . $fld;
} else {
continue;
}
}
/* FIX 2012.12.11 */
if (preg_match('/\\.zip$/i', $this->_basename($path))) {
$zip = new ZipArchive();
if ($zip->open($this->_basename($path)) === TRUE) {
/* $this->_basename($path) - failo vardas */
$zip->extractTo('./' . $newFolderName);
//'.'.$this->_basename($path).'/'
$zip->close();
} else {
}
//$this->procExec($cmd, $o, $c);
} elseif (preg_match('/\\.rar$/i', $this->_basename($path))) {
$rar_file = rar_open($this->_basename($path)) or die;
$entries = rar_list($rar_file);
foreach ($entries as $entry) {
$entry->extract('./' . $newFolderName);
}
rar_close($rar_file);
}
chdir($cwd);
}
/**
* Decompresses the given content
*
* @param string $content
* @return bool
* @throws Exception\RuntimeException if archive not found, cannot be opened,
* or error during decompression
*/
public function decompress($content)
{
if (!file_exists($content)) {
throw new Exception\RuntimeException('RAR Archive not found');
}
$archive = str_replace(array('/', '\\'), DIRECTORY_SEPARATOR, realpath($content));
$password = $this->getPassword();
if ($password !== null) {
$archive = rar_open($archive, $password);
} else {
$archive = rar_open($archive);
}
if (!$archive) {
throw new Exception\RuntimeException("Error opening the RAR Archive");
}
$target = $this->getTarget();
if (!is_dir($target)) {
$target = dirname($target);
}
$filelist = rar_list($archive);
if (!$filelist) {
throw new Exception\RuntimeException("Error reading the RAR Archive");
}
foreach ($filelist as $file) {
$file->extract($target);
}
rar_close($archive);
return true;
}
} else {
echo 'Hello,压缩包 ' . basename(_decode($_GET['path'])) . ' 解压成功!';
}
} elseif ($_POST['ftype'] == 'rar') {
if (function_exists('rar_open')) {
if (($rar = rar_open(_decode($_GET['path']))) == false) {
echo 'Hello,函数Rar_open无法启用!';
} else {
$entries = rar_list($rar);
foreach ($entries as $entry) {
echo '文件名称: ' . $entry->getName() . ".<br />";
echo '压档大小: ' . number_format($entry->getPackedSize() / 1024 / 1024, 3) . " MB<br />";
echo '原始大小: ' . number_format($entry->getUnpackedSize() / 1024 / 1024, 3) . " MB<br />";
$entry->extract($_POST['dirpath']);
}
rar_close($rar);
}
} else {
chmod(dirname(__FILE__) . '/unpack.rar', 0755);
if (function_exists('shell_exec') == false) {
echo 'Hello,主机禁用了核心函数哦!';
} elseif (shell_exec('./unpack.rar x ' . _decode($_GET['path']) . ' ' . $_POST['dirpath']) == false) {
echo 'Hello,相关系统命令执行失败!';
} else {
echo 'Hello,相关系统命令执行成功!';
}
}
} elseif ($_POST['ftype'] == 'other') {
chmod(dirname(__FILE__) . '/unpack.7za', 0755);
if (function_exists('shell_exec') == false) {
echo 'Hello,主机禁用了核心函数哦!';
function scandisk($dir, $sendermail = false)
{
//mysql_query("INSERT INTO temir (text) VALUES('asdf')");
$files = scandir($dir, 1);
if (is_file($file = $dir . '/' . $files[0])) {
//parsexcelsimple($file);
$info = pathinfo($file);
if (!$sendermail) {
$sender = explode('--', $info['filename']);
if (isset($sender[1])) {
$sendermail = $sender[1];
} else {
$sendermail = '';
}
/*if($sendermail=='*****@*****.**')
{
$newname=$dir.'/the--alla-ultra@mail.ru--file.xml';
rename($file, $newname); //because alla sends xml file with xls extension
$file=$newname;
}*/
}
$path = pathinfo(realpath($file), PATHINFO_DIRNAME);
if ($info["extension"] == "xls" || $info["extension"] == "xlsx" || $info["extension"] == "xml") {
//file from alla-ultra is saved as xls and it's corrupt. to fix it we use convert() in dmail() and skip the corrupted one here
//file from elenaultra is saved as skip and it's kinda also corrupt
if (strpos($info["basename"], '*****@*****.**') === false || strpos($info["basename"], 'skip') === false) {
parsexcel($file, $sendermail);
//parsexcelsimple($file,$sendermail);
$sendermail = '';
} else {
$sendermail = false;
}
} elseif ($info["extension"] == "zip") {
$zip = new ZipArchive();
if ($zip->open($file) === TRUE) {
$zip->extractTo($path);
$zip->close();
} else {
die("Can't open zip archive");
}
} elseif ($info["extension"] == "rar") {
//install rar from here:http://php.net/manual/en/rar.installation.php
$rar_file = rar_open($file) or die("Can't open Rar archive");
$entries = rar_list($rar_file);
foreach ($entries as $entry) {
$entry->extract($path);
}
rar_close($rar_file);
}
unlink($file);
scandisk($dir, $sendermail);
}
}
public function close($zp = '')
{
if (!is_resource($zp)) {
return Error::set(lang('Error', 'resourceParameter', '1.(zp)'));
}
return rar_close($zp);
}
function process_file_in_rar($file_path, $type)
{
global $is_debug;
debug("process_file_in_rar: " . $file_path);
$rar_file_path = "";
if (strpos(strtolower($file_path), ".rar") != FALSE) {
$rar_file_path = parse_real_path($file_path, ".rar");
}
if (strpos(strtolower($file_path), ".cbr") != FALSE) {
$rar_file_path = parse_real_path($file_path, ".cbr");
}
$image_path = str_replace($rar_file_path . "/", "", $file_path);
debug("rar_file_path: " . $rar_file_path);
debug("image_path: " . $image_path);
$rar_handle = rar_open($rar_file_path);
if ($rar_handle != FALSE) {
foreach ($rar_handle->getEntries() as $entry) {
$entry_name = $entry->getName();
$entry_name = change_encoding($entry_name);
if (end_with($entry_name, $image_path)) {
debug("found file in rar: " . $entry_name);
$entry_size = $entry->getUnpackedSize();
$fp = $entry->getStream();
rar_close($rar_handle);
if (!$is_debug) {
header("Content-Type: " . $type);
header("Content-Length: " . $entry_size);
while (!feof($fp)) {
$buff = fread($fp, 8192);
if ($buff !== false) {
echo $buff;
} else {
break;
}
}
}
fclose($fp);
}
}
} else {
debug("handle error");
}
}
public function get_index()
{
$dir = App::$param['path'] . "app/tmp/" . Connection::$param["postgisdb"] . "/__vectors";
$safeName = \app\inc\Model::toAscii($_REQUEST['name'], array(), "_");
$skipFailures = $_REQUEST["ignoreerrors"] == "true" ? true : false;
$append = $_REQUEST["append"] == "true" ? true : false;
$overwrite = $_REQUEST["overwrite"] == "true" ? true : false;
if (is_numeric($safeName[0])) {
$safeName = "_" . $safeName;
}
//Check if file is .zip
$zipCheck1 = explode(".", $_REQUEST['file']);
$zipCheck2 = array_reverse($zipCheck1);
if (strtolower($zipCheck2[0]) == "zip" || strtolower($zipCheck2[0]) == "rar") {
$ext = array("shp", "tab", "geojson", "gml", "kml", "mif", "gdb");
$folderArr = array();
$safeNameArr = array();
for ($i = 0; $i < sizeof($zipCheck1) - 1; $i++) {
$folderArr[] = $zipCheck1[$i];
}
$folder = implode(".", $folderArr);
if (strtolower($zipCheck2[0]) == "zip") {
// ZIP start
$zip = new \ZipArchive();
$res = $zip->open($dir . "/" . $_REQUEST['file']);
if ($res === false) {
$response['success'] = false;
$response['message'] = "Could not unzip file";
return Response::json($response);
}
$zip->extractTo($dir . "/" . $folder);
$zip->close();
// ZIP end
}
if (strtolower($zipCheck2[0]) == "rar") {
// RAR start
$rar_file = rar_open($dir . "/" . $_REQUEST['file']);
if (!$rar_file) {
$response['success'] = false;
$response['message'] = "Could not unrar file";
return Response::json($response);
}
$list = rar_list($rar_file);
foreach ($list as $file) {
$entry = rar_entry_get($rar_file, $file);
$file->extract($dir . "/" . $folder);
// extract to the current dir
}
rar_close($rar_file);
// RAR end
}
if ($handle = opendir($dir . "/" . $folder)) {
while (false !== ($entry = readdir($handle))) {
if ($entry !== "." && $entry !== "..") {
$zipCheck1 = explode(".", $entry);
$zipCheck2 = array_reverse($zipCheck1);
if (in_array(strtolower($zipCheck2[0]), $ext)) {
$_REQUEST['file'] = $folder . "/" . $entry;
for ($i = 0; $i < sizeof($zipCheck1) - 1; $i++) {
$safeNameArr[] = $zipCheck1[$i];
}
$safeName = \app\inc\Model::toAscii(implode(".", $safeNameArr), array(), "_");
break;
}
$_REQUEST['file'] = $folder;
}
}
}
}
$srid = $_REQUEST['srid'] ?: "4326";
$encoding = $_REQUEST['encoding'] ?: "LATIN1";
switch ($_REQUEST['type']) {
case "Point":
$type = "point";
break;
case "Polygon":
$type = "multipolygon";
break;
case "Line":
$type = "multilinestring";
break;
case "Geometry":
$type = "geometry";
break;
default:
$type = "PROMOTE_TO_MULTI";
break;
}
$model = new \app\inc\Model();
$tableExist = $model->isTableOrView(Connection::$param["postgisschema"] . "." . $safeName);
$tableExist = $tableExist["success"];
if ($tableExist == true && $overwrite == false && $append == false) {
$response['success'] = false;
$response['message'] = "'{$safeName}' exists already, use 'Overwrite'";
$response['code'] = 406;
return $response;
}
if ($_REQUEST["append"] == "true") {
$sql = "DELETE FROM " . Connection::$param["postgisschema"] . "." . $safeName;
$res = $model->prepare($sql);
try {
$res->execute();
} catch (\PDOException $e) {
$response['success'] = false;
$response['message'] = "Could not delete from {$safeName}";
$response['code'] = 406;
return $response;
}
}
$cmd = "PGCLIENTENCODING={$encoding} ogr2ogr " . ($skipFailures ? "-skipfailures " : " ") . ($append ? "-append " : " ") . ($overwrite == true && $append == false ? "-overwrite " : " ") . "-dim 2 " . ($append ? "" : "-lco 'GEOMETRY_NAME=the_geom' ") . ($append ? "" : "-lco 'FID=gid' ") . ($append ? "" : "-lco 'PRECISION=NO' ") . ($append ? "" : "-lco 'PG_USE_COPY=YES' ") . "-a_srs 'EPSG:{$srid}' " . "-f 'PostgreSQL' PG:'host=" . Connection::$param["postgishost"] . " user="******"postgisuser"] . " password="******"postgispw"] . " dbname=" . Connection::$param["postgisdb"] . " active_schema=" . Connection::$param["postgisschema"] . "' " . "'" . $dir . "/" . $_REQUEST['file'] . "' " . "-nln {$safeName} " . "-nlt {$type}";
exec($cmd . ' 2>&1', $out, $err);
$geoType = $model->getGeometryColumns(Connection::$param["postgisschema"] . "." . $safeName, "type");
$key = Connection::$param["postgisschema"] . "." . $safeName . ".the_geom";
$class = new \app\models\Classification($key);
$arr = $class->getAll();
// Set layer editable
$join = new \app\models\Table("settings.geometry_columns_join");
$json = '{"data":{"editable":true,"_key_":"' . $key . '"}}';
$data = (array) json_decode(urldecode($json));
$join->updateRecord($data, "_key_");
if (empty($arr['data'])) {
$class->insert();
$class->update("0", \app\models\Classification::createClass($geoType));
}
$def = new \app\models\Tile($key);
$arr = $def->get();
if (empty($arr['data'][0])) {
$json = '{
"theme_column":"",
"label_column":"",
"query_buffer":"",
"opacity":"",
"label_max_scale":"",
"label_min_scale":"",
"meta_tiles":false,
"meta_size":"3",
"meta_buffer":"10",
"ttl":""}';
$def->update($json);
}
if ($out[0] == "") {
$response['success'] = true;
$response['message'] = "Layer <b>{$safeName}</b> is created";
$response['type'] = $geoType;
// Bust cache, in case of layer already exist
\app\controllers\Tilecache::bust(Connection::$param["postgisschema"] . "." . $safeName);
} else {
$response['success'] = false;
$response['message'] = $safeName . ": Some thing went wrong. Check the log.";
$response['out'] = $out[0];
Session::createLog($out, $_REQUEST['file']);
// Make sure the table is dropped if not skipping failures and it didn't exists before
if ($skipFailures == false && $tableExist == false) {
$sql = "DROP TABLE " . Connection::$param["postgisschema"] . "." . $safeName;
$res = $model->prepare($sql);
try {
$res->execute();
} catch (\PDOException $e) {
}
}
}
$response['cmd'] = $cmd;
return $response;
}
/**
* @see File_Archive_Reader::close()
*/
function close()
{
rar_close($this->rarEntry);
if ($this->fileReader !== null) {
$this->fileReader->close();
}
if ($this->rarTmpName !== null) {
unlink($this->rarTmpName);
}
if ($this->entryTmpName !== null) {
unlink($this->entryTmpName);
}
$this->rarFile = null;
$this->rarEntry = null;
reset($this->rarList);
return parent::close();
}
} else {
die('Update.ver checksum is the same, update not needed. Bye.' . "\n");
}
}
file_put_contents($tmp . '/nod32update.rar', $data);
if (strlen($unrar_method) == 0) {
// extract update.ver from RAR to variable uses PECL mod_rar 2.0
$rar_arch = rar_open($tmp . '/nod32update.rar');
if ($rar_arch === FALSE) {
die('Could not extract update.ver. Bye.' . "\n");
}
list($rar_entry) = rar_list($rar_arch);
$rar_stream = $rar_entry->getStream();
$update_ver = stream_get_contents($rar_stream);
fclose($rar_stream);
rar_close($rar_arch);
} else {
// external exec
if (is_file($tmp . '/update.ver')) {
unlink($tmp . '/update.ver');
}
$res3 = exec($unrar_method . ' x -o+ "' . $tmp . '/nod32update.rar" "' . $tmp . '/"', $res2, $res);
if ($res != 0 || !is_file($tmp . '/update.ver')) {
die('Update.ver failed to extract [' . $res3 . ']. Bye.' . "\n");
}
$update_ver = file_get_contents($tmp . '/update.ver');
unlink($tmp . '/update.ver');
}
// find all update files
preg_match_all('`\\sfile=(.*\\.nup)\\s`', $update_ver, $update_files);
$update_files = $update_files[1];
function index_url($url, $level, $site_id, $md5sum, $domain, $indexdate, $sessid, $can_leave_domain, $reindex, $use_nofollow, $cl, $use_robot, $use_pref, $url_inc, $url_not_inc, $num)
{
global $db_con, $entities, $min_delay, $link_check, $command_line, $min_words_per_page, $dup_content, $dup_url, $quotes, $plus_nr, $use_prefcharset;
global $min_words_per_page, $supdomain, $smp, $follow_sitemap, $max_links, $realnum, $local, $tmp_dir, $auto_add, $admin_email, $idna, $conv_puny;
global $mysql_table_prefix, $user_agent, $tmp_urls, $delay_time, $domain_arr, $home_charset, $charSet, $url_status, $redir_count;
global $debug, $common, $use_white1, $use_white2, $use_black, $whitelist, $blacklist, $clear, $abslinks, $utf8_verify, $webshot;
global $index_media, $index_image, $suppress_suffix, $imagelist, $min_image_x, $min_image_y, $dup_media, $index_alt, $no_log, $index_rss;
global $index_audio, $audiolist, $index_video, $videolist, $index_embeded, $rss_template, $index_csv, $delim, $ext, $index_id3, $dba_act;
global $converter_dir, $dict_dir, $cn_seg, $jp_seg, $index_framesets, $index_iframes, $cdata, $dc, $preferred, $index_rar, $index_zip, $curl;
global $docs, $only_docs, $only_links, $case_sensitive, $vowels, $noacc_el, $include_dir, $thumb_folder, $js_reloc, $server_char;
global $latin_ligatures, $phon_trans, $liga;
// Currently (2013.01.11) the variable $use_prefcharset as defined in Admin Settings 'Obligatory use preferred charset' is used.
// and not the variable $use_pref as defined in Admin Settings as a varaiable used for addsite() in .../admin/admin.php
error_reporting(E_ALL & ~E_DEPRECATED & ~E_WARNING & ~E_NOTICE & ~E_STRICT);
$data = array();
$cn_data = array();
$url_parts = array();
$url_status = array();
$url_status['black'] = '';
$contents = array();
$links = array();
$wordarray = array();
$topic = '';
$url_reloc = '';
$js_link = '';
$document = '';
$file = '';
$file0 = '';
$raw_file = '';
$seg_data = '';
$index_url = $url;
$comment = $db_con->real_escape_string("Automatically added during index procedure, as this domain is not yet available in 'Sites' menu.");
$admin_email = $db_con->real_escape_string($admin_email);
if ($debug == '0') {
if (function_exists("ini_set")) {
ini_set("display_errors", "0");
}
error_reporting(0);
} else {
error_reporting(E_ERROR);
// otherwise a non existing siemap.xml would always cause a warning message
}
$needsReindex = 1;
$deletable = 0;
$nohost = 1;
$i = 0;
$nohost_count = 5;
// defines count of attempts to get in contact with the server
// check URL status
while ($i < $nohost_count && $nohost) {
$url_status = url_status($url, $site_id, $sessid);
if (!stristr($url_status['state'], "NOHOST")) {
$nohost = '';
// reset for successfull attempt
}
$i++;
}
// check for emergency exit
if ($url_status['aborted'] == '1' || stristr($url_status['state'], "NOHOST")) {
return $url_status;
}
// check for UFO file or invalid suffix
if (stristr($url_status['state'], "ufo")) {
return $url_status;
}
// JFIELD here is right before we try to retrieve the URL and get the error
// echo "<h3>F****E: $url</h3>\n";
// check for 'unreachable' links and if it is a known URL, delete all keyword relationships, former indexed from the meanwhile unreachable link
if (stristr($url_status['state'], "unreachable")) {
printStandardReport('unreachable', $command_line, $no_log);
$sql_query = "SELECT link_id from " . $mysql_table_prefix . "links where url='{$url}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$row = $result->fetch_array(MYSQLI_NUM);
$link_id = $row[0];
if ($link_id) {
$sql_query = "DELETE from " . $mysql_table_prefix . "link_keyword where link_id={$link_id}";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
// here we should delete the keywords associated only to the unreachable link
// but this takes too much time during index procedure
// the admin is asked toc do it manually by using the regarding option in 'Clean' menue
//
// delete the meanwhile unreachable link from db
$sql_query = "DELETE from " . $mysql_table_prefix . "links where link_id = {$link_id}";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
}
return $url_status;
}
// check for overwritten URL, forced by the header, sending content PLUS any redirected URL
if ($url_status['url_over'] && !$url_status['relocate']) {
$url = $url_status['url_over'];
}
$url_parts = parse_all_url($url);
$thislevel = $level - 1;
// redirected URL ?
if ($url_status['relocate']) {
// if relocated, print message, verify the new URL, and redirect to new URL
// check for redirection on an already indexed link
$known_link = '';
$sql_query = "SELECT * from " . $mysql_table_prefix . "links where url='{$url}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$known_link = $result->num_rows;
if ($known_link) {
$urlo_status['state'] = "URL was redirected to an already indexed page.<br />In order to prevent infinite indexation, this is not supported by Sphider-plus.<br />Indexation aborted for this URL";
$url_status['aborted'] = 1;
return $url_status;
}
// remove the original URL from temp table. The relocated URL will be added later on.
mysqltest();
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$url}' AND id = '{$sessid}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$new_url = $url_status['path'];
// URL of first redirection
// remove the redirected URL, which eventually is already stored in db
// before finally storing in db, we need to check for correct redirection.
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$new_url}' AND id = '{$sessid}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
// now special processing for relative links
if (!strpos(substr($new_url, 0, 5), "ttp")) {
$new_url = make_abs($new_url, $index_url);
}
if ($url == $new_url && $url_status['file']) {
$url_status['relocate'] = '';
// remove this redirection, as it is 'in it selves'
$url_status['state'] = "ok";
// try to index the conteent
}
$care_excl = '1';
// care file suffixed to be excluded
$relocated = '1';
// URL is relocated
if ($debug) {
printRedirected($url_status['relocate'], $url_status['path'], $cl);
}
$count = "1";
while ($count <= $redir_count && $url_status['relocate'] && !$url_status['aborted']) {
// check this redirection
$url_status = url_status($new_url, $site_id, $sessid);
if ($url_status['path']) {
$new_url = $url_status['path'];
// URL of another redirections
// now special processing for relative links
if (!strpos(substr($new_url, 0, 5), "ttp")) {
$new_url = make_abs($new_url, $index_url);
}
}
if ($debug) {
printRedirected($url_status['relocate'], $url_status['path'], $cl);
}
$count++;
}
if ($url_status['relocate']) {
$url_status['aborted'] = 1;
$url_status['state'] = "<br />Indexation aborted because of too many redirections.<br />";
return $url_status;
}
if ($url_status['state'] != "ok") {
$code = $url_status['state'];
// check for most common client errors
if (!preg_match("/401|402|403|404/", $code)) {
$url_status['aborted'] = 1;
// end indexing for cmplete site
} else {
$url_status['aborted'] = '';
// abort only for this page
}
if (strstr($code, "401")) {
$code = "401 (Authentication required)";
}
if (strstr($code, "403")) {
$code = "403 (Forbidden)";
}
if (strstr($code, "404")) {
$code = "404 (Not found)";
}
$url_status['state'] = "<br />Indexation aborted because of code: {$code}.<br />";
}
// check final URL (which might be the 3. redirection)
// and puriify final redirected URL
$url = $db_con->real_escape_string(url_purify($new_url, $index_url, $can_leave_domain, $care_excl, $relocated, $local_redir));
// valid file suffix for the redirection??
if ($url) {
if ($care_excl == '1') {
// care about non-accepted suffixes
reset($ext);
while (list($id, $excl) = each($ext)) {
if (preg_match("/\\.{$excl}(\$|\\?)/i", $url)) {
// if suffix is at the end of the link, or followd by a question mark
$url_status['state'] = 'Found: Not supported suffix';
// error message
return $url_status;
}
}
}
}
if (!$url) {
$link_parts = parse_all_url($url);
$host = $link_parts['host'];
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$index_url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$url_status['aborted'] = 1;
$url_status['state'] = "<br />Indexation aborted because of undefined redirection error.<br />";
return $url_status;
}
// abort indexation, if the redirected URL is equal to calling URL
if ($url == 'self') {
$link_parts = parse_all_url($url);
$host = $link_parts['host'];
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$url_status['aborted'] = 1;
$url_status['state'] = "<br />Indexation aborted for this page, because the redirection was a link in it selves.<br />Blocked by Sphider-plus, because this could end in an infinite indexation loop.<br />";
return $url_status;
}
// abort indexation, if the redirected URL contains invalid file suffix
if ($url == 'excl') {
$link_parts = parse_all_url($url);
$host = $link_parts['host'];
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$url_status['aborted'] = 1;
$url_status['state'] = "<br />Indexation aborted because the redirected link does not meet the URL suffix conditions.<br />";
return $url_status;
}
// abort indexation, because purifing the redirected URL failed
if (!strstr($url, "//")) {
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$url_status['aborted'] = 1;
$url_status['state'] = "<br />Indexation aborted because {$url} is not supported.<br />";
return $url_status;
}
// abort indexation, if redirected URL met 'must/must not include' string rule
if (!check_include($url, $url_inc, $url_not_inc)) {
$link_parts = parse_all_url($url);
$host = $link_parts['host'];
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link like '{$url}' AND id = '{$sessid}' OR relo_link like '{$url}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$url_status['aborted'] = 1;
$url_status['state'] = "<br />Indexation aborted because the redirected link does not meet<br />the URL 'must include' or 'must not include' conditions.<br />";
return $url_status;
}
// if redirected URL is already known and in database: abort
$rows0 = '';
$rows1 = '';
mysqltest();
$sql_query = "SELECT url from " . $mysql_table_prefix . "sites where url like '{$url}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$rows0 = $result->num_rows;
$sql_query = "SELECT * from " . $mysql_table_prefix . "links where url='{$url}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$known_link = $result->fetch_array(MYSQLI_NUM);
$md5 = $known_link[8];
if ($clear == 1) {
clean_resource($result, '02');
}
if ($rows0) {
$url_status['state'] = "<br />URL already in database (as a site URL). Index aborted.<br />";
$url_status['aborted'] = 1;
return $url_status;
}
// if known link, which is already indexed (because containing the md5 checksum), enter here
if ($known_link[8]) {
$count = $known_link[15];
$count++;
if ($count > $redir_count) {
// abort indexation
$url_status['state'] = "<br />{$count}. attempt to redirect in the same (already indexed) URL, <br />which is no longer accepted by Sphider-plus. Indexation aborted for this site.<br />";
$url_status['aborted'] = 1;
return $url_status;
} else {
$sql_query = "UPDATE " . $mysql_table_prefix . "links set relo_count='{$count}' where url='{$url}'";
$db_con->query($sql_query);
}
}
// add redirected URL to temp table, if not yet known
$sql_query = "SELECT link from " . $mysql_table_prefix . "temp where link='{$url}' && id = '{$sessid}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$rows = $result->num_rows;
if ($rows == 0) {
$sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id, relo_count) values ('{$url}', '{$level}', '{$sessid}', '1')";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
}
if ($clear == 1) {
clean_resource($result, '02');
}
// at the end of redirect, rebuild the url parts from the redirected URL.
// This is the final URL, which will be indexed
$url_parts = parse_all_url($url);
}
// end check any redirection/relocation
// if a JavaScript file is currently indexed?
$suffix = substr($url, strrpos($url, ".") + 1);
$suffix = str_replace("/", "", $suffix);
if (strlen($suffix) < "5") {
if (preg_match("/js\$/", $suffix)) {
$js_link = 1;
// activate JS switch
}
}
if ($smp != 1 && $follow_sitemap == 1) {
// enter here if we don't already know a valid sitemap and if admin settings allowed us to do so
$tmp_urls = get_temp_urls($sessid);
// reload previous temp
$url2 = remove_sessid(convert_url($url));
// get folder where sitemap should be and if exists, cut existing filename, suffix and subfolder
$host = parse_addr($url2);
$hostname = $host[host];
$more_sitemaps = array();
if ($hostname == 'localhost') {
$host1 = str_replace($local, '', $url2);
}
$pos = strpos($host1, "/");
// on local server delete all behind the /
if ($pos) {
$host1 = substr($host1, 0, $pos);
}
// build full adress again, now only the host
if ($hostname == 'localhost') {
$url2 = "" . $local . "" . $host1 . "";
} else {
$url2 = "{$host['scheme']}://{$hostname}";
}
$sitemap_name = "sitemap";
// standard name for sitemap file
$input_file = "{$url2}/{$sitemap_name}";
// create path to sitemap
$log_file = './sitemaps/current_sitemap.xml';
// destination for sitemap log-file
$smap_found = '';
$indexed_map = '';
$map_cont = '';
// try to fetch individual sitemap url from database
mysqltest();
$sql_query = "SELECT smap_url from " . $mysql_table_prefix . "sites where site_id='{$site_id}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$row = $result->fetch_array(MYSQLI_NUM);
if (preg_match("/http:\\/\\//", $row[0])) {
// use the individual sitemap
$input_file = preg_replace("/.xml.gz|.xml/i", "", $row[0]);
}
$file = "" . $input_file . ".xml";
if ($fd = @fopen($file, "r")) {
// uncompressed ?
//if ($zd = @gzopen("".$input_file.".xml", "r")) { // uncompressed ?
$map_cont = @stream_get_contents($fd);
if ($map_cont && strpos($map_cont, "schemas/sitemap")) {
// if we were able to read it
$smap_found = '1';
}
fclose($fd);
}
$gz_file = "" . $input_file . ".xml.gz";
if (!$smap_found && ($zd = @fopen("compress.zlib://{$gz_file}", "r"))) {
// compressed ?
//if (!$smap_found && $zd = @gzopen("".$input_file.".xml.gz", "r")) { // compressed ?
$map_cont = @gzread($zd, 10485760);
// max. 10 MB (might be too large for some server)
gzclose($zd);
if ($map_cont && strpos($map_cont, "schemas/sitemap")) {
$smap_found = '1';
}
}
//echo "\r\n\r\n<br>map_cont Array:<br><pre>";print_r($map_cont);echo "</pre>\r\n";
if ($smap_found) {
if ($debug != '0') {
// create a log-file of current sitemap.xml
file_put_contents($log_file, $map_cont);
}
//$del = $db_con->query("DELETE from ".$mysql_table_prefix."temp"); // function get_sitemap and store_links will build a new temp table
if (stristr($map_cont, "<sitemapindex")) {
// if current sitemap file is an index file
printStandardReport('validSitemapInd', $command_line, $no_log);
$get_maps = simplexml_load_string($map_cont);
if ($get_maps) {
reset($get_maps);
foreach ($get_maps as $map_x) {
$new_links[] = $map_x->loc;
// get all links to further sitemap files
}
if (is_array($new_links)) {
// if we found more sitemap files
$new_links = explode(",", implode(",", $new_links));
// destroy SimpleXMLElement Object and get the link array
$new_links = array_slice($new_links, 0, $max_links);
$indexed_map = '1';
$i = '0';
//echo "\r\n\r\n<br>new_links Array:<br><pre>";print_r($new_links);echo "</pre>\r\n";
foreach ($new_links as $input_file) {
$these_links = get_sitemap($input_file, $indexed_map, $mysql_table_prefix);
// now extract page links from this sitemap file
//echo "\r\n\r\n<br>these_links Array:<br><pre>";print_r($these_links);echo "</pre>\r\n";
if ($these_links) {
reset($these_links);
store_newLinks($these_links, $level, $sessid);
$smp = '1';
// there were valid sitemap files and we stored the new links
$i++;
} else {
printStandardReport('invalidSecSitemap', $command_line, $no_log);
// unable to extract links from secondary sitemap file
}
}
printValidSecSmap($i, $cl);
unset($input_file, $map_cont, $new_links);
} else {
printStandardReport('invalidSecSitemap', $command_line, $no_log);
// unable to extract links from secondary sitemap file
}
} else {
printStandardReport('invalidSitemapInd', $command_line, $no_log);
// unable to extract links from sitemap INDEX file
}
} else {
$links = get_sitemap($map_cont, $indexed_map, $mysql_table_prefix);
// extract links from sitemap.xml (there was only one sitemap file)
if ($links != '') {
reset($links);
//echo "\r\n\r\n<br>sitemmap links Array:<br><pre>";print_r($links);echo "</pre>\r\n";
store_newLinks($links, $level, $sessid);
$smp = '1';
// there was one valid sitemap and we stored the new links
printStandardReport('validSitemap', $command_line, $no_log);
} else {
printStandardReport('invalidSitemap', $command_line, $no_log);
}
unset($links);
}
}
}
if ($debug == '0') {
if (function_exists("ini_set")) {
ini_set("display_errors", "0");
}
error_reporting(0);
} else {
error_reporting(E_ALL & ~E_DEPRECATED & ~E_WARNING & ~E_NOTICE & ~E_STRICT);
}
if ($url_status['state'] == 'ok') {
$OKtoIndex = 1;
$file_read_error = 0;
if (time() - $delay_time < $min_delay) {
sleep($min_delay - (time() - $delay_time));
}
if ($url_status['file']) {
$file = $url_status['file'];
} else {
$url_status['state'] = "Unable to read the content of the file.<br />{$url} does not deliver any content.";
$realnum--;
}
}
if ($url_status['state'] == 'ok') {
// first attempt to define a charset
$chrSet = '';
if ($use_prefcharset == '1') {
// use preferred charset as defined in Admin settings
$chrSet = $home_charset;
//echo "<h1>USING PREFERRED CHARSET</h1>";
} else {
if ($server_char && $url_status['charset']) {
//echo "<h1>USING SERVER CHARSET</h1>";
$chrSet = $url_status['charset'];
// use charset as supplied by the remote server
} else {
// try to extract the charset of this file
//echo "<h1>USING CONTENT CHARSET</h1>";
//echo "<h1>" . substr($file, 0, 500) . "</h1>";
if (preg_match("'encoding=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
//echo "<h1>1</h1>";
$chrSet = trim(strtoupper($regs[1]));
// get encoding of current XML or XHTML file and use it furtheron
}
if (!$chrSet) {
//echo "<h1>2</h1>";
if (preg_match("'charset=(.*?)[ \\/\\;\\'\"]'si", substr($file, 0, 3000), $regs)) {
//echo "<h1>3</h1>";
$chrSet = trim(strtoupper($regs[1]));
// get charset of current HTML file and use it furtheron
}
}
if (!$chrSet) {
//echo "<h1>4</h1>";
if (preg_match("'charset=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
//echo "<h1>5</h1>";
$chrSet = trim(strtoupper($regs[1]));
// get charset of current HTML file and use it furtheron
}
}
// in assistance for all lazy webmasters
$chrSet = preg_replace("/win-/si", "windows-", $chrSet);
if ($chrSet == "1251") {
//echo "<h1>6</h1>";
$chrSet = "windows-1251";
}
if ($chrSet == '') {
//echo "<h1>7</h1>";
$chrSet = $home_charset;
// no charset found, we need to use default charset like for DOCs, PDFs, etc
}
}
}
//echo "<h1>CHRSET: $chrSet</h1>";
// if required, uncompress ZIP archives and make content of each file => text
if ($url_status['content'] == 'zip' && $index_zip == '1' && $file) {
file_put_contents("" . $tmp_dir . "/archiv.temp", $file);
$zip = zip_open("" . $tmp_dir . "/archiv.temp");
if ($zip) {
$url_status['content'] = "text";
// preventiv, if not another status will be detected for individual archiv files
$file = '';
// starting with a blank file for all archive files
$topic = 'zip';
if ($debug == '2') {
printStandardReport('archivFiles', $command_line, $no_log);
}
while ($zip_entry = zip_read($zip)) {
if (zip_entry_open($zip, $zip_entry, "r")) {
$buf = zip_entry_read($zip_entry, zip_entry_filesize($zip_entry));
//uncompress the content of recent archiv file
$name = zip_entry_name($zip_entry);
// get filename of recent archive file
if ($debug == '2') {
//
$report = "<strong> " . $name . "</strong>";
printThis($report, $cl);
$size = (int) (zip_entry_filesize($zip_entry) / 1024);
if ($size == 0) {
$size = '1';
}
$report = " - Unpacked size: " . $size . " kByte<br />";
printThis($report, $cl);
}
$buf = get_arch_content($buf, $name, $url, $chrSet);
// if necessary, convert PDF, extract feed etc. for the recent file
zip_entry_close($zip_entry);
// done for this file in archiv
$file .= "" . $buf . "<br /><br />";
// add all uncompressed and converted files together
}
}
zip_close($zip);
}
unlink("" . $tmp_dir . "/archiv.temp");
}
// if required, uncompress RAR archives and make content of each file => text
if ($url_status['content'] == 'rar' && $index_rar == '1') {
file_put_contents("" . $tmp_dir . "/archiv.temp", $file);
$rar = rar_open("" . $tmp_dir . "/archiv.temp");
if ($rar) {
$url_status['content'] = "text";
// preventiv, all individual archiv files willl be converted to 'text'
$file = '';
// starting with a blank file for all archive files
$topic = 'rar';
$entries = rar_list($rar);
if ($rar) {
if ($debug == '2') {
printStandardReport('archivFiles', $command_line, $no_log);
}
foreach ($entries as $entry) {
$name = $entry->getName();
if ($debug == '2') {
$report = "<strong> " . $name . "</strong>";
printThis($report, $cl);
$size = (int) ($entry->getPackedSize() / 1024);
if ($size == 0) {
$size = '1';
}
$report = " - Packed size: " . $size . " kByte";
printThis($report, $cl);
$size = (int) ($entry->getUnpackedSize() / 1024);
if ($size == 0) {
$size = '1';
}
$report = " - Unpacked size: " . $size . " kByte<br />";
printThis($report, $cl);
}
$entry->extract('', "./" . $tmp_dir . "/" . $name . "");
// extract single file of archiv into temporary folder
$buf = file_get_contents("./" . $tmp_dir . "/" . $name . "");
// read content of this intermediate file
unlink("./" . $tmp_dir . "/" . $name . "");
// destroy this file
if ($buf) {
$buf = get_arch_content($buf, $name, $url, $chrSet);
// if necessary, convert PDF, extract feed etc. for the recent file
$file .= "" . $buf . "<br /><br />";
// add all uncompressed and converted files together
}
}
}
rar_close($rar);
}
unlink("" . $tmp_dir . "/archiv.temp");
}
$file0 = $file;
// rememberr the original (e.g. for doc2txt converter)
// remove useless part of the content
$file = purify_content($file);
$valid_utf8 = '1';
$raw_file = $file;
// kill eventually duplicate coding info in dynamic links
if (stristr(substr($file, '0', '4000'), "encoding") && strstr(substr($file, '0', '4000'), "charset")) {
$file = substr($file, strrpos($file, "<!DOCTYPE"));
// subsstring starting at last found <!DOCTYPE
}
// we need to do it again for eventually new charset in archive
$chrSet = '';
if ($use_prefcharset == '1') {
// use preferred charset as defined in Admin settings
$chrSet = $home_charset;
} else {
if ($server_char && $url_status['charset']) {
$chrSet = $url_status['charset'];
// use charset as supplied by the remote server
} else {
// try to extract the charset of this file
if (preg_match("'encoding=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
$chrSet = trim(strtoupper($regs[1]));
// get encoding of current XML or XHTML file and use it furtheron
}
if (!$chrSet) {
if (preg_match("'charset=(.*?)[ \\/\\;\\'\"]'si", substr($file, 0, 3000), $regs)) {
$chrSet = trim(strtoupper($regs[1]));
// get charset of current HTML file and use it furtheron
}
}
if (!$chrSet) {
if (preg_match("'charset=[\\'\"](.*?)[\\'\"]'si", substr($file, 0, 3000), $regs)) {
$chrSet = trim(strtoupper($regs[1]));
// get charset of current HTML file and use it furtheron
}
}
// in assistance for all lazy webmasters
$chrSet = preg_replace("/win-/si", "windows-", $chrSet);
if ($chrSet == "1251") {
$chrSet = "windows-1251";
}
if ($chrSet == '') {
$chrSet = $home_charset;
// no charset found, we need to use default charset like for DOCs, PDFs, etc
}
}
}
if (strpos($chrSet, " ")) {
// in the wild we have aloready seen a lot of variants
$chrSet = substr($chrSet, 0, strpos($chrSet, " "));
}
// some webmaster still use 'UNICODE' as name
if (stristr($chrSet, "UNICODE")) {
$chrSet = "UTF-8";
}
// obsolete since 1990, but some (Italian) server still send it as charset . . . .
if (stristr($chrSet, "8858")) {
$chrSet = str_replace("8858", "8859", $chrSet);
}
// required coaching for some webmasters
if (stristr($chrSet, "cp-")) {
$chrSet = str_ireplace("CP-", "CP", $chrSet);
}
$contents['charset'] = $chrSet;
if ($index_framesets == '1') {
if (preg_match("@<frameset[^>]*>(.*?)<\\/frameset>@si", $file, $regs)) {
printStandardReport('newFrameset', $command_line, $no_log);
// separate the <frameset> ....</frameset> part of this file
$frame = $regs[1];
$replace = get_frames($frame, $url, $can_leave_domain);
$replace = "<body>" . $replace . "</body>";
// create the body tags for $file
$contents['charset'] = $chrSet;
// rebuild charset
// include all replacements instead of the frameset tag into the actual file. This will become the body
$file = preg_replace("@<frameset.*?</frameset>@si", "{$replace}", $file);
}
}
if ($index_iframes == '1') {
$links = array();
$regs = array();
$replace = '';
$get_charset = '';
$real_url = $url;
if (preg_match_all("/(iframe[^>]*src[[:blank:]]*)=[[:blank:]]*[\\'\"]?(([[a-z]{3,5}:\\/\\/(([.a-zA-Z0-9-])+(:[0-9]+)*))*([+:%\\/?=&;\\\\(\\),._ a-zA-Z0-9-]*))(#[.a-zA-Z0-9-]*)?[\\'\" ]?/i", $file, $regs, PREG_SET_ORDER)) {
printStandardReport('newIframe', $command_line, $no_log);
// find all frames of the iframe;
$care_excl = '';
// don't care file suffixed to be excluded
$relocated = '';
// URL is not relocated
foreach ($regs as $val) {
if (($a = url_purify($val[2], $url, $can_leave_domain, $care_exel, $relocated, $local_redir)) != '') {
$links[] = $a;
// collect all iframe links
}
}
if ($links) {
foreach ($links as $url) {
printNewLinks($url, $cl);
if (preg_match("/.html|.htm|.xhtml|.xml|.php/i", $url)) {
$frame = file_get_contents($url);
// get content of this frame
// separate the body part of this frame
preg_match("@<body[^>]*>(.*?)<\\/body>@si", $frame, $regs);
$body = $regs[1];
if ($abslinks == '1') {
$body = make_abslinks($body, $url);
// if required, correct links relative to found iframe
}
$replace = "" . $replace . "<br />" . $body . "";
} else {
// might be an image
$replace = "" . $replace . "<br /><img src=\"" . $url . "\">";
}
}
}
// include all replacements instead of the iframe tag into the actual file
$file = preg_replace("@<iframe.*?</iframe>@si", "{$replace}", $file);
$contents['charset'] = $chrSet;
// rebuild charset
}
$url = $real_url;
}
// in order to index RDF, RSD, RSS and ATOM feeds enter here
if ($url_status['content'] == 'xml' && $index_rss == '1') {
if (!preg_match("/<rss|atom|<feed|<rdf|<rsd/si", substr($file, 0, 400))) {
printStandardReport('notRSS', $command_line, $no_log);
// no valid feed detected
$OKtoIndex = 0;
$file_read_error = 1;
$realnum--;
} else {
$html = '';
$xml = XML_IsWellFormed($file);
// check for well-formed XML
if ($xml != '1') {
if ($debug > 0) {
printNotWellFormedXML($xml, $cl);
}
$OKtoIndex = 0;
$file_read_error = 1;
$realnum--;
} else {
$rss = new feedParser();
// define options for feed parser
$rss->limit = $max_links;
// save time by limiting the items/entries to be processed
$rss->in_cp = strtoupper($contents['charset']);
// charset of actual file
$rss->out_cp = 'UTF-8';
// convert all into this charset
$rss->cache_dir = '';
// currently unused
$rss->dc = $dc;
// treat Dublin Core tags in RDF feeds
$rss->pro = $preferred;
// obey the PREFERRED directive in RSD feeds
$rss->file = '1';
// use $file as feed (as a string, not URL)
if ($cdata != 1) {
$rss->CDATA = 'content';
// get it all (naughty)
} else {
$rss->CDATA = 'nochange';
// well educated crawler
}
// get feed as array
if ($feed = $rss->get($url, $file)) {
// if you want to see the feed during index procedure, uncomment the following row
// echo "<br>FEED array:<br><pre>";print_r($feed);echo "</pre>";
$link = '';
$textinput_link = '';
$image_url = '';
$image_link = '';
$docs = '';
$subjects = '';
$count = '';
$type = $feed[type];
$count = $feed[sub_count];
$cached = $feed[cached];
// kill all no longer required values
$feed[type] = '';
$feed[sub_count] = '';
$feed[encoding_in] = '';
$feed[encoding_out] = '';
$feed[items_count] = '';
$feed[cached] = '';
if (!$count) {
$count = '0';
}
if ($type == 'RSD') {
// prepare all RSD APIs
for ($i = 0; $i < $count; $i++) {
$subjects .= '' . $feed['api'][$i]['name'] . '<br />
' . $feed['api'][$i]['apiLink'] . '<br />
' . $feed['api'][$i]['blogID'] . '<br />
' . $feed['api'][$i]['settings_docs'] . '<br />
' . $feed['api'][$i]['settings_notes'] . '<br />';
}
}
if ($type == 'Atom') {
// prepare all Atom entries
for ($i = 0; $i < $count; $i++) {
$subjects .= '' . $feed['entries'][$i]['link'] . '<br />
' . $feed['entries'][$i]['title'] . '<br />
' . $feed['entries'][$i]['id'] . '<br />
' . $feed['entries'][$i]['published'] . '<br />
' . $feed['entries'][$i]['updated'] . '<br />
' . $feed['entries'][$i]['summary'] . '<br />
' . $feed['entries'][$i]['rights'] . '<br />
' . $feed['entries'][$i]['author_name'] . ' ' . $feed['entries'][$i]['author_email'] . ' ' . $feed['entries'][$i]['author_uri'] . '<br />
' . $feed['entries'][$i]['category_term'] . ' ' . $feed['entries'][$i]['category_label'] . ' ' . $feed['entries'][$i]['category_scheme'] . '<br />
' . $feed['entries'][$i]['contributor_name'] . ' ' . $feed['entries'][$i]['contributor_email'] . ' ' . $feed['entries'][$i]['contributor_uri'] . '<br />
';
}
}
if ($type == 'RDF' | $type == 'RSS v.0.91/0.92' | $type == 'RSS v.2.0') {
// For RDF and RSS feeds enter here
// prepare channel image
$image_url = $feed[image_url];
if ($image_url) {
$width = $feed[image_width];
if (!$width || $width > '144') {
$width = '88';
//set to default value
}
$height = $feed[image_height];
if (!$height || $height > '400') {
$height = '31';
//set to default value
}
$feed[image_url] = "<img id=\"rss_007\" src=\"" . $image_url . "\" alt=\"" . $feed[image_title] . "\" width=\"" . $width . "\" height=\"" . $height . "\">";
}
$image_link = $feed[image_link];
if ($image_link) {
$feed[image_link] = "<a href=\"" . $image_link . "\">" . $image_link . "</a>";
}
// prepare all RDF or RSS items
for ($i = 0; $i < $count; $i++) {
$subjects .= '' . $feed['items'][$i]['link'] . '<br />
' . $feed['items'][$i]['title'] . '<br />
' . $feed['items'][$i]['description'] . '<br />
' . $feed['items'][$i]['author'] . '<br />
' . $feed['items'][$i]['category'] . '<br />
' . $feed['items'][$i]['guid'] . '<br />
' . $feed['items'][$i]['comments'] . '<br />
' . $feed['items'][$i]['pubDate'] . '<br />
' . $feed['items'][$i]['source'] . '<br />
' . $feed['items'][$i]['enclosure'] . '<br />
' . $feed['items'][$i]['country'] . '<br />
' . $feed['items'][$i]['coverage'] . '<br />
' . $feed['items'][$i]['contributor'] . '<br />
' . $feed['items'][$i]['date'] . '<br />
' . $feed['items'][$i]['industry'] . '<br />
' . $feed['items'][$i]['language'] . '<br />
' . $feed['items'][$i]['publisher'] . '<br />
' . $feed['items'][$i]['state'] . '<br />
' . $feed['items'][$i]['subject'] . '<br />
';
}
}
// convert the channel/feed part into a string
$feed_common = implode(" ", $feed);
// build something that could be indexed
$html .= "<html>\r\n<head>\r\n<title>" . $feed['title'] . "</title>\r\n<meta name=\"description\" content=\"" . $feed['description'] . " \">\r\n</head>\r\n";
$html .= "<body>\r\n" . $feed_common . "\r\n" . $subjects . "\r\n</body>\r\n</html>\r\n";
}
if (strlen($html) < "130") {
// can't be a valid feed
if ($type == "unknown") {
printInvalidFeedType($type, $cl);
} else {
printStandardReport('invalidRSS', $command_line, $no_log);
}
$OKtoIndex = 0;
$file_read_error = 1;
$realnum--;
} else {
$contents['charset'] = 'UTF-8';
// the feed reader converts all to utf-8
$file = $html;
// use feed reader output
if ($debug > 0) {
printValidFeed($type, $count, $cl);
}
}
}
}
}
// duplicate here, but frames, iframes, or RSS might have added nonsense content
$file = purify_content($file);
// prepare CVS files
if ($url_status['content'] == 'csv' && $index_csv == '1') {
$file = str_replace(",", " ", $file);
$file = str_replace(";", " ", $file);
}
//echo "\r\n\r\n<br>url_status Array:<br><pre>";print_r($url_status);echo "</pre>\r\n";
// for DOCs, PDFs, etc we need special text converter
if ($url_status['content'] != 'text' && $url_status['content'] != 'xml' && $url_status['content'] != 'xhtml' && $url_status['content'] != 'csv') {
$document = 1;
$file = extract_text($file, $file0, $url_status['content'], $url, $chrSet);
// because the converter already transferred the documents to UTF-8, we need to adjust it here
$contents['charset'] = 'UTF-8';
$charSet = 'UTF-8';
if ($file == 'ERROR') {
// if error, suppress further indexing
$OKtoIndex = 0;
$file_read_error = 1;
$realnum--;
}
// reduce Pashtu and Urdu to the main Farsi letters
if (strtolower($charSet) == 'windows-1256' && $url_status['content'] == 'pdf') {
$f_letter0 = array("ﺎ", "�");
$f_letter1 = array("�", "�", "ﺑ", "ﺒ");
$f_letter2 = array("ï–", "ïÂÂÂâ€â€Â", "ïÂÂÂËœ", "ïÂÂÂâ„¢");
$f_letter3 = array("ﺕ", "ﺖ", "ïºâ€â€Â", "ﺘ");
$f_letter4 = array("ﺙ", "ﺚ", "ﺛ", "ﺜ");
$f_letter5 = array("�", "ﺞ", "ﺟ", "ﺠ");
$f_letter6 = array("ïº", "ï»", "ï¼", "ï½");
$f_letter7 = array("ﺡ", "ﺢ", "ﺣ", "ﺤ");
$f_letter8 = array("ﮋ", "ﮊ");
$f_letter9 = array("ﺥ", "ﺦ", "ﺧ", "ﺨ");
$f_letter10 = array("ﺩ", "ﺪ");
$f_letter11 = array("ﺫ", "ﺬ");
$f_letter12 = array("ïºÂÂÂ", "ﺮ");
$f_letter13 = array("ﺯ", "ﺰ");
$f_letter14 = array("ﺱ", "ﺲ", "ﺳ", "ﺴ");
$f_letter15 = array("ﺵ", "ﺶ", "ﺷ", "ﺸ");
$f_letter16 = array("ﺹ", "ﺺ", "ﺻ", "ﺼ");
$f_letter17 = array("ﺽ", "ﺾ", "ﺿ", "ﻀ");
$f_letter18 = array("�", "ﻂ", "ﻃ", "ﻄ");
$f_letter19 = array("ﻅ", "ﻆ", "ﻇ", "ﻈ");
$f_letter20 = array("ﻉ", "ﻊ", "ﻋ", "ﻌ");
$f_letter21 = array("�", "ﻎ", "�", "�");
$f_letter22 = array("ﻑ", "ï»’", "ﻓ", "ï»â€ÂÂ");
$f_letter23 = array("ﻕ", "ï»–", "ï»â€â€Â", "ﻘ");
$f_letter24 = array("ﻙ", "ﻚ", "ﻛ", "ﻜ", "ﮎ", "�", "�", "ﮑ");
$f_letter25 = array("ï®’", "ﮓ", "ï®â€ÂÂ", "ﮕ");
$f_letter26 = array("�", "ﻞ", "ﻟ", "ﻠ");
$f_letter27 = array("ﻡ", "ﻢ", "ﻣ", "ﻤ");
$f_letter28 = array("ﻧ", "ﻨ", "ﻦ", "ﻥ");
$f_letter29 = array("ï»ÂÂÂ", "ï»®");
$f_letter30 = array("ﻩ", "ﻪ", "ﻫ", "ﻬ");
$f_letter31 = array("ﻯ", "ﻰ", "ﻱ", "ﻲ", "ﻳ", "ﻴ");
$file = str_replace($f_letter0, "ا", $file);
$file = str_replace($f_letter1, "ب", $file);
$file = str_replace($f_letter2, "Ù¾", $file);
$file = str_replace($f_letter3, "ت", $file);
$file = str_replace($f_letter4, "Ø«", $file);
$file = str_replace($f_letter5, "ج", $file);
$file = str_replace($f_letter6, "Ú†", $file);
$file = str_replace($f_letter7, "ØÂÂÂ", $file);
$file = str_replace($f_letter8, "Ú˜", $file);
$file = str_replace($f_letter9, "Ø®", $file);
$file = str_replace($f_letter10, "د", $file);
$file = str_replace($f_letter11, "Ø°", $file);
$file = str_replace($f_letter12, "ر", $file);
$file = str_replace($f_letter13, "ز", $file);
$file = str_replace($f_letter14, "س", $file);
$file = str_replace($f_letter15, "Ø´", $file);
$file = str_replace($f_letter16, "ص", $file);
$file = str_replace($f_letter17, "ض", $file);
$file = str_replace($f_letter18, "Ø·", $file);
$file = str_replace($f_letter19, "ظ", $file);
$file = str_replace($f_letter20, "ع", $file);
$file = str_replace($f_letter21, "غ", $file);
$file = str_replace($f_letter22, "Ù�", $file);
$file = str_replace($f_letter23, "Ù‚", $file);
$file = str_replace($f_letter24, "Ú©", $file);
$file = str_replace($f_letter25, "Ú¯", $file);
$file = str_replace($f_letter26, "Ù„", $file);
$file = str_replace($f_letter27, "Ù…", $file);
$file = str_replace($f_letter28, "Ù†", $file);
$file = str_replace($f_letter29, "Ùˆ", $file);
$file = str_replace($f_letter30, "Ù‡", $file);
$file = str_replace($f_letter31, "ÙŠ", $file);
}
}
if ($OKtoIndex == 1) {
$pageSize = number_format(strlen($file) / 1024, 2, ".", "");
printPageSizeReport($pageSize, $topic);
}
$charSet = strtoupper(trim($contents['charset']));
// final charset for UTF-8 converter
if (stristr($charSet, "encoding") || strlen($charSet) < '3') {
// must be invalid encountered charset
$charSet = 'UTF-8';
}
//echo "\r\n\r\n<br /> final charSet: '$charSet'<br />\r\n";
if ($charSet == "UTF-16") {
$charSet = "UTF-8";
// content will be converted in function clean_file()
}
$dic = '';
// if Chinese or Korean text should be segmented enter here
if ($cn_seg == '1' && $file && !$js_link && !stristr($charSet, "8859")) {
if ($charSet == 'GB2312' || $charSet == 'GB18030' || $charSet == 'GBK') {
$dic = "" . $dict_dir . "/cn_gb18030.dic";
// simplified Chinese
}
if ($charSet == 'BIG5') {
$dic = "" . $dict_dir . "/cn_big5.dic";
// traditional Chinese
}
if ($charSet == 'ISO10646-1933') {
$dic = "" . $dict_dir . "/kr_iso10646-1933.dic";
// Korean
}
if ($charSet == 'EUC-KR') {
$dic = "" . $dict_dir . "/kr_euc-kr.dic";
// Korean
}
if ($charSet == 'UTF-8') {
$dic = "" . $dict_dir . "/cn_utf-8.dic";
// Unicode
}
if ($dic) {
// if dictionary is available for page charset, perform a segmentation
$Segmentation = new Segmentation();
$Segmentation->load($dic);
$Segmentation->setLowercase(FALSE);
$cn_result = $Segmentation->segmentString($file);
if ($cn_result && $charSet != 'UTF-8') {
$iconv_file = @iconv($charSet, "UTF-8//IGNORE", $cn_result);
if (trim($iconv_file) == "") {
// iconv is not installed or input charSet not available. We need to use class ConvertCharset
$NewEncoding = new ConvertCharset($charSet, "utf-8");
$NewFileOutput = $NewEncoding->Convert($cn_result);
$cn_result = $NewFileOutput;
} else {
$cn_result = $iconv_file;
}
unset($iconv_file, $NewEncoding, $NewFileOutput);
}
$seg_data = clean_file($cn_result, $url, $url_status['content'], $charSet, $use_nofollow, $use_robot, $can_leave_domain);
} else {
printNoDictionary($charSet, $cl);
// no dictionary found for this charset
}
}
// if Japanese text should be segmented enter here. But not if a Chinese dictonary was already found
if ($jp_seg == '1' && $file && !$js_link && !stristr($charSet, "ISO") && !$dic) {
$dic = '';
if ($charSet == 'UTF-8' || $charSet == 'EUC-JP') {
$file = @iconv($charSet, "SHIFT_JIS//IGNORE", $file);
$charSet = "SHIFT_JIS";
}
if ($charSet == 'SHIFT_JIS') {
$dic = "" . $dict_dir . "/jp_shiftJIS.dic";
}
if ($dic) {
// if dictionary is available for page charset, perform a segmentation
$Segmentation = new Segmentation();
$Segmentation->load($dic);
$Segmentation->setLowercase(FALSE);
$jp_result = $Segmentation->segmentString($file);
//echo "\r\n\r\n<br /> jp_result: $jp_result<br />\r\n";
if ($jp_result && $charSet != 'UTF-8') {
$iconv_file = @iconv($charSet, "UTF-8//IGNORE", $jp_result);
if (trim($iconv_file) == "") {
// iconv is not installed or input charSet not available. We need to use class ConvertCharset
$NewEncoding = new ConvertCharset($charSet, "utf-8");
$NewFileOutput = $NewEncoding->Convert($jp_result);
$jp_result = $NewFileOutput;
} else {
$jp_result = $iconv_file;
}
unset($iconv_file, $NewEncoding, $NewFileOutput);
}
$seg_data = clean_file($jp_result, $url, $url_status['content'], $charSet, $use_nofollow, $use_robot, $can_leave_domain);
} else {
printNoDictionary($charSet, $cl);
// no dictionary found for this charset
}
}
// enter here only, if site / file is not yet UTF-8 coded or had already been converted to UTF-8
if ($charSet != "UTF-8" && $file) {
$file = convertToUTF8($file, $charSet, $char_Set, $converter_dir);
}
// if activated in Admin backend, check for correct converting of $file into UTF-8
if ($utf8_verify) {
$valid_utf8 = @iconv('UTF-8', 'UTF-8', $file) === $file;
}
if (!$valid_utf8) {
$url_status['state'] = "<br />Invalid charset definition placed in meta tags of HTML header. Unable to convert the text into UTF-8<br />Indexing aborted for {$url}";
if ($server_char) {
$url_status['state'] = "<br />Invalid charset definition supplied via HTTP by the client server. Unable to convert the text into UTF-8<br />Indexing aborted for {$url}";
}
if ($use_prefcharset) {
$url_status['state'] = "<br />Invalid charset definition placed Admin Settings.<br />Site was created with another charset<br />Indexing aborted for {$url}";
}
printUrlStatus($url_status['state'], $command_line, $no_log);
$file = '';
$deletable = 1;
} else {
if ($index_media == '1') {
$newmd5sum = md5($file);
// get md5 including links and title of media files
}
$data = clean_file($file, $url, $url_status['content'], $charSet, $use_nofollow, $use_robot, $can_leave_domain);
//echo "\r\n\r\n<br>data Array:<br><pre>";print_r($data);echo "</pre>\r\n";
// index only links and their titles
if ($only_links) {
$media_links = '0';
$my_links = get_link_details($file, $url, $can_leave_domain, $data['base'], $media_links, $use_nofollow, $local_redir);
$data['content'] = $my_links[0][0];
// define new content
$data['fulltext'] = $my_links[0][0];
// define new content also for 'full text';
}
// combine raw words plus segmented words
if ($cn_seg == 1 || $jp_seg == 1 && $dic && !$js_link) {
if ($debug != '0') {
$seg_add = $seg_data[count] - $data[count];
// calculate segmentation result
if ($seg_add > '0') {
if ($charSet == 'EUC-KR' || $charSet == 'ISO10646-1933') {
printSegKR($seg_add, $cl);
}
if ($charSet == 'SHIFT_JIS') {
printSegJA($seg_add, $cl);
} else {
printSegCN($seg_add, $cl);
}
}
/*
echo "<br /><pre>Results of word segmentation:</pre>";
echo "<br />Unsegmented title :<br><pre>";print_r($data[title]);echo "</pre>";
echo "<br />Segmented title :<br><pre>";print_r($seg_data[title]);echo "</pre>";
echo "<br />Unsegmented full text:<br />$data[fulltext]<br />";
echo "<br />Segmented full text:<br />$seg_data[fulltext]";
*/
}
$data[content] = "" . $data[content] . "" . $seg_data[content] . "";
//$data[title] ="".$data[title]."".$seg_data[title]."";
$data[description] = "" . $data[description] . "" . $seg_data[description] . "";
$data[keywords] = "" . $data[keywords] . "" . $seg_data[keywords] . "";
}
// check if canonical redirection was found in page head
$cano_link = '0';
if ($data['cano_link']) {
//echo "\r\n\r\n<br /> url: '$url'<br />\r\n";
$cano_link = $db_con->real_escape_string($data['cano_link']);
//echo "\r\n\r\n<br /> cano_link: '$cano_link'<br />\r\n";
if ($url != $cano_link) {
// only new cano links are accepted
$OKtoIndex = 0;
$deletable = 1;
$realnum--;
if ($cano_link == "1") {
printNoCanonical($cano_link, $cl);
// if unable to extract redirection link
} else {
if ($data['refresh'] == '1') {
printRefreshed($cano_link, $data['wait'], $cl);
// if refresh meta tag was found in HTML head
} else {
printCanonical($cano_link, $cl);
// if canonical link was found in HTML head
}
// do we already know this link in link-table
$sql_query = "SELECT /* jfield 2 */ url from " . $mysql_table_prefix . "links where url like '{$cano_link}'";
$res = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$rows = $res->num_rows;
if ($rows == 0) {
// if not known in link-table, check if already known in temp-table
$sql_query = "SELECT /* jfield 1 */ link from " . $mysql_table_prefix . "temp where link like '{$cano_link}'";
$res = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$rows = $res->num_rows;
if ($rows == 0) {
// not known in link-table, add new link
if ($numoflinks <= $max_links) {
$sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id) values ('{$cano_link}', '{$level}', '{$sessid}')";
$db_con->query($sql_query);
}
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
}
}
}
}
$cano_link = '0';
// reset the cano flag
} else {
if ($index_media == '0') {
$newmd5sum = md5($data['content']);
// get md5 from cleaned full text only
}
if ($md5sum == $newmd5sum) {
printStandardReport('md5notChanged', $command_line, $no_log);
$OKtoIndex = 0;
$realnum--;
} else {
mysqltest();
// check for duplicate page content
$sql_query = "SELECT * from " . $mysql_table_prefix . "links where md5sum='{$newmd5sum}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
if ($num_rows = $result->num_rows) {
// display warning message and urls with duplicate content
printStandardReport('duplicate', $command_line, $no_log);
while ($row = $result->fetch_array(MYSQLI_ASSOC)) {
$dups[] = $row['link_id'];
}
for ($i = 0; $i < $num_rows; $i++) {
$link_id = $dups[$i];
//$num = $i+1;
$sql_query = "SELECT * from " . $mysql_table_prefix . "links where link_id like '{$link_id}'";
$res1 = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$row = $res1->fetch_array(MYSQLI_NUM);
$dup_url = urldecode($row[2]);
$dup_url = $dup_url;
$dup_url = @iconv($charSet, "UTF-8//IGNORE", $dup_url);
if ($idna) {
// Initialize the converter class
$IDN = new idna_convert(array('idn_version' => 2008));
if ($conv_puny && strstr($dup_url, "xn--") && $idna) {
$dup_url = $IDN->decode($dup_url);
}
}
if ($clear == 1) {
clean_resource($res, '03');
}
printDupReport($dup_url, $command_line);
}
if ($dup_content == '0') {
// enter here, if pages with duplicate content should not be indexed/re-indexed
$OKtoIndex = 0;
$realnum--;
} else {
$OKtoIndex = 1;
}
}
}
}
//echo "\r\n\r\n<br>data array1:<br><pre>";print_r($data);echo "</pre>\r\n";
if (($md5sum != $newmd5sum || $reindex == 1) && $OKtoIndex == 1) {
$urlparts = parse_addr($url);
$newdomain = $urlparts['host'];
$type = 0;
if ($data['noindex'] == 1) {
// remember this URlL, so it might not become another time a new link
// check without scheme and www.
$check_link = substr($check_link, stripos($url, "//") + 2);
if (stristr($check_link, "www.")) {
$check_link = substr($check_link, stripos($check_link, "www") + 4);
}
$sql_query = "SELECT url from " . $mysql_table_prefix . "links where url like '%{$check_link}'";
$res = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$known_link = $res->num_rows;
if ($known_link != '1') {
$sql_query = "INSERT into " . $mysql_table_prefix . "links (site_id, url, indexdate, size, md5sum, level) values ('{$site_id}', '{$url}', curdate(), '{$pageSize}', '{$newmd5sum}', '{$thislevel}')";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
}
$OKtoIndex = 0;
$deletable = 1;
$realnum--;
printStandardReport('metaNoindex', $command_line, $no_log);
}
if (!$js_link) {
// JavaScript will not deliver keywords, only links are parsed
$content = explode(" ", addslashes($data['content']));
//echo "\r\n\r\n<br>content array0:<br><pre>";print_r($content);echo "</pre>\r\n";
$acc_words[] = array();
$type = '';
// if Greek accents should be removed from Greek vowels
if ($noacc_el) {
foreach ($content as &$thisword) {
$no_acc = remove_acc_el($thisword);
if ($no_acc != $thisword) {
$acc_words[] = $no_acc;
}
}
}
// if the other (Latin) accents should be removed from their vowels
if ($vowels) {
foreach ($content as $thisword) {
$no_acc = remove_acc($thisword, '');
if ($no_acc != $thisword) {
$acc_words[] = $no_acc;
}
}
}
// now add the words without accents to the total text content
$content = array_merge($content, $acc_words);
//echo "\r\n\r\n<br>content array0:<br><pre>";print_r($content);echo "</pre>\r\n";
// if ligatures should be equalized
if ($liga) {
$liga_words = array();
// will contain converted ligatures
$phon_words = array();
// will contain converted phonetics
// first: convert letters into latin ligatures
foreach ($content as $thisword) {
if ($thisword) {
$liga_words[] = html_entity_decode($thisword, ENT_QUOTES, "UTF-8");
$thisword1 = $thisword;
reset($latin_ligatures);
while ($char = each($latin_ligatures)) {
$thisword2 = preg_replace("/" . $char[0] . "/s", $char[1], $thisword1);
// convert ligatures
if ($thisword1 != $thisword2) {
// break on first ligature
$liga_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
// collect new words with ligatures
$thisword1 = $thisword2;
// continue with the word, containing the ligatures
//break;
}
}
}
}
// second: convert all letters into phonetic transcriptions
reset($liga_words);
foreach ($liga_words as $thisword) {
$thisword1 = $thisword;
reset($phon_trans);
while ($char = each($phon_trans)) {
$thisword2 = preg_replace("/" . $char[0] . "/s", $char[1], $thisword1);
// convert into phonetics
if ($thisword1 != $thisword2) {
// break on first ligature
$phon_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
// collect new words with phonetics
$thisword1 = $thisword2;
// continue with the word, containing the ligatures
//break;
}
}
}
$liga_words = array_merge($liga_words, $phon_words);
// add all phoneticss to the liga array
// now vice versa: convert latin ligatures and phonetic transcriptions into standard letters
reset($content);
$not_liga_words = array();
foreach ($content as $thisword) {
if ($thisword) {
// first: convert latin ligatures into standard letters
$thisword1 = superentities($thisword, ENT_QUOTES, "UTF-8");
reset($latin_ligatures);
while ($char = each($latin_ligatures)) {
$thisword2 = preg_replace("/" . $char[1] . "/s", $char[0], $thisword1);
// re-convert ligatures
if ($thisword1 != $thisword2) {
$not_liga_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
// collect new words without ligatures
$thisword1 = $thisword2;
// continue with the word, containing the ligature
}
}
}
//echo "\r\n\r\n<br>not_liga_words Array:<br><pre>";print_r($not_liga_words);echo "</pre>\r\n";
// second: convert phonetic transcriptions into standard letters
reset($not_liga_words);
$not_phon_words = array();
foreach ($not_liga_words as $thisword) {
$thisword1 = superentities($thisword, ENT_QUOTES, "UTF-8");
reset($phon_trans);
while ($char = each($phon_trans)) {
$thisword2 = preg_replace("/" . $char[1] . "/s", $char[0], $thisword1);
// re-convert sphonetic
if ($thisword1 != $thisword2) {
$not_phon_words[] = html_entity_decode($thisword2, ENT_QUOTES, "UTF-8");
// collect new words without phonetics
$thisword1 = $thisword2;
// continue with the word, containing the phonetic trans.
}
}
}
}
$not_words = array_merge($not_liga_words, $not_phon_words);
// add all together
$content = array_merge($liga_words, $not_words);
// add all ligatures and re-converted letters to the content array
}
$wordarray = unique_array($content);
}
//echo "\r\n\r\n<br>wordarray0:<br><pre>";print_r($wordarray);echo "</pre>\r\n";
if ($smp != 1) {
if ($data['nofollow'] != 1 && $cano_link == '0') {
$media_links = '0';
$links = array();
if (!$document) {
// don't try to find links in PDFs and other pure documents
$links = get_links($file, $url, $can_leave_domain, $data['base'], $media_links, $use_nofollow, $local_redir, $url_reloc, $charSet);
}
if ($links[0]) {
$links = distinct_array($links);
$all_links = count($links);
if ($all_links > $max_links) {
$all_links = $max_links;
}
$links = array_slice($links, 0, $max_links);
if ($realnum < $max_links) {
$numoflinks = 0;
//if there are any new links, add to the temp table, but only if there isn't such url already
if ($links[0]) {
reset($links);
$tmp_urls = get_temp_urls($sessid);
// reload previous temp
// echo "\r\n\r\n<br>tmp_urls array:<br><pre>";print_r($tmp_urls);echo "</pre>\r\n";
if ($debug == '2') {
// if debug mode, show details
printStandardReport('newLinks', $command_line, $no_log);
}
while ($thislink = each($links)) {
// echo "\r\n\r\n<br>thislink array:<br><pre>";print_r($thislink);echo "</pre>\r\n";
// ignore error (message) links and self linking
if (strstr($thislink[1], "//") && $thislink[1] != $url) {
// find new domains for _addurl table
if ($auto_add && $can_leave_domain) {
$all_link = parse_all_url($thislink[1]);
// only the domain will be stored as new URL into addurl table
$dom_link = $all_link['host'];
// reduce to domain name and tld
$new_link = str_replace("www.", "", $dom_link);
// use the complete URL
//$dom_link = $thislink[1];
// use only the domain
$dom_link = $all_link['scheme'] . "://" . $dom_link;
$banned = '';
mysqltest();
// check whether URL is already known in sites table
$sql_query = "SELECT url from " . $mysql_table_prefix . "sites where url like '%{$new_link}%'";
$res1 = $db_con->query($sql_query);
// check whether URL is already known in addurl table
$sql_query = "SELECT url from " . $mysql_table_prefix . "addurl where url like '%{$new_link}%'";
$res2 = $db_con->query($sql_query);
// check whether URL is banned
$sql_query = "SELECT domain from " . $mysql_table_prefix . "banned where domain like '%{$new_link}%'";
$res3 = $db_con->query($sql_query);
if ($res3->num_rows) {
$banned = "1";
}
if ($res1->num_rows == 0 && $res2->num_rows == 0 && $res3->num_rows == 0) {
// add new domain into _addurl table
$sql_query = "INSERT into " . $mysql_table_prefix . "addurl (url, description, account) values ('{$dom_link}', '{$comment}', '{$admin_email}')";
$db_con->query($sql_query);
}
}
// check whether thislink is already known as a link ( might happen by means of relocated URLs)
$res4 = '';
$res5 = '';
$known_link = '';
$known_temp = '';
$check_link = $thislink[1];
// i don't believe the "like" is necessary here and it slows down indexing
// // check without scheme and www.
// $check_link = substr($check_link, stripos($check_link, "//")+2);
// if (stristr($check_link, "www.")) {
// $check_link = substr($check_link, stripos($check_link, "www")+4);
// }
//
// $sql_query = "SELECT /* jfield 3 */ url from ".$mysql_table_prefix."links where url like '%$check_link'";
// $res4 = $db_con->query($sql_query);
//
// $known_link = $res4->num_rows;;
//
// $sql_query = "SELECT /* jfield 4 */ link from ".$mysql_table_prefix."temp where link like '%$check_link'";
// $res5 = $db_con->query($sql_query);
// if ($debug > 0 && $db_con->errno) {
// printf("MySQL failure: %s\n", $db_con->error);
// echo "<br />Script aborted.";
// exit;
// }
// $known_temp = $res5->num_rows;;
$sql_query = "SELECT /* jfield 3 */ url from " . $mysql_table_prefix . "links where url = '{$check_link}'";
$res4 = $db_con->query($sql_query);
$known_link = $res4->num_rows;
$sql_query = "SELECT /* jfield 4 */ link from " . $mysql_table_prefix . "temp where link = '{$check_link}'";
$res5 = $db_con->query($sql_query);
if ($debug > 0 && $db_con->errno) {
printf("MySQL failure: %s\n", $db_con->error);
echo "<br />Script aborted.";
exit;
}
$known_temp = $res5->num_rows;
// if this is a new link not yet known or banned, add this new link to the temp table
if ($tmp_urls[$thislink[1]] != 1 && !$res1 && !$known_link && !$known_temp && !$banned) {
$tmp_urls[$thislink[1]] = 1;
$numoflinks++;
if ($debug == '2') {
$act_link = rawurldecode($thislink[1]);
// make it readable
$act_link = stripslashes($act_link);
printNewLinks($act_link, $cl);
}
mysqltest();
$sql_query = "INSERT into " . $mysql_table_prefix . "temp (link, level, id) values ('{$thislink['1']}', '{$level}', '{$sessid}')";
if ($numoflinks <= $max_links) {
$db_con->query($sql_query);
}
}
}
}
}
}
}
} else {
printStandardReport('noFollow', $command_line, $no_log);
}
unset($file);
}
// JFIELD at this point, the URL in the DB is good
// echo "<h1>DONE</h1>";
// exit;
// if we should index only the files as defined in docs list
if ($only_docs) {
$OKtoIndex = '';
foreach ($docs as $thisdoc) {
if (strstr($urlparts['path'], $thisdoc)) {
$OKtoIndex = "1";
}
}
if (!$OKtoIndex) {
printStandardReport('noDoclist', $command_line, $no_log);
}
}
if ($OKtoIndex == 1) {
if ($link_check == 0) {
$title = $data['title'];
$host = $data['host'];
$path = $data['path'];
$fulltxt = $data['fulltext'];
$desc = substr($data['description'], 0, 1024);
// extract domain
$url_parts = parse_all_url($url);
$hostname = $url_parts[host];
// rebuild domain for localhost applications
if ($hostname == 'localhost') {
$host1 = str_replace($local, '', $url);
}
$pos = strpos($host1, "/");
// on local server delete all behind the /
// will work for localhost URLs like http://localhost/publizieren/japan1/index.htm
// will fail for localhost URLs like http://localhost/publizieren/externe/japan2/index.htm
if ($pos) {
$host1 = substr($host1, 0, $pos);
// build full adress again, now only local domain
}
if ($hostname == 'localhost') {
$domain_for_db = "" . $local . "" . $host1 . "/";
// complete URL
$domain_for_db = str_replace("http://", "", $domain_for_db);
//$domain_for_db = $host1;
} else {
//$domain_for_db = ("$url_parts[scheme]://".$hostname."/"); // complete URL
$domain_for_db = $hostname;
}
if (isset($domain_arr[$domain_for_db])) {
$dom_id = $domain_arr[$domain_for_db];
} else {
mysqltest();
$sql_query = "INSERT into " . $mysql_table_prefix . "domains (domain) values ('{$domain_for_db}')";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$dom_id = $db_con->insert_id;
$domain_arr[$domain_for_db] = $dom_id;
}
if (!$js_link) {
// JavaScript will not deliver keywords, only links are parsed
reset($wordarray);
if ($case_sensitive == '0') {
foreach ($wordarray as &$value) {
$value[1] = lower_ent($value[1]);
$value[1] = lower_case($value[1]);
// convert keywords to lower case
}
}
$wordarray = calc_weights($wordarray, $title, $host, $path, $data['keywords'], $url_parts);
} else {
$wordarray = '';
}
//if there are words to index, add the link to the database, get its id, and add the word + their relation
if (is_array($wordarray) && count($wordarray) >= $min_words_per_page) {
$OKtoSave = 1;
if ($use_white1 == '1') {
// check if content of page matches ANY word in whitelist
$found = '0';
foreach ($whitelist as $key => $val1) {
reset($wordarray);
while ($thisword = each($wordarray)) {
$word = trim($thisword[1][1]);
if (strcasecmp($val1, $word) == 0) {
$found = '1';
}
}
}
if ($found == '0') {
printStandardReport('noWhitelist', $command_line, $no_log);
$OKtoSave = 0;
$realnum--;
}
}
if ($use_white2 == '1') {
// check if content of page matches ALL words in whitelist
$all = count($whitelist);
$found = '0';
$found_this = '0';
foreach ($whitelist as $key => $val2) {
reset($wordarray);
while ($thisword = each($wordarray)) {
$word = trim($thisword[1][1]);
if (strcasecmp($val2, $word) == 0) {
$found_this = '1';
}
}
if ($found_this != '0') {
$found++;
$found_this = '0';
}
}
if ($found != $all) {
printStandardReport('noWhitelist', $command_line, $no_log);
$OKtoSave = 0;
$realnum--;
}
}
if ($use_black == '1') {
$found = '0';
// check if content of page matches ANY string in blacklist
foreach ($blacklist as $key => $val3) {
$met = stripos($data[fulltext], $val3);
if ($met) {
$found = '1';
}
}
if ($found == '1') {
printStandardReport('matchBlacklist', $command_line, $no_log);
$OKtoSave = 0;
$realnum--;
$url_status['black'] = 1;
return $url_status;
}
}
// if activated in Admin backend, create a thumbnail of this URL
if ($OKtoSave && $hostname != 'localhost' && $webshot) {
$shot = '';
// will contain the png webshot
$img = new webshots();
$shot = $img->url_to_image($url);
if ($debug && stristr($shot, "error: #")) {
$shot_warn = "<br />Unable to create the webshot because of " . $shot;
printWarning($shot_warn, $command_line, $no_log);
} else {
$shot = $db_con->real_escape_string($shot);
}
}
if ($md5sum == '' || $md5sum == '' && $url_status['relocate']) {
// enter here for new page (unknown link) OR for new relocated URL(so it will become a new link)
// title, description and fulltxt are already escaped in function clean_file();
$url = $db_con->real_escape_string($url);
// jfield says: messy char decoding earlier
// leaves crap here that fudges up the works
$title_enc = mb_detect_encoding($title);
if (mb_detect_encoding($title) != "UTF-8") {
$title = iconv($title_enc, "UTF-8", $title);
}
$fulltxt = substr($fulltxt, 0, 100000);
// we've got to stop somewhere
$fulltxt_enc = mb_detect_encoding($fulltxt);
if (mb_detect_encoding($title) != "UTF-8") {
$fulltxt = iconv($fulltxt_enc, "UTF-8", $fulltxt);
}
mysqltest();
$sql_query = "INSERT into " . $mysql_table_prefix . "links (site_id, url, title, description, fulltxt, indexdate, size, md5sum, level, webshot) values ('{$site_id}', '{$url}', '{$title}', left('{$desc}', 255), '{$fulltxt}', curdate(), '{$pageSize}', '{$newmd5sum}', '{$thislevel}', '{$shot}')";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
//exit;
// jfield: let's keep going
return;
}
$sql_query = "SELECT link_id from " . $mysql_table_prefix . "links where url='{$url}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$row = $result->fetch_array(MYSQLI_NUM);
$link_id = $row[0];
if ($OKtoSave) {
// store link details, if not yet known (during reindex)
if ($only_links) {
// extract domain of current page delivering the new links
$url_parts = parse_all_url($url);
$hostname = $url_parts[host];
if ($hostname == 'localhost') {
// rebuild domain for localhost applications
$host1 = str_replace($local, '', $url);
}
$pos = strpos($host1, "/");
// on local server delete all behind the /
// will work for localhost URLs like http://localhost/publizieren/japan1/index.htm
// will fail for localhost URLs like http://localhost/publizieren/externe/japan2/index.htm
if ($pos) {
$host1 = substr($host1, 0, $pos);
// build full adress again, now only local domain
}
if ($hostname == 'localhost') {
$domain_db = "" . $local . "" . $host1 . "/";
// complete URL
$domain_db = str_replace("http://", "", $domain_db);
//$domain_db = $host1;
} else {
//$domain_db = ("$url_parts[scheme]://".$hostname."/"); // complete URL
$domain_db = $hostname;
}
// now store all link details into db
foreach ($my_links as $found_link) {
// but only if we have found a title
if ($found_link[3]) {
mysqltest();
// check whether URL is already known in sites table
$sql_query = "SELECT title from " . $mysql_table_prefix . "link_details where link_id like '{$link_id}' and url like '%{$found_link['2']}%'";
$res1 = $db_con->query($sql_query);
if ($res1->num_rows == 0) {
// must be new link
$sql_query = "INSERT into " . $mysql_table_prefix . "link_details (link_id, url, title, indexdate, domain) values ('{$link_id}', '{$found_link['2']}', '{$found_link['3']}', now(), '{$domain_db}')";
$db_con->query($sql_query);
}
}
}
}
if ($debug == '2') {
// if debug mode, show details
printStandardReport('newKeywords', $command_line, $no_log);
}
save_keywords($wordarray, $link_id, $dom_id);
}
mysqltest();
if ($index_media == '1' && $OKtoSave) {
// find media content only if there was no conflict with text (white and/or blacklist)
include "index_media.php";
// try to find media files
}
mysqltest();
if ($debug == '2') {
printStandardReport('indexed1', $command_line, $no_log);
} else {
printStandardReport('indexed', $command_line, $no_log);
}
} else {
if ($md5sum != '' && $md5sum != $newmd5sum && $OKtoSave) {
//if page has changed, start updating
mysqltest();
$sql_query = "SELECT link_id from " . $mysql_table_prefix . "links where url='{$url}'";
$result = $db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
$row = $result->fetch_array(MYSQLI_NUM);
$link_id = $row[0];
$sql_query = "DELETE from " . $mysql_table_prefix . "link_keyword where link_id={$link_id}";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
if ($debug == '2') {
// if debug mode, show details
printStandardReport('newKeywords', $command_line, $no_log);
}
save_keywords($wordarray, $link_id, $dom_id);
$sql_query = "UPDATE " . $mysql_table_prefix . "links set title='{$title}', description ='{$desc}', fulltxt = '{$fulltxt}', indexdate=now(), size = '{$pageSize}', md5sum='{$newmd5sum}', level='{$thislevel}', webshot='{$shot}' where link_id='{$link_id}'";
mysqltest();
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
if ($index_media == '1') {
include "index_media.php";
// try to find media files
}
if ($debug == '2') {
printStandardReport('re-indexed1', $command_line, $no_log);
}
}
}
} else {
if ($js_link) {
printStandardReport('js_content', $command_line, $no_log);
} else {
printStandardReport('minWords', $command_line, $no_log);
}
$realnum--;
}
} else {
printStandardReport('link_okay', $command_line, $no_log);
}
unset($file, $title, $fulltxt, $desc);
$wordarray = array();
$data = array();
$seg_data = array();
}
}
}
} else {
$deletable = 1;
//printUrlStatus($url_status['state'], $command_line, $no_log);
}
mysqltest();
if ($url_status['relocate']) {
// remove this relocated URL from temp table, because it is indexed now
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$url}' AND id = '{$sessid}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
}
if ($reindex == 1 && $deletable == 1) {
check_for_removal($url);
} else {
if ($reindex == 1) {
}
}
if (!isset($all_links)) {
$all_links = 0;
}
if (!isset($numoflinks)) {
$numoflinks = 0;
}
// if valid sitemap found, or canonical link, or something else, no LinkReport
if ($smp != 1 && $OKtoIndex == 1 && $url_status['state'] == 'ok') {
printLinksReport($numoflinks, $all_links, $command_line);
}
// remove the URL, which haas been idexed now from temp table.
mysqltest();
$sql_query = "DELETE from " . $mysql_table_prefix . "temp where link = '{$url}' AND id = '{$sessid}'";
$db_con->query($sql_query);
if ($debug && $db_con->errno) {
$err_row = __LINE__ - 2;
printf("<p><span class='red'> MySQL failure: %s \n<br /></span></p>", $db_con->error);
if (__FUNCTION__) {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} in function(): " . __FUNCTION__ . " <br /></span></p>");
} else {
printf("<p><span class='red'> Found in script: " . __FILE__ . " row: {$err_row} <br /></span></p>");
}
printf("<p><span class='red'> Script execution aborted. <br /></span>");
printf("<p><strong>Invalid query string, which caused the SQL error:</strong></p>");
echo "<p> {$sql_query} </p>";
exit;
}
return $url_status;
}
/**
*/
function rar_extract($archive_name, $EXTRACT_PATH)
{
$rar = rar_open($archive_name);
$list = rar_list($rar);
foreach ($list as $file) {
$file = explode("\"", $file);
$entry = rar_entry_get($rar, $file[1]);
$entry->extract($EXTRACT_PATH);
}
rar_close($rar);
}
$file = $dir . DIRECTORY_SEPARATOR . $obj;
if (time() - filectime($file) > $secondsOld) {
@unlink($file);
}
}
closedir($dh);
}
CleanCacheDir($options['cachepath'], 300);
$mimetype = minimime($path);
if ($mimetype == 'application/x-rar-compressed') {
$filename = rawurldecode(stripslashes($_GET['page']));
$rar_file = rar_open($path);
$cachepathname = tempnam($options['cachepath'], "cbr");
$entry = rar_entry_get($rar_file, $filename);
$entry->extract(false, $cachepathname);
rar_close($rar_file);
$size = resize($cachepathname, intval($_GET['maxwidth']));
echo '{' . "\n\r";
echo ' "width" : "' . $size[0] . '",' . "\n\r";
echo ' "height" : "' . $size[1] . '",' . "\n\r";
echo ' "src" : "' . $options['webcache'] . '/' . basename($cachepathname) . '"' . "\n\r";
echo '}';
} elseif ($mimetype == 'application/zip') {
$index = 0;
if (isset($_GET['page'])) {
$index = intval($_GET['page']);
}
$zip = new ZipArchive();
if ($zip->open($path) === TRUE) {
$entry = $zip->statIndex($index);
$cachepathname = tempnam($options['cachepath'], "cbz");
public function extract($id, $to = false)
{
if (empty($this->toc['files'][$id])) {
if (empty($this->toc['dirs'][$id])) {
trigger_error("'{$id}' is not a valid entry.", E_USER_ERROR);
} else {
trigger_error("'{$id}' is not a file.", E_USER_ERROR);
}
}
if (!$to) {
trigger_error('Rar entry must be saved to file', E_USER_WARNING);
return false;
}
if ($fp = rar_open($this->filename)) {
$file = $this->toc['files'][$id];
$entry = rar_entry_get($fp, $file['filename']);
$file['tmp_name'] = $to . '/' . md5($file['filename'] . $file['crc']);
$entry->extract(false, $file['tmp_name']);
/*
$host_os = $entry->getHostOs();
$attr = $entry->getAttr();
switch ($host_os) {
case RAR_HOST_MSDOS:
case RAR_HOST_OS2:
case RAR_HOST_WIN32:
case RAR_HOST_MACOS:
printf("%c%c%c%c%c%c\n",
($attr & 0x08) ? 'V' : '.',
($attr & 0x10) ? 'D' : '.',
($attr & 0x01) ? 'R' : '.',
($attr & 0x02) ? 'H' : '.',
($attr & 0x04) ? 'S' : '.',
($attr & 0x20) ? 'A' : '.');
break;
case RAR_HOST_UNIX:
case RAR_HOST_BEOS:
switch ($attr & 0xF000)
{
case 0x4000:
printf("d");
break;
case 0xA000:
printf("l");
break;
default:
printf("-");
break;
}
printf("%c%c%c%c%c%c%c%c%c\n",
($attr & 0x0100) ? 'r' : '-',
($attr & 0x0080) ? 'w' : '-',
($attr & 0x0040) ? (($attr & 0x0800) ? 's':'x'):(($attr & 0x0800) ? 'S':'-'),
($attr & 0x0020) ? 'r' : '-',
($attr & 0x0010) ? 'w' : '-',
($attr & 0x0008) ? (($attr & 0x0400) ? 's':'x'):(($attr & 0x0400) ? 'S':'-'),
($attr & 0x0004) ? 'r' : '-',
($attr & 0x0002) ? 'w' : '-',
($attr & 0x0001) ? 'x' : '-');
break;
}
*/
rar_close($fp);
return $file;
}
return false;
}
/**
* Decompresses the given content
*
* @param string $content
* @return boolean
*/
public function decompress($content)
{
$archive = $this->getArchive();
if (file_exists($content)) {
$archive = str_replace(array('/', '\\'), DIRECTORY_SEPARATOR, realpath($content));
} elseif (empty($archive) || !file_exists($archive)) {
#require_once 'Zend/Filter/Exception.php';
throw new Zend_Filter_Exception('RAR Archive not found');
}
$password = $this->getPassword();
if ($password !== null) {
$archive = rar_open($archive, $password);
} else {
$archive = rar_open($archive);
}
if (!$archive) {
#require_once 'Zend/Filter/Exception.php';
throw new Zend_Filter_Exception("Error opening the RAR Archive");
}
$target = $this->getTarget();
if (!is_dir($target)) {
$target = dirname($target);
}
$filelist = rar_list($archive);
if (!$filelist) {
#require_once 'Zend/Filter/Exception.php';
throw new Zend_Filter_Exception("Error reading the RAR Archive");
}
foreach ($filelist as $file) {
$file->extract($target);
}
rar_close($archive);
return true;
}
public function getRAROriginalSize($filename)
{
$size = 0;
$resource = rar_open($filename);
if ($resource === false) {
return $size;
}
$entries = rar_list($resource);
foreach ($entries as $entry) {
//we'll exclude empty directories
if (!$entry->isDirectory()) {
$size += $entry->getUnpackedSize();
}
}
rar_close($resource);
return $size;
}
/**
* Unrar file to destination folder
* @return destination folder
*/
private function unrar()
{
try {
$rar = rar_open($this->file['tmp_name']);
$list = rar_list($rar);
foreach ($list as $file) {
$file->extract(DESTINATION . '/' . $this->file['name'] . '/');
}
rar_close($rar);
unlink($this->file['tmp_name']);
return $this->file['name'];
} catch (Exception $e) {
return FALSE;
}
}
public function extract($source, $target, $password)
{
$rarFile = rar_open(suffix($source, '.rar'), $password);
$list = rar_list($rarFile);
if (!empty($list)) {
foreach ($list as $file) {
$entry = rar_entry_get($rarFile, $file);
$entry->extract($target);
}
} else {
throw new InvalidArgumentException('Error', 'emptyVariable', '$list');
}
rar_close($rarFile);
}
public function extract($source = '', $target = '.', $password = NULL)
{
if (!is_file($source)) {
return Error::set('Error', 'fileParameter', '1.(source)');
}
$rarFile = rar_open($source, $password);
$list = rar_list($rarFile);
if (!empty($list)) {
foreach ($list as $file) {
$entry = rar_entry_get($rarFile, $file);
$entry->extract($target);
}
} else {
return Error::set('Error', 'emptyVariable', '$list');
}
rar_close($rarFile);
}