include_once '../common/sql/SqlStatement.php';
include_once '../common/tool/FetchFopen.php';
$conf = new Config('osint');
$helper = new Common('osint', $conf->global_path);
$fetch = new FetchFopen();
$dbh = new SQL_Class("aifs");
$sql = $dbh->execute("SELECT id, url FROM osint_url WHERE dead_link = 0 ORDER BY rand() LIMIT 1");
list($uid, $url) = $sql->fetch_array();
// fetch lastest version
$sql = $dbh->execute("SELECT size, date\r\n FROM osint_version\r\n WHERE fk_osint_url_id=" . $uid . " ORDER BY date desc LIMIT 1");
list($size, $date) = $sql->fetch_array();
if (strpos($date, date("Y") . '-' . date("m") . '-' . date("d")) !== false) {
die;
}
// fetch online copy
$content = $fetch->getContent($url);
// compare and save
if ($size != strlen(addslashes($content))) {
$dbh->execute("INSERT INTO osint_version SET fk_osint_url_id = '" . $uid . "',\r\n content ='" . addslashes($content) . "',\r\n size ='" . strlen(addslashes($content)) . "',\r\n date = NOW()");
$sql = $dbh->execute("SELECT tag FROM osint_tags WHERE fk_osint_url_id='" . $uid . "' LIMIT 1");
list($tag) = $sql->fetch_array();
if (strlen($tag) > 1) {
$sql = $dbh->execute("SELECT COUNT(*) FROM osint_tags_changelog WHERE tag='" . addslashes($tag) . "'");
if ($sql->sql_result() == 0) {
$dbh->execute("INSERT INTO osint_tags_changelog SET tag='" . addslashes($tag) . "', date=NOW()");
} else {
$dbh->execute("UPDATE osint_tags_changelog SET date=NOW() WHERE tag='" . addslashes($tag) . "'");
}
}
$sql = $dbh->execute("SELECT date FROM osint_version ORDER BY id DESC LIMIT 1");
list($date) = $sql->fetch_array();
error_reporting($conf->debug);
ini_set('error_reporting', $conf->debug);
$sql = $dbh->execute("SELECT url FROM url_tld_com \n WHERE osint_submit=0 AND subchance = 0 ORDER by id DESC LIMIT 1");
list($myurl) = $sql->fetch_array();
$sql = $dbh->execute('SELECT * FROM dnint_url ORDER BY id DESC LIMIT 1');
list($id, $url, $time) = $sql->fetch_array();
if ($myurl != $url) {
$sql = $dbh->execute("INSERT INTO dnint_url SET url='" . $myurl . "'");
}
$sql = $dbh->execute('SELECT * FROM dnint_url ORDER BY id DESC LIMIT 1');
list($id, $url, $time) = $sql->fetch_array();
$sql = $dbh->execute("SELECT COUNT(*) FROM dnint_url_contents WHERE fk_dnint_url_id=" . $id);
if ($sql->sql_result() != 0) {
die;
}
$content = $fetch->getContent($url, "", "", false);
if ($content == '') {
// There was a problem getting the content of this URL
$dbh->execute("DELETE FROM dnint_url WHERE id=" . $id);
die;
}
// Content was obtained so update or add in mysql
$query = 'SELECT * from dnint_url_contents WHERE url="' . addslashes($url) . '"';
$stm2 = $dbh->execute($query);
if ($stmt->num_row() == 0) {
//this URL is new and its contents are not fetched yet
$query = "INSERT INTO dnint_url_contents \n SET url='" . addslashes($url) . "', \n url_content='" . addslashes($content) . "', \n fk_dnint_url_id=" . $id;
$dbh->execute($query);
} else {
//this URL exists so just overwrite its contents (and also its FK to make sure we are in sync)
$query = "UPDATE dnint_url_contents \n SET url_content='" . addslashes($content) . "', \n fk_dnint_url_id=" . addslashes($id) . " \n WHERE url='" . $url . "'";