function GetIndex()
{
$WORKING_DIR = "/home/working_toulouse_databases";
$WORKING_DOWNLOAD = "{$WORKING_DIR}/dowloads";
$WORKING_UPLOAD = "{$WORKING_DIR}/uploads";
@mkdir($WORKING_DOWNLOAD, 0755, true);
@mkdir($WORKING_UPLOAD, 0755, true);
$unix = new unix();
$tar = $unix->find_program("tar");
$catz = new mysql_catz();
$tmpfile = $unix->FILE_TEMP();
$tmpdir = $unix->TEMP_DIR();
$rm = $unix->find_program("rm");
$mainuri = "ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib";
$EXEC_NICE = $unix->EXEC_NICE();
$ufdbGenTable = $unix->find_program("ufdbGenTable");
$curl = new ccurl("{$mainuri}/MD5SUM.LST");
if (!$curl->GetFile($tmpfile)) {
echo "Failed to download MD5SUM.LST\n";
return;
}
$tr = explode("\n", @file_get_contents($tmpfile));
while (list($index, $line) = each($tr)) {
if (preg_match("#(.+?)\\s+(.+)#", $line, $re)) {
$filename = trim($re[1]);
$md5 = trim($re[2]);
$array[$md5] = $filename;
}
}
@unlink($tmpfile);
$q = new mysql_squid_builder();
$TLSE_CONVERTION = TLSE_CONVERTION();
$FINAL_ARRAY = array();
while (list($filename, $md5) = each($array)) {
$category = str_replace(".tar.gz", "", $filename);
if (isset($TLSE_CONVERTION[$category])) {
$FINAL_ARRAY[$filename] = $md5;
}
}
$UPDATED = 0;
$MAIN_ARRAY = unserialize(base64_decode(@file_get_contents("{$WORKING_UPLOAD}/index.txt")));
while (list($filename, $md5) = each($FINAL_ARRAY)) {
$TargetFile = "{$WORKING_DOWNLOAD}/{$filename}";
$categoryname = str_replace(".tar.gz", "", $filename);
if ($categoryname == "adult") {
continue;
}
if ($categoryname == "aggressive") {
continue;
}
if ($categoryname == "agressif") {
continue;
}
if ($categoryname == "redirector") {
continue;
}
if ($categoryname == "ads") {
continue;
}
if ($categoryname == "drogue") {
continue;
}
$MyStoreMd5 = md5_file($TargetFile);
if ($MyStoreMd5 == $md5) {
echo "Skipping {$filename}\n";
continue;
}
echo "Downloading {$filename}\n";
$curl = new ccurl("{$mainuri}/{$filename}");
$tmpfile = $unix->FILE_TEMP();
if (!$curl->GetFile($tmpfile)) {
echo "Failed {$curl->error}\n";
@unlink($tmpfile);
continue;
}
$md5_tmp = md5_file($tmpfile);
if ($md5_tmp != $md5) {
echo "Failed Corrupted file\n";
@unlink($tmpfile);
continue;
}
if (is_file($TargetFile)) {
@unlink($TargetFile);
}
if (!@copy($tmpfile, $TargetFile)) {
echo "Failed Copy file\n";
@unlink($tmpfile);
@unlink($TargetFile);
continue;
}
@unlink($tmpfile);
$MyStoreMd5 = md5_file($TargetFile);
if ($MyStoreMd5 != $md5) {
echo "Failed MD5 file\n";
@unlink($TargetFile);
continue;
}
@mkdir("{$WORKING_DIR}/{$categoryname}", 0755, true);
echo "Extracting {$TargetFile}\n";
$cmd = "{$tar} xvf {$TargetFile} -C {$WORKING_DIR}/{$categoryname}/";
echo $cmd . "\n";
system($cmd);
$SOURCE_DIR = find_sources("{$WORKING_DIR}/{$categoryname}");
if (!is_file("{$SOURCE_DIR}/domains")) {
echo "Failed {$SOURCE_DIR}/domains no such file\n";
@unlink($TargetFile);
continue;
}
$COUNT_OF_DOMAINS = $unix->COUNT_LINES_OF_FILE("{$SOURCE_DIR}/domains");
echo "{$categoryname} {$COUNT_OF_DOMAINS} domains\n";
if ($COUNT_OF_DOMAINS == 0) {
shell_exec("{$rm} -rf {$WORKING_DIR}/{$categoryname}");
@unlink($TargetFile);
continue;
}
if (is_file("{$SOURCE_DIR}/domains.ufdb")) {
@unlink("{$SOURCE_DIR}/domains.ufdb");
}
if (!is_file("{$SOURCE_DIR}/urls")) {
@touch("{$SOURCE_DIR}/urls");
}
$u = " -u {$SOURCE_DIR}/urls";
$d = " -d {$SOURCE_DIR}/domains";
$cmd = "{$EXEC_NICE}{$ufdbGenTable} -n -q -W -t {$categoryname}{$d}{$u}";
echo $cmd . "\n";
shell_exec($cmd);
if (!is_file("{$SOURCE_DIR}/domains.ufdb")) {
echo "Failed to compile {$categoryname}\n";
@unlink($TargetFile);
continue;
}
$MD5SRC = md5_file("{$SOURCE_DIR}/domains.ufdb");
if (is_file("{$WORKING_UPLOAD}/{$categoryname}.gz")) {
@unlink("{$WORKING_UPLOAD}/{$categoryname}.gz");
}
$unix->compress("{$SOURCE_DIR}/domains.ufdb", "{$WORKING_UPLOAD}/{$categoryname}.gz");
$MD5GZ = md5_file("{$WORKING_UPLOAD}/{$categoryname}.gz");
$UPDATED++;
$NOTIFICATIONS[] = "{$categoryname} updated with {$COUNT_OF_DOMAINS} domains";
$MAIN_ARRAY[$categoryname]["ROWS"] = $COUNT_OF_DOMAINS;
$MAIN_ARRAY[$categoryname]["MD5SRC"] = $MD5SRC;
$MAIN_ARRAY[$categoryname]["MD5GZ"] = $MD5GZ;
$MAIN_ARRAY[$categoryname]["TIME"] = time();
$MAIN_ARRAY[$categoryname]["SIZE"] = @filesize("{$WORKING_UPLOAD}/{$categoryname}.gz");
@file_put_contents("{$WORKING_UPLOAD}/index.txt", base64_encode(serialize($MAIN_ARRAY)));
}
if ($UPDATED > 0) {
PushToRepo_alls();
sendEmail("{$UPDATED} Toulouse Unversity databases uploaded.", @implode("\n", $NOTIFICATIONS));
}
}
if ($compiler === false) {
error("ERR003", "No suitable compiler found for language " . $task['language']);
}
$debugger = find_best_debugger($task['language']);
$profiler = find_best_profiler($task['language']);
// Add tool versions to buildhost description
$buildhost_description['compiler_version'] = $compiler['version'];
if ($debugger) {
$buildhost_description['debugger_version'] = $debugger['version'];
}
if ($profiler) {
$buildhost_description['profiler_version'] = $profiler['version'];
}
update_status("STA001", "Starting up...");
// Begin process
$filelist = find_sources($task, $instance);
if ($filelist == array()) {
error("ERR004", "No sources found");
}
$exe_file = instance_path($instance) . "/bs_exec_{$program_id}";
$debug_exe_file = $exe_file . "_debug";
// Compile
if ($task['compile'] === "true") {
$compile_result = do_compile($filelist, $exe_file, $compiler, $task['compiler_options'], $instance);
if ($compile_result['status'] == COMPILE_SUCCESS) {
update_status("STA002", "Compile succeeded");
} else {
update_status("STA003", "Compile failed");
done();
}
} else {
