function GetIndex() { $WORKING_DIR = "/home/working_toulouse_databases"; $WORKING_DOWNLOAD = "{$WORKING_DIR}/dowloads"; $WORKING_UPLOAD = "{$WORKING_DIR}/uploads"; @mkdir($WORKING_DOWNLOAD, 0755, true); @mkdir($WORKING_UPLOAD, 0755, true); $unix = new unix(); $tar = $unix->find_program("tar"); $catz = new mysql_catz(); $tmpfile = $unix->FILE_TEMP(); $tmpdir = $unix->TEMP_DIR(); $rm = $unix->find_program("rm"); $mainuri = "ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib"; $EXEC_NICE = $unix->EXEC_NICE(); $ufdbGenTable = $unix->find_program("ufdbGenTable"); $curl = new ccurl("{$mainuri}/MD5SUM.LST"); if (!$curl->GetFile($tmpfile)) { echo "Failed to download MD5SUM.LST\n"; return; } $tr = explode("\n", @file_get_contents($tmpfile)); while (list($index, $line) = each($tr)) { if (preg_match("#(.+?)\\s+(.+)#", $line, $re)) { $filename = trim($re[1]); $md5 = trim($re[2]); $array[$md5] = $filename; } } @unlink($tmpfile); $q = new mysql_squid_builder(); $TLSE_CONVERTION = TLSE_CONVERTION(); $FINAL_ARRAY = array(); while (list($filename, $md5) = each($array)) { $category = str_replace(".tar.gz", "", $filename); if (isset($TLSE_CONVERTION[$category])) { $FINAL_ARRAY[$filename] = $md5; } } $UPDATED = 0; $MAIN_ARRAY = unserialize(base64_decode(@file_get_contents("{$WORKING_UPLOAD}/index.txt"))); while (list($filename, $md5) = each($FINAL_ARRAY)) { $TargetFile = "{$WORKING_DOWNLOAD}/{$filename}"; $categoryname = str_replace(".tar.gz", "", $filename); if ($categoryname == "adult") { continue; } if ($categoryname == "aggressive") { continue; } if ($categoryname == "agressif") { continue; } if ($categoryname == "redirector") { continue; } if ($categoryname == "ads") { continue; } if ($categoryname == "drogue") { continue; } $MyStoreMd5 = md5_file($TargetFile); if ($MyStoreMd5 == $md5) { echo "Skipping {$filename}\n"; continue; } echo "Downloading {$filename}\n"; $curl = new ccurl("{$mainuri}/{$filename}"); $tmpfile = $unix->FILE_TEMP(); if (!$curl->GetFile($tmpfile)) { echo "Failed {$curl->error}\n"; @unlink($tmpfile); continue; } $md5_tmp = md5_file($tmpfile); if ($md5_tmp != $md5) { echo "Failed Corrupted file\n"; @unlink($tmpfile); continue; } if (is_file($TargetFile)) { @unlink($TargetFile); } if (!@copy($tmpfile, $TargetFile)) { echo "Failed Copy file\n"; @unlink($tmpfile); @unlink($TargetFile); continue; } @unlink($tmpfile); $MyStoreMd5 = md5_file($TargetFile); if ($MyStoreMd5 != $md5) { echo "Failed MD5 file\n"; @unlink($TargetFile); continue; } @mkdir("{$WORKING_DIR}/{$categoryname}", 0755, true); echo "Extracting {$TargetFile}\n"; $cmd = "{$tar} xvf {$TargetFile} -C {$WORKING_DIR}/{$categoryname}/"; echo $cmd . "\n"; system($cmd); $SOURCE_DIR = find_sources("{$WORKING_DIR}/{$categoryname}"); if (!is_file("{$SOURCE_DIR}/domains")) { echo "Failed {$SOURCE_DIR}/domains no such file\n"; @unlink($TargetFile); continue; } $COUNT_OF_DOMAINS = $unix->COUNT_LINES_OF_FILE("{$SOURCE_DIR}/domains"); echo "{$categoryname} {$COUNT_OF_DOMAINS} domains\n"; if ($COUNT_OF_DOMAINS == 0) { shell_exec("{$rm} -rf {$WORKING_DIR}/{$categoryname}"); @unlink($TargetFile); continue; } if (is_file("{$SOURCE_DIR}/domains.ufdb")) { @unlink("{$SOURCE_DIR}/domains.ufdb"); } if (!is_file("{$SOURCE_DIR}/urls")) { @touch("{$SOURCE_DIR}/urls"); } $u = " -u {$SOURCE_DIR}/urls"; $d = " -d {$SOURCE_DIR}/domains"; $cmd = "{$EXEC_NICE}{$ufdbGenTable} -n -q -W -t {$categoryname}{$d}{$u}"; echo $cmd . "\n"; shell_exec($cmd); if (!is_file("{$SOURCE_DIR}/domains.ufdb")) { echo "Failed to compile {$categoryname}\n"; @unlink($TargetFile); continue; } $MD5SRC = md5_file("{$SOURCE_DIR}/domains.ufdb"); if (is_file("{$WORKING_UPLOAD}/{$categoryname}.gz")) { @unlink("{$WORKING_UPLOAD}/{$categoryname}.gz"); } $unix->compress("{$SOURCE_DIR}/domains.ufdb", "{$WORKING_UPLOAD}/{$categoryname}.gz"); $MD5GZ = md5_file("{$WORKING_UPLOAD}/{$categoryname}.gz"); $UPDATED++; $NOTIFICATIONS[] = "{$categoryname} updated with {$COUNT_OF_DOMAINS} domains"; $MAIN_ARRAY[$categoryname]["ROWS"] = $COUNT_OF_DOMAINS; $MAIN_ARRAY[$categoryname]["MD5SRC"] = $MD5SRC; $MAIN_ARRAY[$categoryname]["MD5GZ"] = $MD5GZ; $MAIN_ARRAY[$categoryname]["TIME"] = time(); $MAIN_ARRAY[$categoryname]["SIZE"] = @filesize("{$WORKING_UPLOAD}/{$categoryname}.gz"); @file_put_contents("{$WORKING_UPLOAD}/index.txt", base64_encode(serialize($MAIN_ARRAY))); } if ($UPDATED > 0) { PushToRepo_alls(); sendEmail("{$UPDATED} Toulouse Unversity databases uploaded.", @implode("\n", $NOTIFICATIONS)); } }
if ($compiler === false) { error("ERR003", "No suitable compiler found for language " . $task['language']); } $debugger = find_best_debugger($task['language']); $profiler = find_best_profiler($task['language']); // Add tool versions to buildhost description $buildhost_description['compiler_version'] = $compiler['version']; if ($debugger) { $buildhost_description['debugger_version'] = $debugger['version']; } if ($profiler) { $buildhost_description['profiler_version'] = $profiler['version']; } update_status("STA001", "Starting up..."); // Begin process $filelist = find_sources($task, $instance); if ($filelist == array()) { error("ERR004", "No sources found"); } $exe_file = instance_path($instance) . "/bs_exec_{$program_id}"; $debug_exe_file = $exe_file . "_debug"; // Compile if ($task['compile'] === "true") { $compile_result = do_compile($filelist, $exe_file, $compiler, $task['compiler_options'], $instance); if ($compile_result['status'] == COMPILE_SUCCESS) { update_status("STA002", "Compile succeeded"); } else { update_status("STA003", "Compile failed"); done(); } } else {