function DownloadByRange($URI, $EXPECTED_SIZE = 0, $LocalFile = null)
{
$HTTP_CODE = 0;
$Time = null;
if (!isset($GLOBALS["WindowsUpdateCachingDir"])) {
$GLOBALS["WindowsUpdateCachingDir"] = @file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateCachingDir");
if ($GLOBALS["WindowsUpdateCachingDir"] == null) {
$GLOBALS["WindowsUpdateCachingDir"] = "/home/squid/WindowsUpdate";
}
}
$WindowsUpdateBandwidthPartial = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateBandwidthPartial"));
$WindowsUpdateUseLocalProxy = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateUseLocalProxy"));
$WindowsUpdateInterface = @file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateInterface");
if ($WindowsUpdateBandwidthPartial == 0) {
$WindowsUpdateBandwidthPartial = 512;
}
if ($EXPECTED_SIZE == 0) {
$EXPECTED_SIZE = GetTargetedSize($URI);
}
$EXPECTED_SIZE_TEXT = xFormatBytes($EXPECTED_SIZE / 1024);
$BaseNameOfFile = basename($URI);
$unix = new unix();
$curl = $unix->find_program("curl");
$rm = $unix->find_program("rm");
$mv = $unix->find_program("mv");
$TempDir = "{$GLOBALS["WindowsUpdateCachingDir"]}/Partials/" . md5($URI);
$TempFile = "{$TempDir}/FILE";
if (is_file($TempFile)) {
$size = @filesize($TempFile);
}
@mkdir($TempDir, 0755, true);
$WorkingPort = $SquidMgrListenPort = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/SquidMgrListenPort"));
@unlink("{$TempDir}/stderr.txt");
$f[] = "{$curl} --location";
if ($WindowsUpdateUseLocalProxy == 1) {
events("Downloading {$BaseNameOfFile} Proxy:127.0.0.1:{$WorkingPort}", __LINE__);
$f[] = "--proxy 127.0.0.1:{$WorkingPort} --url \"{$URI}\"";
}
if ($WindowsUpdateInterface != null) {
$INTERFACES = $unix->NETWORK_ALL_INTERFACES();
$ipaddr = $INTERFACES[$WindowsUpdateInterface]["IPADDR"];
if ($ipaddr == "0.0.0.0") {
$ipaddr = null;
}
if ($ipaddr != null) {
events("Downloading {$BaseNameOfFile} Interface:{$ipaddr}", __LINE__);
$f[] = "--interface {$ipaddr}";
}
}
$f[] = "--show-error --write-out \"RRRR:%{http_code} TTT:%{time_total}\"";
$f[] = "--stderr {$TempDir}/stderr.txt";
$f[] = "--output \"{$TempFile}\"";
$f[] = "--continue-at -";
$f[] = "--limit-rate {$WindowsUpdateBandwidthPartial}K";
$f[] = "--url \"{$URI}\"";
$f[] = "2>&1";
$cmd = @implode(" ", $f);
if ($GLOBALS["VERBOSE"]) {
echo "{$cmd}\n";
}
$T1 = time();
events("Downloading {$BaseNameOfFile} {$EXPECTED_SIZE_TEXT} limit:{$WindowsUpdateBandwidthPartial}Kb/s", __LINE__);
exec($cmd, $results);
$stderr = trim(@file_get_contents("{$TempDir}/stderr.txt"));
if ($stderr != null) {
while (list($num, $val) = each($results)) {
if (preg_match("#RRRR:(.+?)\\s+TTT:(.+)#", $val, $re)) {
$HTTP_CODE = $re[1];
$Time = $re[2];
continue;
}
events("INFO: {$val}", __LINE__);
}
if (intval($HTTP_CODE) != 200) {
if (preg_match("#Resuming transfer from byte position\\s+([0-9]+)#", $stderr, $re)) {
events("Downloading {$BaseNameOfFile} stopped duration:{$Time} HTTP Code:{$HTTP_CODE} and resuming position at : {$re[1]} (" . xFormatBytes($re[1] / 1024) . ")", __LINE__);
return false;
}
events("Failed: duration:{$Time} {$BaseNameOfFile} with error {$HTTP_CODE} «{$stderr}»", __LINE__);
return false;
}
}
$size = @filesize($TempFile);
$size_text = xFormatBytes($size);
events("INFO: {$BaseNameOfFile} ({$size_text}) HTTP Code:{$HTTP_CODE} Duration:{$Time}", __LINE__);
if ($size < $EXPECTED_SIZE) {
events("Warning: {$BaseNameOfFile} {$size} is not {$EXPECTED_SIZE} (broken download) retry next time", __LINE__);
return false;
}
events("INFO: {$BaseNameOfFile} move to {$LocalFile}", __LINE__);
@mkdir(dirname($LocalFile));
$cmd = "{$mv} \"{$TempFile}\" \"{$LocalFile}\"";
exec($cmd, $output, $return_val);
if ($return_val == 0) {
events("Success: Retranslate {$BaseNameOfFile} to target directory", __LINE__);
if (is_dir($TempDir)) {
shell_exec("{$rm} -rf {$TempDir}");
}
return true;
} else {
events("Failed: {$BaseNameOfFile}, unable to move to target directory!", __LINE__);
return false;
}
}
function xstart()
{
$T1 = time();
$curl = new ccurl();
$unix = new unix();
$GLOBALS["MYPID"] = getmypid();
$pidfile = "/etc/artica-postfix/pids/windowupdate.processor.pid";
$pid = $unix->get_pid_from_file($pidfile);
if ($unix->process_exists($pid, basename(__FILE__))) {
die;
}
$pids = $unix->PIDOF_PATTERN_ALL(basename(__FILE__), true);
if (count($pids) > 0) {
while (list($i, $line) = each($pids)) {
events("Already executed PID:{$i}... aborting ", __LINE__);
}
die;
}
$TEMPDIR = $unix->TEMP_DIR() . "/WindowsUpdates";
$rm = $unix->find_program("rm");
@file_put_contents($pidfile, $GLOBALS["MYPID"]);
if (is_dir($TEMPDIR)) {
shell_exec("{$rm} -rf {$TEMPDIR}");
}
@mkdir($TEMPDIR, 0755, true);
$WindowsUpdateMaxPartition = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxPartition"));
if ($WindowsUpdateMaxPartition == 0) {
$WindowsUpdateMaxPartition = 80;
}
$CheckPartitionPercentage = CheckPartitionPercentage();
if ($CheckPartitionPercentage > $WindowsUpdateMaxPartition) {
$time = $unix->file_time_min("/etc/squid3/WindowsUpdatePartitionExceed");
if ($time > 10) {
@unlink("/etc/squid3/WindowsUpdatePartitionExceed");
events("Failed: Storage Partition exceed {$WindowsUpdateMaxPartition}%, Stopping retreivals", __LINE__);
@touch("/etc/squid3/WindowsUpdatePartitionExceed");
DirectorySize();
}
return;
}
$WindowsUpdateInProduction = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateInProduction"));
if ($WindowsUpdateInProduction == 0) {
if ($unix->IsProductionTime()) {
$time = $unix->file_time_min("/etc/artica-postfix/pids/WindowsUpdateInProduction");
if ($time > 15) {
@unlink("/etc/artica-postfix/pids/WindowsUpdateInProduction");
@touch("/etc/artica-postfix/pids/WindowsUpdateInProduction");
events("INFO: Aborting, No download during production time", __LINE__);
DirectorySize();
}
return;
}
}
if (is_file("/etc/squid3/WindowsUpdatePartitionExceed")) {
@unlink("/etc/squid3/WindowsUpdatePartitionExceed");
}
$q = new mysql_squid_builder();
$sql = "CREATE TABLE IF NOT EXISTS `windowsupdate` (\n\t\t\t`filemd5` VARCHAR( 90 ) NOT NULL ,\n\t\t\t`zDate` DATETIME NOT NULL ,\n\t\t\t`zUri` VARCHAR( 255 ) NOT NULL ,\n\t\t\t`localpath` VARCHAR( 255 ) NOT NULL ,\n\t\t\t`filesize` BIGINT UNSIGNED DEFAULT '0',\n\t\t\t INDEX ( `filesize` ,`zDate`) ,\n\t\t\t KEY `localpath`(`localpath`),\n\t\t\t KEY `zUri`(`zUri`),\n\t\t\t PRIMARY KEY (`filemd5`)) ENGINE=MYISAM;";
$q->QUERY_SQL($sql);
if (!$q->ok) {
events("MySQL Failed {$q->mysql_error}", __LINE__);
die;
}
$GLOBALS["WindowsUpdateMaxToPartialQueue"] = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateMaxToPartialQueue"));
if ($GLOBALS["WindowsUpdateMaxToPartialQueue"] == 0) {
$GLOBALS["WindowsUpdateMaxToPartialQueue"] = 350;
}
$GLOBALS["WindowsUpdateCachingDir"] = @file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateCachingDir");
if ($GLOBALS["WindowsUpdateCachingDir"] == null) {
$GLOBALS["WindowsUpdateCachingDir"] = "/home/squid/WindowsUpdate";
}
$filepath = "{$GLOBALS["WindowsUpdateCachingDir"]}/Queue.log";
$WindowsUpdateDownTimeout = intval(@file_get_contents("/etc/artica-postfix/settings/Daemons/WindowsUpdateDownTimeout"));
if ($WindowsUpdateDownTimeout == 0) {
$WindowsUpdateDownTimeout = 600;
}
$WindowsUpdateMaxToPartialQueue = $GLOBALS["WindowsUpdateMaxToPartialQueue"] * 1000;
$WindowsUpdateMaxToPartialQueue = $WindowsUpdateMaxToPartialQueue * 1000;
$LinesCount = $unix->COUNT_LINES_OF_FILE($filepath);
if (!is_file($filepath)) {
return;
}
$md5start = md5_file($filepath);
$handle = @fopen($filepath, "r");
if (!$handle) {
events("Fopen failed on {$filepath}", __LINE__);
return false;
}
$NEWBUFFER = array();
$URLALREADY = array();
$FinalSize = 0;
$FF = 0;
$c = 0;
while (!feof($handle)) {
$buffer = trim(fgets($handle));
$c++;
if ($buffer == null) {
continue;
}
$TR = explode("|||", $buffer);
$prc = $c / $LinesCount;
$prc = round($prc * 100);
$LocalFile = $TR[0];
$URI = $TR[1];
$ExpectedSize = 0;
if (strpos($URI, $GLOBALS["WindowsUpdateCachingDir"]) > 0) {
events("FOUND! directory in URI", __LINE__);
$TTR = explode($GLOBALS["WindowsUpdateCachingDir"], $URI);
$URI = $TTR[0];
$LocalFile = "{$GLOBALS["WindowsUpdateCachingDir"]}{$TTR[1]}";
events("FOUND! URI:{$URI}", __LINE__);
events("FOUND! NEXT:{$LocalFile}", __LINE__);
}
$BASENAMELL = basename($LocalFile);
if (strlen($BASENAMELL) > 20) {
$BASENAMELL = substr($BASENAMELL, 0, 17) . "...";
}
build_progressG("{$BASENAMELL} {$c}/{$LinesCount} {files}", $prc);
if (isset($URLALREADY[$URI])) {
continue;
}
$URLALREADY[$URI] = true;
if (isBlacklisted($URI)) {
events(basename($URI) . " blacklisted...");
continue;
}
if (is_file($LocalFile)) {
$size = @filesize($LocalFile);
if ($size > 5) {
events("SKIP " . basename($LocalFile) . " " . xFormatBytes($size / 1024), __LINE__);
update_mysql($LocalFile, $URI);
continue;
} else {
@unlink($LocalFile);
}
}
$dirname = dirname($LocalFile);
if (!is_dir($dirname)) {
@mkdir($dirname, true, 0755);
}
$curl = new ccurl($URI);
$Headers = $curl->getHeaders();
$TIMEDOWN = time();
$TMPFILE = "{$TEMPDIR}/" . basename($LocalFile);
$GLOBALS["previousProgress"] = 0;
$GLOBALS["DOWNLOADED_FILE"] = basename($LocalFile);
$GLOBALS["TMPFILE"] = $TMPFILE;
$ExpectedSize = GetTargetedSize($URI);
if ($ExpectedSize == 0) {
events("Failed to download {$URI} ( unable to get expected size)", __LINE__);
continue;
}
if ($ExpectedSize > $WindowsUpdateMaxToPartialQueue) {
$ExpectedSizeText = xFormatBytes($ExpectedSize / 1024, true);
events(basename($URI) . " ({$ExpectedSizeText} {$ExpectedSize}/{$WindowsUpdateMaxToPartialQueue}) Limit {$WindowsUpdateMaxToPartialQueue} to BigFiles queue", __LINE__);
AddToPartialQueue($URI, $ExpectedSize, $LocalFile);
continue;
}
$curl = new ccurl($URI);
$curl->WriteProgress = true;
$curl->Timeout = $WindowsUpdateDownTimeout * 60;
$curl->ProgressFunction = "xdownload_progress";
events("Downloading " . basename($URI) . " to {$TMPFILE} (" . xFormatBytes($ExpectedSize / 1024, true) . " max:{$WindowsUpdateDownTimeout} Minutes)", __LINE__);
if (!$curl->GetFile($TMPFILE)) {
events("Failed: TMP: «{$TMPFILE}»", __LINE__);
events("Failed: URL: «{$URI}»", __LINE__);
events("Failed: After: " . $unix->distanceOfTimeInWords($TIMEDOWN, time(), true), __LINE__);
events("Failed: With error: {$curl->error} http code: {$curl->CURLINFO_HTTP_CODE} (" . count($curl->CURL_ALL_INFOS) . ") infos", __LINE__);
reset($curl->CURL_ALL_INFOS);
while (list($index, $value) = each($curl->CURL_ALL_INFOS)) {
events("Failed: «{$index}» [{$value}]", __LINE__);
}
if ($curl->CURLINFO_HTTP_CODE == 404) {
continue;
}
@unlink($TMPFILE);
$NEWBUFFER[] = "{$buffer}";
continue;
}
if (!is_file($TMPFILE)) {
events("Fatal {$TMPFILE}: no such file", __LINE__);
continue;
}
$size = filesize($TMPFILE);
$sizeT = xFormatBytes($size / 1024);
if ($size < 5) {
@unlink($TMPFILE);
events("Failed: File less than 5 Bytes ({$size}), aborting", __LINE__);
continue;
}
if ($ExpectedSize > 0) {
if ($size != $ExpectedSize) {
$ExpectedSizeT = xFormatBytes($ExpectedSize / 1024);
events("Failed: corrupted download " . basename($URI) . " expected size {$ExpectedSizeT}/{$ExpectedSize} current:({$sizeT}/{$size})", __LINE__);
@unlink($TMPFILE);
continue;
}
}
if (!@copy($TMPFILE, $LocalFile)) {
@unlink($TMPFILE);
events("Failed: Translating to {$LocalFile}", __LINE__);
$NEWBUFFER[] = "{$buffer}";
continue;
}
events("Success: " . basename($TMPFILE) . " ({$sizeT})", __LINE__);
@unlink($TMPFILE);
$FF++;
$size = @filesize($LocalFile);
$FinalSize = $FinalSize + $size;
update_mysql($LocalFile, $URI);
}
$took = $unix->distanceOfTimeInWords($T1, time(), true);
if ($FinalSize > 0) {
$CURLINFO_SPEED_DOWNLOAD = $curl->CURL_ALL_INFOS["CURLINFO_SPEED_DOWNLOAD"];
events("Downloaded {$FF} files for " . xFormatBytes($FinalSize / 1024, true) . " ({$CURLINFO_SPEED_DOWNLOAD}) took: {$took}", __LINE__);
}
$md5finish = md5_file($filepath);
if (count($NEWBUFFER) > 0) {
events("Retry " . count($NEWBUFFER) . " requests next time...", __LINE__);
@file_put_contents($filepath, @implode("\n", $NEWBUFFER));
} else {
events("No new file downloaded....", __LINE__);
events("Removing queue {$filepath}", __LINE__);
@unlink($filepath);
}
DirectorySize();
events(" * * * END TOOK: {$took} * * *", __LINE__);
}
