<?php
include './archive_common.php.inc';
/**
* Loop through all of the test directories cleaning out any tests that have
* finished processing and have been archived successfully.
*/
archive_scan_dirs(function ($info) {
if (isset($info['dir']) && strlen($info['dir']) && is_file("{$info['dir']}/processing.done")) {
$dir = $info['dir'];
$files = scandir($dir);
if ($files !== FALSE) {
$children = array();
foreach ($files as $file) {
if ($file != '.' && $file != '..' && is_dir("{$dir}/{$file}")) {
$children[] = $file;
}
}
if (count($children)) {
echo "Pruning {$dir}\n";
foreach ($children as $child) {
delTree("{$dir}/{$child}");
}
}
}
CleanupProcessing($dir);
}
}, true);
archive_scan_dirs(function ($info) {
global $now, $bucket_base, $download_path;
$dir = $info['dir'];
$id = $info['id'];
$dir_lock = Lock("archive-{$id}", false, 86400);
if (isset($dir_lock)) {
if (is_file("{$dir}/testing.complete") && is_file("{$dir}/har.complete")) {
$suffix = '';
// Re-archive anything that was archived > 30 days ago but still hasn't shown up as a valid archive
if (is_file("{$dir}/archive.dat") && !ArchiveExists("{$dir}/archive.dat")) {
$modified = filemtime("{$dir}/archive.dat");
if ($modified > 0 && $modified < $now && $now - $modified > 2592000) {
$suffix = '_' . date('ymd');
unlink("{$dir}/archive.dat");
}
}
$archiveDir = "{$dir}/archive";
if (is_dir($archiveDir)) {
delTree($archiveDir);
}
if (!is_file("{$dir}/archive.dat")) {
$children = array();
$files = scandir($dir);
if ($files !== FALSE) {
foreach ($files as $file) {
if ($file != '.' && $file != '..' && strpos($file, 'archive') === false && is_dir("{$dir}/{$file}")) {
$children[] = $file;
}
}
}
$count = count($children);
if ($count > 0) {
logMessage("Archiving {$id} ({$count} tests)...");
mkdir($archiveDir, 0777, true);
$index = 0;
foreach ($children as $child) {
$index++;
logMessage("Compressing {$index} of {$count} - {$id}/{$child}...");
ZipDirectory("{$dir}/{$child}", "{$archiveDir}/{$id}_{$child}.zip", true);
}
$zipFile = "{$dir}/{$id}.zip";
if (is_file($zipFile)) {
unlink($zipFile);
}
logMessage("Combining tests into {$zipFile}...");
ZipDirectory($archiveDir, $zipFile, true);
delTree($archiveDir);
if (is_file($zipFile)) {
$bucket = "{$bucket_base}{$info['year']}_{$info['month']}_{$info['day']}_{$info['group']}{$suffix}";
logMessage("Uploading {$zipFile} to {$bucket}");
if (ArchiveFile($zipFile, $bucket)) {
logMessage("Archiving {$id} Complete");
file_put_contents("{$dir}/archive.dat", "{$download_path}{$bucket}/{$id}.zip");
} else {
logMessage("Archiving {$id} FAILED");
}
unlink($zipFile);
} else {
logMessage("Failed to combine files");
}
} else {
// Nothing to process, mark the group as done
file_put_contents("{$dir}/archive.dat", "");
file_put_contents("{$dir}/archive.dat.valid", "");
logMessage("{$id} - No tests to Archive");
}
} else {
logMessage("{$id} - Already Archived");
}
} else {
logMessage("{$id} - Tests still running");
}
Unlock($dir_lock);
} else {
logMessage("{$id} is already being processed");
}
});
archive_scan_dirs(function ($info) {
global $now, $bucket_base, $download_path;
$dir = $info['dir'];
$id = $info['id'];
$dir_lock = Lock("har-{$id}", false, 86400);
if (isset($dir_lock)) {
if (is_file("{$dir}/testing.complete") && !is_file("{$dir}/har.complete")) {
if (is_file("{$dir}/tests.json")) {
$tests = json_decode(file_get_contents("{$dir}/tests.json"), true);
}
if (isset($tests) && is_array($tests) && isset($tests['tests']) && count($tests['tests']) && isset($tests['crawls']) && count($tests['crawls'])) {
$ok = true;
logMessage("{$id} - Processing");
foreach ($tests['crawls'] as $crawlId => $crawl) {
if (!is_file("{$dir}/{$crawlId}.har")) {
$name = $crawl['name'];
$type = array_shift(explode('-', $name));
$harDir = "{$dir}/{$name}";
if (is_dir($harDir)) {
delTree($harDir);
}
$traceDir = "{$dir}/traces-{$name}";
if (is_dir($traceDir)) {
delTree($traceDir);
}
$testIDs = array();
foreach ($tests['tests'] as $test) {
if (isset($test['crawl']) && isset($test['id']) && $test['crawl'] == $crawlId) {
$testIDs[] = $test['id'];
}
}
$count = count($testIDs);
if ($count > 0) {
logMessage("{$id} - {$count} {$type} tests: {$harDir}");
mkdir($harDir, 0777, true);
if (CollectHARs($testIDs, $harDir)) {
logMessage("Uploading to Google storage...");
if (gsUpload($harDir, $name)) {
logMessage("Upload complete...");
} else {
$ok = false;
}
} else {
$ok = false;
}
delTree($harDir);
logMessage("{$id} - {$count} {$type} tests (traces): {$traceDir}");
mkdir($traceDir, 0777, true);
if (CollectTraces($testIDs, $traceDir)) {
gsUpload($traceDir, "traces-{$name}");
}
delTree($traceDir);
}
}
}
if ($ok) {
file_put_contents("{$dir}/har.complete", '');
}
} else {
// Bogus test data, mark it as done
file_put_contents("{$dir}/har.complete", '');
logMessage("{$id} - No tests available");
}
} else {
if (!is_file("{$dir}/testing.complete")) {
logMessage("{$id} - Tests still running");
}
if (is_file("{$dir}/har.complete")) {
logMessage("{$id} - HAR already processed");
}
}
Unlock($dir_lock);
} else {
logMessage("{$id} is already being processed");
}
});
archive_scan_dirs(function ($info) {
global $UTC, $now, $checkedCrawl, $pendingTests, $finishedTests, $crawls;
$dir = $info['dir'];
$id = $info['id'];
// Only look at tests that were started at least 1 day ago (eliminates any possible race conditions)
$date = DateTime::createFromFormat('ymd', "{$info['year']}{$info['month']}{$info['day']}", $UTC);
$daytime = $date->getTimestamp();
$elapsed = max($now - $daytime, 0) / 86400;
if ($elapsed >= 1 && !is_file("{$dir}/testing.complete")) {
if (!$checkedCrawl) {
$checkedCrawl = UpdateCrawlState();
}
$pending = 0;
$complete = 0;
if (isset($pendingTests[$id])) {
$pending = $pendingTests[$id];
}
if (isset($finishedTests[$id])) {
$complete = count($finishedTests[$id]);
}
$total = $pending + $complete;
echo "{$id} : {$complete} of {$total} complete ({$pending} pending)\n";
if ($complete > 0 && !$pending) {
// Write out the test and crawl information
$tests = array('crawls' => $crawls, 'tests' => $finishedTests[$id]);
file_put_contents("{$dir}/tests.json", json_encode($tests));
// mark it as complete
file_put_contents("{$dir}/testing.complete", '');
} elseif (!$total) {
// flag any groups that are not part of a crawl as complete
file_put_contents("{$dir}/testing.complete", '');
}
}
// See if all of the individual processing steps are done. If so, mark mrocessing as complete.
echo "Checking {$dir}\n";
if (is_file("{$dir}/testing.complete") && is_file("{$dir}/har.complete") && is_file("{$dir}/archive.dat") && ArchiveExists("{$dir}/archive.dat")) {
MarkDone($dir);
} else {
if (!is_file("{$dir}/testing.complete")) {
echo " Missing {$dir}/testing.complete\n";
}
if (!is_file("{$dir}/har.complete")) {
echo " Missing {$dir}/har.complete\n";
}
if (!is_file("{$dir}/archive.dat")) {
echo " Missing {$dir}/archive.dat\n";
} else {
if (!ArchiveExists("{$dir}/archive.dat")) {
echo " Archive not valid\n";
}
}
}
});
