function ProcessDir($Dir)
{
msg(" Process dir : " . $Dir, 'blue');
if ($Dh = opendir($Dir)) {
while (($File = readdir($Dh)) !== false) {
if ($File == '.' || $File == '..') {
continue;
}
$Path = $Dir . '/' . $File;
if (is_dir($Path)) {
ProcessDir($Path);
} else {
ProcessFile($Path);
}
}
closedir($Dh);
}
}
#!/usr/bin/php
<?php
// Needs to point out the web code... TODO: Avoid this config issue
$sWebPath = "/var/www/html/includes/";
require_once $sWebPath . "/processing.php";
$iErr = 0;
if ($argc < 2) {
printf("Usage: add_database.php <PDF file>\n");
$iErr = 255;
} else {
if (($iErr = InitializeWork()) == 0) {
if (($iErr = ProcessFile($argv[1], $argc == 3 ? $argv[2] : 0)) != 0) {
printf("ProcessFile() failed with %d\n", $iErr);
}
if (($iErr = CleanupWork()) != 0) {
printf("CleanupWork() failed with %d\n", $iErr);
}
} else {
printf("InitializeWork() failed with %d\n", $iErr);
}
}
exit($iErr);
function ProcessSubmissions($projectid)
{
$qs = "SELECT id, filename, filesize, filemd5sum, attempts FROM submission " . "WHERE projectid='" . $projectid . "' AND status=0 ORDER BY id LIMIT 1";
$query = pdo_query($qs);
add_last_sql_error("ProcessSubmissions-1");
$iterations = 0;
$mypid = getmypid();
@($sleep_in_loop = $_GET['sleep_in_loop']);
@($intentional_nonreturning_submit = $_GET['intentional_nonreturning_submit']);
$n = pdo_num_rows($query);
while ($n > 0) {
if ($sleep_in_loop) {
sleep($sleep_in_loop);
}
$query_array = pdo_fetch_array($query);
add_last_sql_error("ProcessSubmissions-1.5");
pdo_free_result($query);
// Verify that *this* process still owns the lock.
//
// If not, log a message and return, presuming that the process that took
// the lock is now looping over pending submissions.
//
if (!ProcessOwnsLock($projectid, $mypid)) {
add_log("pid '{$mypid}' does not own lock anymore: abandoning loop...", "ProcessSubmissions", LOG_INFO, $projectid);
return false;
}
$submission_id = $query_array['id'];
$filename = $query_array['filename'];
$new_attempts = $query_array['attempts'] + 1;
$md5 = $query_array['filemd5sum'];
// Mark the submissionprocessing table each time through the loop so that
// we do not become known as an "apparently stalled" processor...
//
SetLockLastUpdatedTime($projectid);
global $CDASH_SUBMISSION_PROCESSING_MAX_ATTEMPTS;
if ($new_attempts > $CDASH_SUBMISSION_PROCESSING_MAX_ATTEMPTS) {
add_log("Too many attempts to process '" . $filename . "'", "ProcessSubmissions", LOG_ERR, $projectid);
$new_status = 5;
// done, called do_submit too many times already
} else {
// Mark it as status=1 (processing) and record started time:
//
$now_utc = gmdate(FMT_DATETIMESTD);
pdo_query("UPDATE submission SET status=1, started='{$now_utc}', " . "lastupdated='{$now_utc}', attempts={$new_attempts} " . "WHERE id='" . $submission_id . "'");
add_last_sql_error("ProcessSubmissions-2");
// Record id in global so that we can mark it as "error status" if we
// get thrown to the error handler...
//
global $PHP_ERROR_SUBMISSION_ID;
$PHP_ERROR_SUBMISSION_ID = $submission_id;
if ($intentional_nonreturning_submit) {
// simulate "error occurred" during do_submit: status will be set
// to 4 in error handler...
trigger_error('ProcessFile: intentional_nonreturning_submit is on', E_USER_ERROR);
}
$new_status = ProcessFile($projectid, $filename, $md5);
}
$PHP_ERROR_SUBMISSION_ID = 0;
// Mark it as done with $new_status and record finished time:
//
$now_utc = gmdate(FMT_DATETIMESTD);
pdo_query("UPDATE submission SET status={$new_status}, finished='{$now_utc}', " . "lastupdated='{$now_utc}' WHERE id='" . $submission_id . "'");
add_last_sql_error("ProcessSubmissions-3");
// Query for more... Continue processing while there are records to
// process:
//
$query = pdo_query($qs);
add_last_sql_error("ProcessSubmissions-4");
$n = pdo_num_rows($query);
$iterations = $iterations + 1;
}
return true;
}
function ProcessTVCommand($value, $name = '', $docid = '')
{
global $modx;
$etomite =& $modx;
$docid = intval($docid) ? intval($docid) : $modx->documentIdentifier;
$nvalue = trim($value);
if (substr($nvalue, 0, 1) != '@') {
return $value;
} else {
list($cmd, $param) = ParseCommand($nvalue);
$cmd = trim($cmd);
switch ($cmd) {
case "FILE":
$output = ProcessFile(trim($param));
$output = str_replace('@FILE ' . $param, $output, $nvalue);
break;
case "CHUNK":
// retrieve a chunk and process it's content
$chunk = $modx->getChunk($param);
$output = $chunk;
break;
case "DOCUMENT":
// retrieve a document and process it's content
$rs = $modx->getDocument($param);
if (is_array($rs)) {
$output = $rs['content'];
} else {
$output = "Unable to locate document {$param}";
}
break;
case "SELECT":
// selects a record from the cms database
$rt = array();
$replacementVars = array('DBASE' => $modx->db->config['dbase'], 'PREFIX' => $modx->db->config['table_prefix']);
foreach ($replacementVars as $rvKey => $rvValue) {
$modx->setPlaceholder($rvKey, $rvValue);
}
$param = $modx->mergePlaceholderContent($param);
$rs = $modx->db->query("SELECT {$param};");
$output = $rs;
break;
case "EVAL":
// evaluates text as php codes return the results
$output = eval($param);
break;
case "INHERIT":
$output = $param;
// Default to param value if no content from parents
$doc = $modx->getPageInfo($docid, 0, 'id,parent');
while ($doc['parent'] != 0) {
$parent_id = $doc['parent'];
// Grab document regardless of publish status
$doc = $modx->getPageInfo($parent_id, 0, 'id,parent,published');
if ($doc['parent'] != 0 && !$doc['published']) {
continue;
}
// hide unpublished docs if we're not at the top
$tv = $modx->getTemplateVar($name, '*', $doc['id'], $doc['published']);
// Modified @INHERIT binding to allow for @FILE bindings (and others) to be inherited,
// as well as allowing content after the @INHERITed values
if ((string) $tv['value'] !== '' && !preg_match('%^@INHERIT[\\s\\n\\r]*$%im', $tv['value'])) {
$output = (string) $tv['value'];
$output = str_replace('@INHERIT', $output, $nvalue);
break 2;
}
}
break;
case 'DIRECTORY':
$files = array();
$path = $modx->config['base_path'] . $param;
if (substr($path, -1, 1) != '/') {
$path .= '/';
}
if (!is_dir($path)) {
die($path);
break;
}
$dir = dir($path);
while (($file = $dir->read()) !== false) {
if (substr($file, 0, 1) != '.') {
$files[] = "{$file}=={$param}{$file}";
}
}
asort($files);
$output = implode('||', $files);
break;
default:
$output = $value;
break;
}
// support for nested bindings
return is_string($output) && $output != $value ? ProcessTVCommand($output, $name, $docid) : $output;
}
}
function ProcessSubmissions($projectid, $mypid, $multi = false)
{
$iterations = 0;
@($sleep_in_loop = $_GET['sleep_in_loop']);
@($intentional_nonreturning_submit = $_GET['intentional_nonreturning_submit']);
$query_array = GetNextSubmission($projectid);
if ($query_array === false) {
return false;
}
$n = count($query_array);
while ($n > 0) {
if ($sleep_in_loop) {
sleep($sleep_in_loop);
}
// Verify that *this* process still owns the lock.
//
// If not, log a message and return, presuming that the process
// that took the lock is now looping over pending submissions.
//
if (!$multi && !ProcessOwnsLock($projectid, $mypid)) {
add_log("pid '{$mypid}' does not own lock anymore: abandoning loop...", 'ProcessSubmissions', LOG_INFO, $projectid);
return false;
}
$submission_id = $query_array['id'];
$filename = $query_array['filename'];
$new_attempts = $query_array['attempts'] + 1;
$md5 = $query_array['filemd5sum'];
// Mark the submissionprocessing table each time through the loop
// so that we do not become known as an "apparently stalled" processor.
SetLockLastUpdatedTime($projectid);
global $CDASH_SUBMISSION_PROCESSING_MAX_ATTEMPTS;
if ($new_attempts > $CDASH_SUBMISSION_PROCESSING_MAX_ATTEMPTS) {
add_log("Too many attempts to process '{$filename}'", 'ProcessSubmissions', LOG_ERR, $projectid);
$new_status = 5;
// done, called do_submit too many times already
} else {
// Record id in global so that we can mark it as "error status"
// if we get thrown to the error handler.
global $PHP_ERROR_SUBMISSION_ID;
$PHP_ERROR_SUBMISSION_ID = $submission_id;
if ($intentional_nonreturning_submit) {
// simulate "error occurred" during do_submit:
// status will be set to 4 in error handler.
trigger_error('ProcessFile: intentional_nonreturning_submit is on', E_USER_ERROR);
}
$new_status = ProcessFile($projectid, $filename, $md5);
}
$PHP_ERROR_SUBMISSION_ID = 0;
global $CDASH_ASYNC_EXPIRATION_TIME;
if ($CDASH_ASYNC_EXPIRATION_TIME === 0 && ($new_status > 1 && $new_status < 6)) {
// If our expiration time is set to 0 we delete finished
// submissions rather than marking them as done in the database.
pdo_query("DELETE FROM submission WHERE id='{$submission_id}'");
add_last_sql_error('ProcessSubmissions-3');
pdo_query("DELETE FROM submission2ip WHERE submissionid='{$submission_id}'");
add_last_sql_error('ProcessSubmissions-3');
} else {
// Mark it as done with $new_status and record finished time:
//
$now_utc = gmdate(FMT_DATETIMESTD);
pdo_query("UPDATE submission SET status={$new_status}, finished='{$now_utc}',\n lastupdated='{$now_utc}' WHERE id='{$submission_id}'");
add_last_sql_error('ProcessSubmissions-3');
}
// Query for more... Continue processing while there are records to
// process:
//
$query_array = GetNextSubmission($projectid);
if ($query_array === false) {
return false;
}
$n = count($query_array);
$iterations = $iterations + 1;
}
return true;
}
require_once "cdash/pdo.php";
ob_start();
set_time_limit(0);
ignore_user_abort(TRUE);
@($projectid = $_REQUEST['projectid']);
@($filename = $_REQUEST['filename']);
@($callit = $_REQUEST['callit']);
if (!is_numeric($projectid) || $projectid == 0) {
trigger_error('processfile.php: projectid no good', E_USER_ERROR);
}
$projectid = pdo_real_escape_numeric($projectid);
if (!$filename) {
trigger_error('processfile.php: filename no good', E_USER_ERROR);
}
$filename = htmlspecialchars(pdo_real_escape_string($filename));
if (!isset($callit)) {
$callit = 1;
}
register_shutdown_function('PHPErrorHandler', $projectid);
echo "<pre>\n";
if ($callit) {
echo "before ProcessFile call\n";
$status = ProcessFile($projectid, $filename);
echo "after ProcessFile call\n";
echo "status = {$status}\n";
} else {
echo "no ProcessFile call\n";
echo "callit = {$callit}\n";
}
echo "</pre>\n";
ob_end_flush();
require_once 'archive.inc';
require_once 'page_data.inc';
require_once 'breakdown.inc';
require_once 'video/avi2frames.inc.php';
set_time_limit(0);
if (count($argv) > 1 && strlen($argv[1])) {
$benchmark = trim($argv[1]);
echo "Reprocessing benchmark '{$benchmark}'\r\n";
if ($lock = Lock("reprocess-{$benchmark}")) {
if (is_file("./log/reprocess-{$benchmark}.log")) {
unlink("./log/reprocess-{$benchmark}.log");
}
$files = LoadStatus();
foreach ($files as &$entry) {
if (!$entry['processed']) {
ProcessFile($entry['file']);
$entry['processed'] = true;
SaveStatus($files);
}
}
echo "Done\n";
logMsg("Done", "./log/reprocess-{$benchmark}.log", true);
Unlock($lock);
}
} else {
echo "Usage: php reprocess.php <benchmark ID>\r\n\r\n";
}
function LoadStatus()
{
global $benchmark;
if (is_file("./results/benchmarks/{$benchmark}/reprocess.json")) {
foreach ($regions as $region) {
// echo $region;
if (stripos($filename, $region) !== false) {
$file_region = $region;
break;
}
}
if ($file_region == '') {
die('Region Not Recognised. Try again.');
}
echo "File: " . $filename;
echo '<br>';
echo "Region: " . ucwords($file_region);
echo '<br>';
echo '<br>';
ProcessFile($_FILES['csv_file']['tmp_name'], $file_region);
echo '<br><br>';
// foreach (scandir('.') as $file) {
// if ($dot = strrpos($file,'.')) {
// if (strtoupper(substr($file,$dot+1))=='CSV') {
// ProcessFile($file);
// }
// }
// }
}
}
?>
<!DOCTYPE html>
<html>
<head>
<style>
border: 1px solid;
}
</style>
</head>
<body>
<?php
if ($iSplit != 0) {
?>
<h1>Split document into multiple documents</h1>
Please wait, working...
<pre>
<?php
// This is complicated, essentially we rescan the document and then afterwards we delete the original
$iErr = 0;
if (($iErr = InitializeWork()) == 0) {
if (($iErr = ProcessFile($aRes["filename"], $iSplit)) != 0) {
printf("ProcessFile() failed with %d\n", $iErr);
} else {
// Success! Remove original document
$objDb->Delete($id);
}
if (($iErr = CleanupWork()) != 0) {
printf("CleanupWork() failed with %d\n", $iErr);
}
} else {
printf("InitializeWork() failed with %d\n", $iErr);
}
?>
</pre>
<a href="index.php">Done</a>
<?php
function ProcessTVCommand($value, $name = '', $docid = '', $src = 'docform')
{
global $modx;
$etomite =& $modx;
$docid = intval($docid) ? intval($docid) : $modx->documentIdentifier;
$nvalue = trim($value);
if (substr($nvalue, 0, 1) != '@') {
return $value;
} elseif ($modx->config['enable_bindings'] != 1 && $src === 'docform') {
return '@Bindings is disabled.';
} else {
list($cmd, $param) = ParseCommand($nvalue);
$cmd = trim($cmd);
$param = trim($param);
switch ($cmd) {
case "FILE":
$output = ProcessFile(trim($param));
$output = str_replace('@FILE ' . $param, $output, $nvalue);
break;
case "CHUNK":
// retrieve a chunk and process it's content
$chunk = $modx->getChunk($param);
$output = $chunk;
break;
case "DOCUMENT":
// retrieve a document and process it's content
$rs = $modx->getDocument($param);
if (is_array($rs)) {
$output = $rs['content'];
} else {
$output = "Unable to locate document {$param}";
}
break;
case "SELECT":
// selects a record from the cms database
$rt = array();
$replacementVars = array('dbase' => $modx->db->config['dbase'], 'DBASE' => $modx->db->config['dbase'], 'prefix' => $modx->db->config['table_prefix'], 'PREFIX' => $modx->db->config['table_prefix']);
foreach ($replacementVars as $rvKey => $rvValue) {
$modx->setPlaceholder($rvKey, $rvValue);
}
$param = $modx->mergePlaceholderContent($param);
$rs = $modx->db->query("SELECT {$param}");
$output = $rs;
break;
case "EVAL":
// evaluates text as php codes return the results
$output = eval($param);
break;
case "INHERIT":
$output = $param;
// Default to param value if no content from parents
if (empty($docid) && isset($_REQUEST['pid'])) {
$doc['parent'] = $_REQUEST['pid'];
} else {
$doc = $modx->getPageInfo($docid, 0, 'id,parent');
}
while ($doc['parent'] != 0) {
$parent_id = $doc['parent'];
// Grab document regardless of publish status
$doc = $modx->getPageInfo($parent_id, 0, 'id,parent');
$tv = $modx->getTemplateVar($name, '*', $doc['id'], null);
// inheritance allows other @ bindings to be inherited
// if no value is inherited and there is content following the @INHERIT binding,
// that content will be used as the output
// @todo consider reimplementing *appending* the output the follows an @INHERIT as opposed
// to using it as a default/fallback value; perhaps allow choice in behavior with
// system setting
if ((string) $tv['value'] !== '' && !preg_match('%^@INHERIT[\\s\\n\\r]*$%im', $tv['value'])) {
$output = (string) $tv['value'];
//$output = str_replace('@INHERIT', $output, $nvalue);
break 2;
}
}
break;
case 'DIRECTORY':
$files = array();
$path = $modx->config['base_path'] . $param;
if (substr($path, -1, 1) != '/') {
$path .= '/';
}
if (!is_dir($path)) {
die($path);
break;
}
$dir = dir($path);
while (($file = $dir->read()) !== false) {
if (substr($file, 0, 1) != '.') {
$files[] = "{$file}=={$param}{$file}";
}
}
asort($files);
$output = implode('||', $files);
break;
case 'NULL':
case 'NONE':
$output = '';
break;
default:
$output = $value;
break;
}
// support for nested bindings
return is_string($output) && $output != $value ? ProcessTVCommand($output, $name, $docid, $src) : $output;
}
}
