public static function addJob(string $className, string $function, array $args = [], int $priority = 0)
{
$redis = Redis::get();
$queueJobs = new JobQueue($redis);
$job = ['class' => $className, 'function' => $function, 'args' => $args];
$queueJobs->push($job, $priority);
}
public static function getInstance()
{
if (!self::$instance) {
self::$instance = new JobQueue();
}
return self::$instance;
}
protected function setUp()
{
global $wgJobTypeConf;
parent::setUp();
$this->setMwGlobals('wgMemc', new HashBagOStuff());
if ($this->getCliArg('use-jobqueue')) {
$name = $this->getCliArg('use-jobqueue');
if (!isset($wgJobTypeConf[$name])) {
throw new MWException("No \$wgJobTypeConf entry for '{$name}'.");
}
$baseConfig = $wgJobTypeConf[$name];
} else {
$baseConfig = array('class' => 'JobQueueDB');
}
$baseConfig['type'] = 'null';
$baseConfig['wiki'] = wfWikiID();
$variants = array('queueRand' => array('order' => 'random', 'claimTTL' => 0), 'queueRandTTL' => array('order' => 'random', 'claimTTL' => 10), 'queueTimestamp' => array('order' => 'timestamp', 'claimTTL' => 0), 'queueTimestampTTL' => array('order' => 'timestamp', 'claimTTL' => 10), 'queueFifo' => array('order' => 'fifo', 'claimTTL' => 0), 'queueFifoTTL' => array('order' => 'fifo', 'claimTTL' => 10));
foreach ($variants as $q => $settings) {
try {
$this->{$q} = JobQueue::factory($settings + $baseConfig);
if (!$this->{$q} instanceof JobQueueDB) {
$this->{$q}->setTestingPrefix('unittests-' . wfRandomString(32));
}
} catch (MWException $e) {
// unsupported?
// @todo What if it was another error?
}
}
}
protected function setUp()
{
global $wgJobTypeConf;
parent::setUp();
if ($this->getCliArg('use-jobqueue')) {
$name = $this->getCliArg('use-jobqueue');
if (!isset($wgJobTypeConf[$name])) {
throw new MWException("No \$wgJobTypeConf entry for '{$name}'.");
}
$baseConfig = $wgJobTypeConf[$name];
} else {
$baseConfig = ['class' => 'JobQueueDB'];
}
$baseConfig['type'] = 'null';
$baseConfig['wiki'] = wfWikiID();
$variants = ['queueRand' => ['order' => 'random', 'claimTTL' => 0], 'queueRandTTL' => ['order' => 'random', 'claimTTL' => 10], 'queueTimestamp' => ['order' => 'timestamp', 'claimTTL' => 0], 'queueTimestampTTL' => ['order' => 'timestamp', 'claimTTL' => 10], 'queueFifo' => ['order' => 'fifo', 'claimTTL' => 0], 'queueFifoTTL' => ['order' => 'fifo', 'claimTTL' => 10]];
foreach ($variants as $q => $settings) {
try {
$this->{$q} = JobQueue::factory($settings + $baseConfig);
} catch (MWException $e) {
// unsupported?
// @todo What if it was another error?
}
}
}
public function actionIndex()
{
Yii::import("application.models.*");
//get the latest idle cron job
/* @var $latestidle JobQueue*/
$latestidle = JobQueue::model()->findByAttributes(array("status" => JobQueue::$JOBQUEUE_STATUS_IDLE));
if (!$latestidle) {
echo "No file queued";
die;
}
//set status to on-going
$latestidle->status = JobQueue::$JOBQUEUE_STATUS_ON_GOING;
$latestidle->save(false);
//retrieve file
$queueFile = new SplFileObject($latestidle->filename);
//read file
$queueFile->setFlags(SplFileObject::READ_CSV | SplFileObject::READ_AHEAD | SplFileObject::DROP_NEW_LINE | SplFileObject::SKIP_EMPTY);
$queueFile->next();
// Total_records ,
$queueFile->seek(PHP_INT_MAX);
$linesTotal = $queueFile->key();
$latestidle->total_records = $linesTotal;
$latestidle->save(false);
$index = 0;
foreach ($queueFile as $currentLine) {
//iterate content
if ($queueFile->key() === 0) {
continue;
}
//TODO: processed_record
$latestidle->processed_record = ++$index;
$latestidle->save(false);
$currentMobile = $currentLine[0];
$newBlackListedmobile = new BlackListedMobile();
//cleaning time
$currentMobile = trim($currentMobile);
$currentMobile = rtrim($currentMobile);
$currentMobile = ltrim($currentMobile);
$newBlackListedmobile->mobile_number = $currentMobile;
$newBlackListedmobile->queue_id = $latestidle->queue_id;
//set queueid
if ($newBlackListedmobile->save()) {
//save content
echo "{$newBlackListedmobile->mobile_number} : Saved \n";
} else {
echo "{$newBlackListedmobile->mobile_number} : Failed \n";
}
}
//when done
//set status to done
$latestidle->status = JobQueue::$JOBQUEUE_STATUS_DONE;
$latestidle->date_done = date("Y-m-d H:i:s");
//set done datetime to now()
$latestidle->save();
echo "Queue DONE \n";
}
function shutdown()
{
if (Database::hasBeenUsed()) {
Database::commit();
}
// Jobs must not write to database!
$queue = JobQueue::getInstance();
$jobs = $queue->executeJobs();
gfDebug("Request ended.\n");
}
/**
* @param $type string
* @return JobQueue Job queue object for a given queue type
*/
public function get($type)
{
global $wgJobTypeConf;
$conf = array('wiki' => $this->wiki, 'type' => $type);
if (isset($wgJobTypeConf[$type])) {
$conf = $conf + $wgJobTypeConf[$type];
} else {
$conf = $conf + $wgJobTypeConf['default'];
}
return JobQueue::factory($conf);
}
protected function copyJobs(JobQueue $src, JobQueue $dst, $jobs)
{
$total = 0;
$totalOK = 0;
$batch = array();
foreach ($jobs as $job) {
++$total;
$batch[] = $job;
if (count($batch) >= $this->mBatchSize) {
$dst->push($batch);
$totalOK += count($batch);
$batch = array();
$dst->waitForBackups();
}
}
if (count($batch)) {
$dst->push($batch);
$totalOK += count($batch);
$dst->waitForBackups();
}
return array($total, $totalOK);
}
<?php
/* @var $this DefaultController */
/* @var $model JobQueue */
$model = JobQueue::model();
$model->unsetAttributes();
$this->breadcrumbs = array($this->module->id);
$this->breadcrumbs = array(ucwords($this->module->id), 'Job Queues');
/*register submit event listener*/
$registerSubmitListener = <<<EOL
jQuery("form:first").submit(function(event) {
if ( \$("#massiveTextArea").val().length > 0 ) {
submitPartition()
event.preventDefault();
return false;
}
});
EOL;
Yii::app()->clientScript->registerScript('registerSubmitListener', $registerSubmitListener, CClientScript::POS_READY);
/*end of register submit*/
/*begin partition upload of from texterea*/
$partitionUploadCode = <<<EOL
var tempTotalInsertedContainer = 0;
var tempTotalDuplicateFound = 0;
/**
* Todo attach this before submit . make sure to change submit text
* @return
*/
function submitPartition() {
// check if multiline has data
if (jQuery("#massiveTextArea").val().length > 0) {
/**
* @param array $params Possible keys:
* - sectionsByWiki : A map of wiki IDs to section names.
* Wikis will default to using the section "default".
* - partitionsBySection : Map of section names to maps of (partition name => weight).
* A section called 'default' must be defined if not all wikis
* have explicitly defined sections.
* - configByPartition : Map of queue partition names to configuration arrays.
* These configuration arrays are passed to JobQueue::factory().
* The options set here are overridden by those passed to this
* the federated queue itself (e.g. 'order' and 'claimTTL').
* - maxPartitionsTry : Maximum number of times to attempt job insertion using
* different partition queues. This improves availability
* during failure, at the cost of added latency and somewhat
* less reliable job de-duplication mechanisms.
* @throws MWException
*/
protected function __construct(array $params)
{
parent::__construct($params);
$section = isset($params['sectionsByWiki'][$this->wiki]) ? $params['sectionsByWiki'][$this->wiki] : 'default';
if (!isset($params['partitionsBySection'][$section])) {
throw new MWException("No configuration for section '{$section}'.");
}
$this->maxPartitionsTry = isset($params['maxPartitionsTry']) ? $params['maxPartitionsTry'] : 2;
// Get the full partition map
$partitionMap = $params['partitionsBySection'][$section];
arsort($partitionMap, SORT_NUMERIC);
// Get the config to pass to merge into each partition queue config
$baseConfig = $params;
foreach (array('class', 'sectionsByWiki', 'maxPartitionsTry', 'partitionsBySection', 'configByPartition') as $o) {
unset($baseConfig[$o]);
// partition queue doesn't care about this
}
// The class handles all aggregator calls already
unset($baseConfig['aggregator']);
// Get the partition queue objects
foreach ($partitionMap as $partition => $w) {
if (!isset($params['configByPartition'][$partition])) {
throw new MWException("No configuration for partition '{$partition}'.");
}
$this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $params['configByPartition'][$partition]);
}
// Ring of all partitions
$this->partitionRing = new HashRing($partitionMap);
}
/**
* Put an entry in the JobQueue to run the discriminator on this tank_id.
*
* @param int $tank_id
*/
protected function queue_discriminator($tank_id)
{
$jq = new JobQueue();
$jq->jq_job = "PERL AIR2_ROOT/bin/run-discriminator {$tank_id}";
$jq->save();
}
public function create()
{
$this->created_at = new \DateTime();
$jq = JobQueue::getInstance();
$jq->add($this);
}
/**
* @return JobQueueMemory
*/
private function newJobQueue()
{
return JobQueue::factory(array('class' => 'JobQueueMemory', 'wiki' => wfWikiID(), 'type' => 'null'));
}
/**
* Recycle or destroy any jobs that have been claimed for too long
*
* @return int Number of jobs recycled/deleted
*/
public function recycleAndDeleteStaleJobs()
{
$now = time();
$count = 0;
// affected rows
$dbw = $this->getMasterDB();
try {
if (!$dbw->lock("jobqueue-recycle-{$this->type}", __METHOD__, 1)) {
return $count;
// already in progress
}
// Remove claims on jobs acquired for too long if enabled...
if ($this->claimTTL > 0) {
$claimCutoff = $dbw->timestamp($now - $this->claimTTL);
// Get the IDs of jobs that have be claimed but not finished after too long.
// These jobs can be recycled into the queue by expiring the claim. Selecting
// the IDs first means that the UPDATE can be done by primary key (less deadlocks).
$res = $dbw->select('job', 'job_id', array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($claimCutoff)}", "job_attempts < {$dbw->addQuotes($this->maxTries)}"), __METHOD__);
$ids = array_map(function ($o) {
return $o->job_id;
}, iterator_to_array($res));
if (count($ids)) {
// Reset job_token for these jobs so that other runners will pick them up.
// Set the timestamp to the current time, as it is useful to now that the job
// was already tried before (the timestamp becomes the "released" time).
$dbw->update('job', array('job_token' => '', 'job_token_timestamp' => $dbw->timestamp($now)), array('job_id' => $ids), __METHOD__);
$affected = $dbw->affectedRows();
$count += $affected;
JobQueue::incrStats('recycles', $this->type, $affected);
$this->aggr->notifyQueueNonEmpty($this->wiki, $this->type);
}
}
// Just destroy any stale jobs...
$pruneCutoff = $dbw->timestamp($now - self::MAX_AGE_PRUNE);
$conds = array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($pruneCutoff)}");
if ($this->claimTTL > 0) {
// only prune jobs attempted too many times...
$conds[] = "job_attempts >= {$dbw->addQuotes($this->maxTries)}";
}
// Get the IDs of jobs that are considered stale and should be removed. Selecting
// the IDs first means that the UPDATE can be done by primary key (less deadlocks).
$res = $dbw->select('job', 'job_id', $conds, __METHOD__);
$ids = array_map(function ($o) {
return $o->job_id;
}, iterator_to_array($res));
if (count($ids)) {
$dbw->delete('job', array('job_id' => $ids), __METHOD__);
$affected = $dbw->affectedRows();
$count += $affected;
JobQueue::incrStats('abandons', $this->type, $affected);
}
$dbw->unlock("jobqueue-recycle-{$this->type}", __METHOD__);
} catch (DBError $e) {
$this->throwDBException($e);
}
return $count;
}
/**
* @params include:
* - sectionsByWiki : A map of wiki IDs to section names.
* Wikis will default to using the section "default".
* - partitionsBySection : Map of section names to maps of (partition name => weight).
* A section called 'default' must be defined if not all wikis
* have explicitly defined sections.
* - configByPartition : Map of queue partition names to configuration arrays.
* These configuration arrays are passed to JobQueue::factory().
* The options set here are overriden by those passed to this
* the federated queue itself (e.g. 'order' and 'claimTTL').
* - partitionsNoPush : List of partition names that can handle pop() but not push().
* This can be used to migrate away from a certain partition.
* @param array $params
*/
protected function __construct( array $params ) {
parent::__construct( $params );
$section = isset( $params['sectionsByWiki'][$this->wiki] )
? $params['sectionsByWiki'][$this->wiki]
: 'default';
if ( !isset( $params['partitionsBySection'][$section] ) ) {
throw new MWException( "No configuration for section '$section'." );
}
// Get the full partition map
$this->partitionMap = $params['partitionsBySection'][$section];
arsort( $this->partitionMap, SORT_NUMERIC );
// Get the partitions jobs can actually be pushed to
$partitionPushMap = $this->partitionMap;
if ( isset( $params['partitionsNoPush'] ) ) {
foreach ( $params['partitionsNoPush'] as $partition ) {
unset( $partitionPushMap[$partition] );
}
}
// Get the config to pass to merge into each partition queue config
$baseConfig = $params;
foreach ( array( 'class', 'sectionsByWiki',
'partitionsBySection', 'configByPartition', 'partitionsNoPush' ) as $o )
{
unset( $baseConfig[$o] );
}
// Get the partition queue objects
foreach ( $this->partitionMap as $partition => $w ) {
if ( !isset( $params['configByPartition'][$partition] ) ) {
throw new MWException( "No configuration for partition '$partition'." );
}
$this->partitionQueues[$partition] = JobQueue::factory(
$baseConfig + $params['configByPartition'][$partition] );
}
// Get the ring of partitions to push jobs into
$this->partitionPushRing = new HashRing( $partitionPushMap );
// Aggregate cache some per-queue values if there are multiple partition queues
$this->cache = count( $this->partitionMap ) > 1 ? wfGetMainCache() : new EmptyBagOStuff();
}
public function output()
{
$jobs = JobQueue::getInstance();
if ($jobs->countJobs() > 0 && !$this->willCloseConnection) {
gfDebug(__METHOD__ . " Forced close connection for background jobs.");
$this->closeConnection(true);
}
$this->outputHttpHeaders();
$this->outputHttpBody();
if ($this->willCloseConnection) {
@ini_set('zlib.output_compression', 'Off');
@apache_setenv('no-gzip', 1);
header("Connection: close");
$outputSize = ob_get_length();
header("Content-Length: {$outputSize}");
}
ob_end_flush();
ob_flush();
flush();
if ($this->willCloseConnection && session_id()) {
session_write_close();
}
}
/**
*
*
* @param unknown $chg_email
* @param unknown $chg_status
*/
private function _queue_manager_status_change($chg_email, $chg_status)
{
$action = false;
// email changed - only tell campaign manager if the email is marked 'good'
if ($chg_email) {
if ($chg_status == self::$STATUS_GOOD || $this->sem_status == self::$STATUS_GOOD) {
$action = 'create';
}
} elseif ($chg_status) {
$status_to_action = array(self::$STATUS_GOOD => 'activate', self::$STATUS_BOUNCED => 'bounce', self::$STATUS_CONFIRMED_BAD => 'trash', self::$STATUS_UNSUBSCRIBED => 'unsubscribe');
if (isset($status_to_action[$chg_status])) {
$action = $status_to_action[$chg_status];
} else {
throw new Exception("Unknown sem_status value {$chg_status}");
}
}
// queue it up
if ($action) {
// mailchimp
$cmd = sprintf("PERL AIR2_ROOT/bin/mailchimp-status-sync --src_id %s", $this->sem_src_id);
$job = new JobQueue();
$job->jq_job = $cmd;
$job->save();
}
}
/**
* Recycle or destroy any jobs that have been claimed for too long
*
* @return integer Number of jobs recycled/deleted
* @throws MWException
*/
public function recycleAndDeleteStaleJobs()
{
if ($this->claimTTL <= 0) {
// sanity
throw new MWException("Cannot recycle jobs since acknowledgements are disabled.");
}
$count = 0;
// For each job item that can be retried, we need to add it back to the
// main queue and remove it from the list of currenty claimed job items.
// For those that cannot, they are marked as dead and kept around for
// investigation and manual job restoration but are eventually deleted.
$conn = $this->getConnection();
try {
$now = time();
static $script = <<<LUA
\t\t\tlocal released,abandoned,pruned = 0,0,0
\t\t\t-- Get all non-dead jobs that have an expired claim on them.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal staleClaims = redis.call('zRangeByScore',KEYS[1],0,ARGV[1])
\t\t\tfor k,id in ipairs(staleClaims) do
\t\t\t\tlocal timestamp = redis.call('zScore',KEYS[1],id)
\t\t\t\tlocal attempts = redis.call('hGet',KEYS[2],id)
\t\t\t\tif attempts < ARGV[3] then
\t\t\t\t\t-- Claim expired and retries left: re-enqueue the job
\t\t\t\t\tredis.call('lPush',KEYS[3],id)
\t\t\t\t\tredis.call('hIncrBy',KEYS[2],id,1)
\t\t\t\t\treleased = released + 1
\t\t\t\telse
\t\t\t\t\t-- Claim expired and no retries left: mark the job as dead
\t\t\t\t\tredis.call('zAdd',KEYS[5],timestamp,id)
\t\t\t\t\tabandoned = abandoned + 1
\t\t\t\tend
\t\t\t\tredis.call('zRem',KEYS[1],id)
\t\t\tend
\t\t\t-- Get all of the dead jobs that have been marked as dead for too long.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal deadClaims = redis.call('zRangeByScore',KEYS[5],0,ARGV[2])
\t\t\tfor k,id in ipairs(deadClaims) do
\t\t\t\t-- Stale and out of retries: remove any traces of the job
\t\t\t\tredis.call('zRem',KEYS[5],id)
\t\t\t\tredis.call('hDel',KEYS[2],id)
\t\t\t\tredis.call('hDel',KEYS[4],id)
\t\t\t\tpruned = pruned + 1
\t\t\tend
\t\t\treturn {released,abandoned,pruned}
LUA;
$res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries), 5);
if ($res) {
list($released, $abandoned, $pruned) = $res;
$count += $released + $pruned;
JobQueue::incrStats('job-recycle', $this->type, $released);
JobQueue::incrStats('job-abandon', $this->type, $abandoned);
}
} catch (RedisException $e) {
$this->throwRedisException($this->server, $conn, $e);
}
return $count;
}
<?php
require_once "../db.inc.php";
require_once "../facilities.inc.php";
class Response
{
var $Percentage;
}
$resp = new Response();
session_start();
if (session_id() === "") {
$resp->Percentage = 0;
} else {
$job = JobQueue::getStatus(session_id());
$resp->SessionID = $job["SessionID"];
$resp->Percentage = $job["Percentage"];
$resp->Status = $job["Status"];
}
header('Content-Type: application/json');
echo json_encode($resp);
/**
* @params include:
* - sectionsByWiki : A map of wiki IDs to section names.
* Wikis will default to using the section "default".
* - partitionsBySection : Map of section names to maps of (partition name => weight).
* A section called 'default' must be defined if not all wikis
* have explicitly defined sections.
* - configByPartition : Map of queue partition names to configuration arrays.
* These configuration arrays are passed to JobQueue::factory().
* The options set here are overriden by those passed to this
* the federated queue itself (e.g. 'order' and 'claimTTL').
* - partitionsNoPush : List of partition names that can handle pop() but not push().
* This can be used to migrate away from a certain partition.
* @param array $params
*/
protected function __construct(array $params)
{
parent::__construct($params);
$this->sectionsByWiki = $params['sectionsByWiki'];
$this->partitionsBySection = $params['partitionsBySection'];
$this->configByPartition = $params['configByPartition'];
if (isset($params['partitionsNoPush'])) {
$this->partitionsNoPush = array_flip($params['partitionsNoPush']);
}
$baseConfig = $params;
foreach (array('class', 'sectionsByWiki', 'partitionsBySection', 'configByPartition', 'partitionsNoPush') as $o) {
unset($baseConfig[$o]);
}
foreach ($this->getPartitionMap() as $partition => $w) {
if (!isset($this->configByPartition[$partition])) {
throw new MWException("No configuration for partition '{$partition}'.");
}
$this->partitionQueues[$partition] = JobQueue::factory($baseConfig + $this->configByPartition[$partition]);
}
// Aggregate cache some per-queue values if there are multiple partition queues
$this->cache = $this->isFederated() ? wfGetMainCache() : new EmptyBagOStuff();
}
function test_get_item_uris()
{
$queue = new JobQueue(dirname(__FILE__) . DIRECTORY_SEPARATOR . 'documents/jobs.rdf');
$queue->request_factory = new FakeJobsHTTPRequestFactory();
$expected = array("http://api.talis.com/stores/kwijibo-dev3/jobs/6c679df3-f137-4cd5-bf4c-3206ad1ecd29");
$actual = $queue->get_item_uris();
$this->assertEquals($expected, $actual);
}
/**
* Returns the data model based on the primary key given in the GET variable.
* If the data model is not found, an HTTP exception will be raised.
* @param integer $id the ID of the model to be loaded
* @return JobQueue the loaded model
* @throws CHttpException
*/
public function loadModel($id)
{
$model = JobQueue::model()->findByPk($id);
if ($model === null) {
throw new CHttpException(404, 'The requested page does not exist.');
}
return $model;
}
/**
* @see JobQueue::doAck()
* @param Job $job
* @return Job|bool
* @throws UnexpectedValueException
* @throws JobQueueError
*/
protected function doAck(Job $job)
{
if (!isset($job->metadata['uuid'])) {
throw new UnexpectedValueException("Job of type '{$job->getType()}' has no UUID.");
}
$uuid = $job->metadata['uuid'];
$conn = $this->getConnection();
try {
static $script = <<<LUA
\t\t\tlocal kClaimed, kAttempts, kData = unpack(KEYS)
\t\t\tlocal uuid = unpack(ARGV)
\t\t\t-- Unmark the job as claimed
\t\t\tredis.call('zRem',kClaimed,uuid)
\t\t\tredis.call('hDel',kAttempts,uuid)
\t\t\t-- Delete the job data itself
\t\t\treturn redis.call('hDel',kData,uuid)
LUA;
$res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('h-data'), $uuid), 3);
if (!$res) {
wfDebugLog('JobQueueRedis', "Could not acknowledge {$this->type} job {$uuid}.");
return false;
}
JobQueue::incrStats('acks', $this->type);
} catch (RedisException $e) {
$this->throwRedisException($conn, $e);
}
return true;
}
/**
* Generate new workbook 'DC_Statistics_<timestamp>.xlsx'
*
* @param array $DProps
* @param PHPExcel $objPHPExcel
* @param string $thisDate
*/
function writeExcelReport(&$DProps, $objPHPExcel, $thisDate)
{
ReportStats::get()->report('Info', 'User: '******'Doc']['User'] . ' Version: ' . $DProps['Doc']['version']);
// Crude status reporting
$config = new Config();
$config->Config();
addColumnIndices($DProps);
// Generate new workbook 'DC_Statistics_<timestamp>.xlsx'
setDocumentProperties($objPHPExcel, $thisDate, $config->ParameterArray['OrgName'], $DProps);
list($DCStats, $Rack_Inv, $limitedUser) = writeDCInventory($DProps, $objPHPExcel, $thisDate);
ReportStats::get()->report('Info', 'DC Inventory');
if (!$limitedUser) {
writeDCStatsSummary($DProps, $objPHPExcel, $DCStats, $thisDate);
ReportStats::get()->report('Info', 'DC Stats');
}
writeRackInventory($DProps, $objPHPExcel, $Rack_Inv, $thisDate);
ReportStats::get()->report('Info', 'Rack Inventory');
writeFrontPage($DProps, $config, $objPHPExcel, $thisDate);
ReportStats::get()->report('Info', 'Front Page');
JobQueue::updateStatus(session_id(), "Preparing to transmit file");
}
/**
* Pop a job off of the queue.
* This requires $wgJobClasses to be set for the given job type.
* Outside callers should use JobQueueGroup::pop() instead of this function.
*
* @throws MWException
* @return Job|bool Returns false if there are no jobs
*/
public final function pop()
{
global $wgJobClasses;
if ($this->wiki !== wfWikiID()) {
throw new MWException("Cannot pop '{$this->type}' job off foreign wiki queue.");
} elseif (!isset($wgJobClasses[$this->type])) {
// Do not pop jobs if there is no class for the queue type
throw new MWException("Unrecognized job type '{$this->type}'.");
}
wfProfileIn(__METHOD__);
$job = $this->doPop();
wfProfileOut(__METHOD__);
// Flag this job as an old duplicate based on its "root" job...
try {
if ($job && $this->isRootJobOldDuplicate($job)) {
JobQueue::incrStats('job-pop-duplicate', $this->type, 1, $this->wiki);
$job = DuplicateJob::newFromJob($job);
// convert to a no-op
}
} catch (MWException $e) {
// don't lose jobs over this
}
return $job;
}
/**
* Recycle or destroy any jobs that have been claimed for too long
* and release any ready delayed jobs into the queue
*
* @return int Number of jobs recycled/deleted/undelayed
* @throws MWException|JobQueueError
*/
public function recyclePruneAndUndelayJobs()
{
$count = 0;
// For each job item that can be retried, we need to add it back to the
// main queue and remove it from the list of currenty claimed job items.
// For those that cannot, they are marked as dead and kept around for
// investigation and manual job restoration but are eventually deleted.
$conn = $this->getConnection();
try {
$now = time();
static $script = <<<LUA
\t\t\tlocal kClaimed, kAttempts, kUnclaimed, kData, kAbandoned, kDelayed = unpack(KEYS)
\t\t\tlocal released,abandoned,pruned,undelayed = 0,0,0,0
\t\t\t-- Get all non-dead jobs that have an expired claim on them.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal staleClaims = redis.call('zRangeByScore',kClaimed,0,ARGV[1])
\t\t\tfor k,id in ipairs(staleClaims) do
\t\t\t\tlocal timestamp = redis.call('zScore',kClaimed,id)
\t\t\t\tlocal attempts = redis.call('hGet',kAttempts,id)
\t\t\t\tif attempts < ARGV[3] then
\t\t\t\t\t-- Claim expired and retries left: re-enqueue the job
\t\t\t\t\tredis.call('lPush',kUnclaimed,id)
\t\t\t\t\tredis.call('hIncrBy',kAttempts,id,1)
\t\t\t\t\treleased = released + 1
\t\t\t\telse
\t\t\t\t\t-- Claim expired and no retries left: mark the job as dead
\t\t\t\t\tredis.call('zAdd',kAbandoned,timestamp,id)
\t\t\t\t\tabandoned = abandoned + 1
\t\t\t\tend
\t\t\t\tredis.call('zRem',kClaimed,id)
\t\t\tend
\t\t\t-- Get all of the dead jobs that have been marked as dead for too long.
\t\t\t-- The score for each item is the last claim timestamp (UNIX).
\t\t\tlocal deadClaims = redis.call('zRangeByScore',kAbandoned,0,ARGV[2])
\t\t\tfor k,id in ipairs(deadClaims) do
\t\t\t\t-- Stale and out of retries: remove any traces of the job
\t\t\t\tredis.call('zRem',kAbandoned,id)
\t\t\t\tredis.call('hDel',kAttempts,id)
\t\t\t\tredis.call('hDel',kData,id)
\t\t\t\tpruned = pruned + 1
\t\t\tend
\t\t\t-- Get the list of ready delayed jobs, sorted by readiness (UNIX timestamp)
\t\t\tlocal ids = redis.call('zRangeByScore',kDelayed,0,ARGV[4])
\t\t\t-- Migrate the jobs from the "delayed" set to the "unclaimed" list
\t\t\tfor k,id in ipairs(ids) do
\t\t\t\tredis.call('lPush',kUnclaimed,id)
\t\t\t\tredis.call('zRem',kDelayed,id)
\t\t\tend
\t\t\tundelayed = #ids
\t\t\treturn {released,abandoned,pruned,undelayed}
LUA;
$res = $conn->luaEval($script, array($this->getQueueKey('z-claimed'), $this->getQueueKey('h-attempts'), $this->getQueueKey('l-unclaimed'), $this->getQueueKey('h-data'), $this->getQueueKey('z-abandoned'), $this->getQueueKey('z-delayed'), $now - $this->claimTTL, $now - self::MAX_AGE_PRUNE, $this->maxTries, $now), 6);
if ($res) {
list($released, $abandoned, $pruned, $undelayed) = $res;
$count += $released + $pruned + $undelayed;
JobQueue::incrStats('job-recycle', $this->type, $released);
JobQueue::incrStats('job-abandon', $this->type, $abandoned);
}
} catch (RedisException $e) {
$this->throwRedisException($conn, $e);
}
return $count;
}
/**
* @return array
*/
protected function getCoalescedQueues()
{
global $wgJobTypeConf;
if ($this->coalescedQueues === null) {
$this->coalescedQueues = array();
foreach ($wgJobTypeConf as $type => $conf) {
$queue = JobQueue::factory(array('wiki' => $this->wiki, 'type' => 'null') + $conf);
$loc = $queue->getCoalesceLocationInternal();
if (!isset($this->coalescedQueues[$loc])) {
$this->coalescedQueues[$loc]['queue'] = $queue;
$this->coalescedQueues[$loc]['types'] = array();
}
if ($type === 'default') {
$this->coalescedQueues[$loc]['types'] = array_merge($this->coalescedQueues[$loc]['types'], array_diff($this->getQueueTypes(), array_keys($wgJobTypeConf)));
} else {
$this->coalescedQueues[$loc]['types'][] = $type;
}
}
}
return $this->coalescedQueues;
}
/**
* Schedule a job_queue to export any submissions in this bin to an excel
* spreadsheet, and email it to the user.
*
* @param User $usr
* @return JobQueue $job
*/
public function queue_xls_export($usr)
{
$cmd = sprintf("PERL AIR2_ROOT/bin/xls-export.pl --user_id=%d --bin_id=%d --format=email --logging", $usr->user_id, $this->bin_id);
foreach ($extra as $param => $bool) {
if (is_int($param) && is_string($bool)) {
$cmd .= " --{$bool}";
} else {
$val = $bool ? '1' : '0';
$cmd .= " --{$param}={$val}";
}
}
// check for email address
if ($usr->UserEmailAddress->count() == 0) {
throw new Exception('User must have an email to export');
}
$job = new JobQueue();
$job->jq_job = $cmd;
$job->save();
return $job;
}
?>
" >
<i class="fa fa-eye"></i>
View
</a>
</li>
</ul>
</div>
</div>
<h5>
<small>
<b>Queue Name</b> <br>
</small>
<strong>
<?php
$model = JobQueue::model()->findByAttributes(array('queue_id' => $data->queue_id));
if ($model) {
echo basename($model->queue_name);
} else {
echo "Not Available";
}
?>
</strong>
<br>
<small>
<b>Date Uploaded</b> <br>
</small>
<strong>
<?php
echo date("jS \\of F Y h:i:s A", strtotime($data->date_created));
?>
/**
* @see JobQueue::doPop()
* @return Job|bool
* @throws JobQueueError
*/
protected function doPop()
{
$job = false;
$conn = $this->getConnection();
try {
do {
$blob = $this->popAndAcquireBlob($conn);
if (!is_string($blob)) {
break;
// no jobs; nothing to do
}
JobQueue::incrStats('job-pop', $this->type, 1, $this->wiki);
$item = $this->unserialize($blob);
if ($item === false) {
wfDebugLog('JobQueueRedis', "Could not unserialize {$this->type} job.");
continue;
}
// If $item is invalid, the runner loop recyling will cleanup as needed
$job = $this->getJobFromFields($item);
// may be false
} while (!$job);
// job may be false if invalid
} catch (RedisException $e) {
$this->throwRedisException($conn, $e);
}
return $job;
}
