public function execute()
{
$task_table = new PhabricatorWorkerActiveTask();
$conn_r = $task_table->establishConnection('r');
$rows = queryfx_all($conn_r, 'SELECT * FROM %T %Q %Q %Q', $task_table->getTableName(), $this->buildWhereClause($conn_r), $this->buildOrderClause($conn_r), $this->buildLimitClause($conn_r));
return $task_table->loadAllFromArray($rows);
}
public function save()
{
if ($this->getID() === null) {
throw new Exception(pht('Trying to archive a task with no ID.'));
}
$other = new PhabricatorWorkerActiveTask();
$conn_w = $this->establishConnection('w');
$this->openTransaction();
queryfx($conn_w, 'DELETE FROM %T WHERE id = %d', $other->getTableName(), $this->getID());
$result = parent::insert();
$this->saveTransaction();
return $result;
}
public function execute()
{
if (!$this->limit) {
throw new Exception('You must setLimit() when leasing tasks.');
}
$task_table = new PhabricatorWorkerActiveTask();
$taskdata_table = new PhabricatorWorkerTaskData();
$lease_ownership_name = $this->getLeaseOwnershipName();
$conn_w = $task_table->establishConnection('w');
// Try to satisfy the request from new, unleased tasks first. If we don't
// find enough tasks, try tasks with expired leases (i.e., tasks which have
// previously failed).
$phases = array(self::PHASE_UNLEASED, self::PHASE_EXPIRED);
$limit = $this->limit;
$leased = 0;
foreach ($phases as $phase) {
// NOTE: If we issue `UPDATE ... WHERE ... ORDER BY id ASC`, the query
// goes very, very slowly. The `ORDER BY` triggers this, although we get
// the same apparent results without it. Without the ORDER BY, binary
// read slaves complain that the query isn't repeatable. To avoid both
// problems, do a SELECT and then an UPDATE.
$rows = queryfx_all($conn_w, 'SELECT id, leaseOwner FROM %T %Q %Q %Q', $task_table->getTableName(), $this->buildWhereClause($conn_w, $phase), $this->buildOrderClause($conn_w, $phase), $this->buildLimitClause($conn_w, $limit - $leased));
// NOTE: Sometimes, we'll race with another worker and they'll grab
// this task before we do. We could reduce how often this happens by
// selecting more tasks than we need, then shuffling them and trying
// to lock only the number we're actually after. However, the amount
// of time workers spend here should be very small relative to their
// total runtime, so keep it simple for the moment.
if ($rows) {
queryfx($conn_w, 'UPDATE %T task
SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + %d
%Q', $task_table->getTableName(), $lease_ownership_name, self::getDefaultLeaseDuration(), $this->buildUpdateWhereClause($conn_w, $phase, $rows));
$leased += $conn_w->getAffectedRows();
if ($leased == $limit) {
break;
}
}
}
if (!$leased) {
return array();
}
$data = queryfx_all($conn_w, 'SELECT task.*, taskdata.data _taskData, UNIX_TIMESTAMP() _serverTime
FROM %T task LEFT JOIN %T taskdata
ON taskdata.id = task.dataID
WHERE leaseOwner = %s AND leaseExpires > UNIX_TIMESTAMP()
%Q %Q', $task_table->getTableName(), $taskdata_table->getTableName(), $lease_ownership_name, $this->buildOrderClause($conn_w, $phase), $this->buildLimitClause($conn_w, $limit));
$tasks = $task_table->loadAllFromArray($data);
$tasks = mpull($tasks, null, 'getID');
foreach ($data as $row) {
$tasks[$row['id']]->setServerTime($row['_serverTime']);
if ($row['_taskData']) {
$task_data = json_decode($row['_taskData'], true);
} else {
$task_data = null;
}
$tasks[$row['id']]->setData($task_data);
}
return $tasks;
}
<?php
// Switch PhabricatorWorkerActiveTask from auto-increment IDs to counter IDs.
// Set the initial counter ID to be larger than any known task ID.
$active_table = new PhabricatorWorkerActiveTask();
$archive_table = new PhabricatorWorkerArchiveTask();
$old_table = 'worker_task';
$conn_w = $active_table->establishConnection('w');
$active_auto = head(queryfx_one($conn_w, 'SELECT auto_increment FROM information_schema.tables
WHERE table_name = %s
AND table_schema = DATABASE()', $old_table));
$active_max = head(queryfx_one($conn_w, 'SELECT MAX(id) FROM %T', $old_table));
$archive_max = head(queryfx_one($conn_w, 'SELECT MAX(id) FROM %T', $archive_table->getTableName()));
$initial_counter = max((int) $active_auto, (int) $active_max, (int) $archive_max);
queryfx($conn_w, 'INSERT INTO %T (counterName, counterValue)
VALUES (%s, %d)
ON DUPLICATE KEY UPDATE counterValue = %d', LiskDAO::COUNTER_TABLE_NAME, $old_table, $initial_counter + 1, $initial_counter + 1);
/**
* Wait for tasks to complete. If tasks are not leased by other workers, they
* will be executed in this process while waiting.
*
* @param list<int> List of queued task IDs to wait for.
* @return void
*/
public static final function waitForTasks(array $task_ids)
{
if (!$task_ids) {
return;
}
$task_table = new PhabricatorWorkerActiveTask();
$waiting = array_fuse($task_ids);
while ($waiting) {
$conn_w = $task_table->establishConnection('w');
// Check if any of the tasks we're waiting on are still queued. If they
// are not, we're done waiting.
$row = queryfx_one($conn_w, 'SELECT COUNT(*) N FROM %T WHERE id IN (%Ld)', $task_table->getTableName(), $waiting);
if (!$row['N']) {
// Nothing is queued anymore. Stop waiting.
break;
}
$tasks = id(new PhabricatorWorkerLeaseQuery())->withIDs($waiting)->setLimit(1)->execute();
if (!$tasks) {
// We were not successful in leasing anything. Sleep for a bit and
// see if we have better luck later.
sleep(1);
continue;
}
$task = head($tasks)->executeTask();
$ex = $task->getExecutionException();
if ($ex) {
throw $ex;
}
}
$tasks = id(new PhabricatorWorkerArchiveTaskQuery())->withIDs($task_ids)->execute();
foreach ($tasks as $task) {
if ($task->getResult() != PhabricatorWorkerArchiveTask::RESULT_SUCCESS) {
throw new Exception(pht('Task %d failed!', $task->getID()));
}
}
}
/**
* Awaken tasks that have yielded.
*
* Reschedules the specified tasks if they are currently queued in a yielded,
* unleased, unretried state so they'll execute sooner. This can let the
* queue avoid unnecessary waits.
*
* This method does not provide any assurances about when these tasks will
* execute, or even guarantee that it will have any effect at all.
*
* @param list<id> List of task IDs to try to awaken.
* @return void
*/
public static final function awakenTaskIDs(array $ids)
{
if (!$ids) {
return;
}
$table = new PhabricatorWorkerActiveTask();
$conn_w = $table->establishConnection('w');
// NOTE: At least for now, we're keeping these tasks yielded, just
// pretending that they threw a shorter yield than they really did.
// Overlap the windows here to handle minor client/server time differences
// and because it's likely correct to push these tasks to the head of their
// respective priorities. There is a good chance they are ready to execute.
$window = phutil_units('1 hour in seconds');
$epoch_ago = PhabricatorTime::getNow() - $window;
queryfx($conn_w, 'UPDATE %T SET leaseExpires = %d
WHERE id IN (%Ld)
AND leaseOwner = %s
AND leaseExpires > %d
AND failureCount = 0', $table->getTableName(), $epoch_ago, $ids, self::YIELD_OWNER, $epoch_ago);
}
public function handleRequest(AphrontRequest $request)
{
$viewer = $this->getViewer();
$window_start = time() - 60 * 15;
// Assume daemons spend about 250ms second in overhead per task acquiring
// leases and doing other bookkeeping. This is probably an over-estimation,
// but we'd rather show that utilization is too high than too low.
$lease_overhead = 0.25;
$completed = id(new PhabricatorWorkerArchiveTaskQuery())->withDateModifiedSince($window_start)->execute();
$failed = id(new PhabricatorWorkerActiveTask())->loadAllWhere('failureTime > %d', $window_start);
$usage_total = 0;
$usage_start = PHP_INT_MAX;
$completed_info = array();
foreach ($completed as $completed_task) {
$class = $completed_task->getTaskClass();
if (empty($completed_info[$class])) {
$completed_info[$class] = array('n' => 0, 'duration' => 0);
}
$completed_info[$class]['n']++;
$duration = $completed_task->getDuration();
$completed_info[$class]['duration'] += $duration;
// NOTE: Duration is in microseconds, but we're just using seconds to
// compute utilization.
$usage_total += $lease_overhead + $duration / 1000000;
$usage_start = min($usage_start, $completed_task->getDateModified());
}
$completed_info = isort($completed_info, 'n');
$rows = array();
foreach ($completed_info as $class => $info) {
$rows[] = array($class, number_format($info['n']), pht('%s us', new PhutilNumber((int) ($info['duration'] / $info['n']))));
}
if ($failed) {
// Add the time it takes to restart the daemons. This includes a guess
// about other overhead of 2X.
$restart_delay = PhutilDaemonHandle::getWaitBeforeRestart();
$usage_total += $restart_delay * count($failed) * 2;
foreach ($failed as $failed_task) {
$usage_start = min($usage_start, $failed_task->getFailureTime());
}
$rows[] = array(phutil_tag('em', array(), pht('Temporary Failures')), count($failed), null);
}
$logs = id(new PhabricatorDaemonLogQuery())->setViewer($viewer)->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE)->setAllowStatusWrites(true)->execute();
$taskmasters = 0;
foreach ($logs as $log) {
if ($log->getDaemon() == 'PhabricatorTaskmasterDaemon') {
$taskmasters++;
}
}
if ($taskmasters && $usage_total) {
// Total number of wall-time seconds the daemons have been running since
// the oldest event. For very short times round up to 15s so we don't
// render any ridiculous numbers if you reload the page immediately after
// restarting the daemons.
$available_time = $taskmasters * max(15, time() - $usage_start);
// Percentage of those wall-time seconds we can account for, which the
// daemons spent doing work:
$used_time = $usage_total / $available_time;
$rows[] = array(phutil_tag('em', array(), pht('Queue Utilization (Approximate)')), sprintf('%.1f%%', 100 * $used_time), null);
}
$completed_table = new AphrontTableView($rows);
$completed_table->setNoDataString(pht('No tasks have completed in the last 15 minutes.'));
$completed_table->setHeaders(array(pht('Class'), pht('Count'), pht('Avg')));
$completed_table->setColumnClasses(array('wide', 'n', 'n'));
$completed_panel = id(new PHUIObjectBoxView())->setHeaderText(pht('Recently Completed Tasks (Last 15m)'))->setTable($completed_table);
$daemon_table = id(new PhabricatorDaemonLogListView())->setUser($viewer)->setDaemonLogs($logs);
$daemon_panel = id(new PHUIObjectBoxView())->setHeaderText(pht('Active Daemons'))->setTable($daemon_table);
$tasks = id(new PhabricatorWorkerLeaseQuery())->setSkipLease(true)->withLeasedTasks(true)->setLimit(100)->execute();
$tasks_table = id(new PhabricatorDaemonTasksTableView())->setTasks($tasks)->setNoDataString(pht('No tasks are leased by workers.'));
$leased_panel = id(new PHUIObjectBoxView())->setHeaderText(pht('Leased Tasks'))->setTable($tasks_table);
$task_table = new PhabricatorWorkerActiveTask();
$queued = queryfx_all($task_table->establishConnection('r'), 'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass
ORDER BY N DESC', $task_table->getTableName());
$rows = array();
foreach ($queued as $row) {
$rows[] = array($row['taskClass'], number_format($row['N']));
}
$queued_table = new AphrontTableView($rows);
$queued_table->setHeaders(array(pht('Class'), pht('Count')));
$queued_table->setColumnClasses(array('wide', 'n'));
$queued_table->setNoDataString(pht('Task queue is empty.'));
$queued_panel = new PHUIObjectBoxView();
$queued_panel->setHeaderText(pht('Queued Tasks'));
$queued_panel->setTable($queued_table);
$upcoming = id(new PhabricatorWorkerLeaseQuery())->setLimit(10)->setSkipLease(true)->execute();
$upcoming_panel = id(new PHUIObjectBoxView())->setHeaderText(pht('Next In Queue'))->setTable(id(new PhabricatorDaemonTasksTableView())->setTasks($upcoming)->setNoDataString(pht('Task queue is empty.')));
$triggers = id(new PhabricatorWorkerTriggerQuery())->setViewer($viewer)->setOrder(PhabricatorWorkerTriggerQuery::ORDER_EXECUTION)->withNextEventBetween(0, null)->needEvents(true)->setLimit(10)->execute();
$triggers_table = $this->buildTriggersTable($triggers);
$triggers_panel = id(new PHUIObjectBoxView())->setHeaderText(pht('Upcoming Triggers'))->setTable($triggers_table);
$crumbs = $this->buildApplicationCrumbs();
$crumbs->addTextCrumb(pht('Console'));
$nav = $this->buildSideNavView();
$nav->selectFilter('/');
$nav->appendChild(array($crumbs, $completed_panel, $daemon_panel, $queued_panel, $leased_panel, $upcoming_panel, $triggers_panel));
return $this->newPage()->setTitle(pht('Console'))->appendChild($nav);
}
public function processRequest()
{
$request = $this->getRequest();
$user = $request->getUser();
$window_start = time() - 60 * 15;
// Assume daemons spend about 250ms second in overhead per task acquiring
// leases and doing other bookkeeping. This is probably an over-estimation,
// but we'd rather show that utilization is too high than too low.
$lease_overhead = 0.25;
$completed = id(new PhabricatorWorkerArchiveTask())->loadAllWhere('dateModified > %d', $window_start);
$failed = id(new PhabricatorWorkerActiveTask())->loadAllWhere('failureTime > %d', $window_start);
$usage_total = 0;
$usage_start = PHP_INT_MAX;
$completed_info = array();
foreach ($completed as $completed_task) {
$class = $completed_task->getTaskClass();
if (empty($completed_info[$class])) {
$completed_info[$class] = array('n' => 0, 'duration' => 0);
}
$completed_info[$class]['n']++;
$duration = $completed_task->getDuration();
$completed_info[$class]['duration'] += $duration;
// NOTE: Duration is in microseconds, but we're just using seconds to
// compute utilization.
$usage_total += $lease_overhead + $duration / 1000000;
$usage_start = min($usage_start, $completed_task->getDateModified());
}
$completed_info = isort($completed_info, 'n');
$rows = array();
foreach ($completed_info as $class => $info) {
$rows[] = array($class, number_format($info['n']), number_format((int) ($info['duration'] / $info['n'])) . ' us');
}
if ($failed) {
// Add the time it takes to restart the daemons. This includes a guess
// about other overhead of 2X.
$usage_total += PhutilDaemonOverseer::RESTART_WAIT * count($failed) * 2;
foreach ($failed as $failed_task) {
$usage_start = min($usage_start, $failed_task->getFailureTime());
}
$rows[] = array(phutil_tag('em', array(), pht('Temporary Failures')), count($failed), null);
}
$logs = id(new PhabricatorDaemonLogQuery())->setViewer($user)->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE)->setAllowStatusWrites(true)->execute();
$taskmasters = 0;
foreach ($logs as $log) {
if ($log->getDaemon() == 'PhabricatorTaskmasterDaemon') {
$taskmasters++;
}
}
if ($taskmasters && $usage_total) {
// Total number of wall-time seconds the daemons have been running since
// the oldest event. For very short times round up to 15s so we don't
// render any ridiculous numbers if you reload the page immediately after
// restarting the daemons.
$available_time = $taskmasters * max(15, time() - $usage_start);
// Percentage of those wall-time seconds we can account for, which the
// daemons spent doing work:
$used_time = $usage_total / $available_time;
$rows[] = array(phutil_tag('em', array(), pht('Queue Utilization (Approximate)')), sprintf('%.1f%%', 100 * $used_time), null);
}
$completed_table = new AphrontTableView($rows);
$completed_table->setNoDataString(pht('No tasks have completed in the last 15 minutes.'));
$completed_table->setHeaders(array(pht('Class'), pht('Count'), pht('Avg')));
$completed_table->setColumnClasses(array('wide', 'n', 'n'));
$completed_panel = new PHUIObjectBoxView();
$completed_panel->setHeaderText(pht('Recently Completed Tasks (Last 15m)'));
$completed_panel->appendChild($completed_table);
$daemon_table = new PhabricatorDaemonLogListView();
$daemon_table->setUser($user);
$daemon_table->setDaemonLogs($logs);
$tasks = id(new PhabricatorWorkerActiveTask())->loadAllWhere('leaseOwner IS NOT NULL');
$rows = array();
foreach ($tasks as $task) {
$rows[] = array($task->getID(), $task->getTaskClass(), $task->getLeaseOwner(), $task->getLeaseExpires() - time(), $task->getPriority(), $task->getFailureCount(), phutil_tag('a', array('href' => '/daemon/task/' . $task->getID() . '/', 'class' => 'button small grey'), pht('View Task')));
}
$daemon_panel = new PHUIObjectBoxView();
$daemon_panel->setHeaderText(pht('Active Daemons'));
$daemon_panel->appendChild($daemon_table);
$leased_table = new AphrontTableView($rows);
$leased_table->setHeaders(array(pht('ID'), pht('Class'), pht('Owner'), pht('Expires'), pht('Priority'), pht('Failures'), ''));
$leased_table->setColumnClasses(array('n', 'wide', '', '', 'n', 'n', 'action'));
$leased_table->setNoDataString(pht('No tasks are leased by workers.'));
$leased_panel = new PHUIObjectBoxView();
$leased_panel->setHeaderText(pht('Leased Tasks'));
$leased_panel->appendChild($leased_table);
$task_table = new PhabricatorWorkerActiveTask();
$queued = queryfx_all($task_table->establishConnection('r'), 'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass
ORDER BY N DESC', $task_table->getTableName());
$rows = array();
foreach ($queued as $row) {
$rows[] = array($row['taskClass'], number_format($row['N']));
}
$queued_table = new AphrontTableView($rows);
$queued_table->setHeaders(array(pht('Class'), pht('Count')));
$queued_table->setColumnClasses(array('wide', 'n'));
$queued_table->setNoDataString(pht('Task queue is empty.'));
$queued_panel = new PHUIObjectBoxView();
$queued_panel->setHeaderText(pht('Queued Tasks'));
$queued_panel->appendChild($queued_table);
$crumbs = $this->buildApplicationCrumbs();
$crumbs->addTextCrumb(pht('Console'));
$nav = $this->buildSideNavView();
$nav->selectFilter('/');
$nav->appendChild(array($crumbs, $completed_panel, $daemon_panel, $queued_panel, $leased_panel));
return $this->buildApplicationPage($nav, array('title' => pht('Console'), 'device' => false));
}
