protected function finishParse() { $commit = $this->commit; PhabricatorSearchCommitIndexer::indexCommit($commit); if ($this->shouldQueueFollowupTasks()) { $task = new PhabricatorWorkerTask(); $task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker'); $task->setData(array('commitID' => $commit->getID())); $task->save(); } }
public function processRequest() { $logs = id(new PhabricatorDaemonLog())->loadAllWhere('`status` != %s ORDER BY id DESC LIMIT 15', 'exit'); $request = $this->getRequest(); $user = $request->getUser(); $daemon_table = new PhabricatorDaemonLogListView(); $daemon_table->setUser($user); $daemon_table->setDaemonLogs($logs); $daemon_panel = new AphrontPanelView(); $daemon_panel->setHeader('Recently Launched Daemons'); $daemon_panel->appendChild($daemon_table); $tasks = id(new PhabricatorWorkerTask())->loadAllWhere('leaseOwner IS NOT NULL'); $rows = array(); foreach ($tasks as $task) { $rows[] = array($task->getID(), $task->getTaskClass(), $task->getLeaseOwner(), $task->getLeaseExpires() - time(), $task->getFailureCount(), phutil_render_tag('a', array('href' => '/daemon/task/' . $task->getID() . '/', 'class' => 'button small grey'), 'View Task')); } $leased_table = new AphrontTableView($rows); $leased_table->setHeaders(array('ID', 'Class', 'Owner', 'Expires', 'Failures', '')); $leased_table->setColumnClasses(array('n', 'wide', '', '', 'n', 'action')); $leased_table->setNoDataString('No tasks are leased by workers.'); $leased_panel = new AphrontPanelView(); $leased_panel->setHeader('Leased Tasks'); $leased_panel->appendChild($leased_table); $task_table = new PhabricatorWorkerTask(); $queued = queryfx_all($task_table->establishConnection('r'), 'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass ORDER BY N DESC', $task_table->getTableName()); $rows = array(); foreach ($queued as $row) { $rows[] = array(phutil_escape_html($row['taskClass']), number_format($row['N'])); } $queued_table = new AphrontTableView($rows); $queued_table->setHeaders(array('Class', 'Count')); $queued_table->setColumnClasses(array('wide', 'n')); $queued_table->setNoDataString('Task queue is empty.'); $queued_panel = new AphrontPanelView(); $queued_panel->setHeader('Queued Tasks'); $queued_panel->appendChild($queued_table); $cursors = id(new PhabricatorTimelineCursor())->loadAll(); $rows = array(); foreach ($cursors as $cursor) { $rows[] = array(phutil_escape_html($cursor->getName()), number_format($cursor->getPosition())); } $cursor_table = new AphrontTableView($rows); $cursor_table->setHeaders(array('Name', 'Position')); $cursor_table->setColumnClasses(array('wide', 'n')); $cursor_table->setNoDataString('No timeline cursors exist.'); $cursor_panel = new AphrontPanelView(); $cursor_panel->setHeader('Timeline Cursors'); $cursor_panel->appendChild($cursor_table); $nav = $this->buildSideNavView(); $nav->selectFilter(''); $nav->appendChild(array($daemon_panel, $cursor_panel, $queued_panel, $leased_panel)); return $this->buildApplicationPage($nav, array('title' => 'Console')); }
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit) { // NOTE: %B was introduced somewhat recently in git's history, so pull // commit message information with %s and %b instead. // Even though we pass --encoding here, git doesn't always succeed, so // we try a little harder, since git *does* tell us what the actual encoding // is correctly (unless it doesn't; encoding is sometimes empty). list($info) = $repository->execxLocalCommand('log -n 1 --encoding=%s --format=%s %s --', 'UTF-8', implode('%x00', array('%e', '%cn', '%ce', '%an', '%ae', '%s%n%n%b')), $commit->getCommitIdentifier()); $parts = explode("", $info); $encoding = array_shift($parts); // See note above - git doesn't always convert the encoding correctly. $do_convert = false; if (strlen($encoding) && strtoupper($encoding) != 'UTF-8') { if (function_exists('mb_convert_encoding')) { $do_convert = true; } } foreach ($parts as $key => $part) { if ($do_convert) { $parts[$key] = mb_convert_encoding($part, 'UTF-8', $encoding); } $parts[$key] = phutil_utf8ize($part); } $committer_name = $parts[0]; $committer_email = $parts[1]; $author_name = $parts[2]; $author_email = $parts[3]; $message = $parts[4]; if (strlen($author_email)) { $author = "{$author_name} <{$author_email}>"; } else { $author = "{$author_name}"; } if (strlen($committer_email)) { $committer = "{$committer_name} <{$committer_email}>"; } else { $committer = "{$committer_name}"; } if ($committer == $author) { $committer = null; } $this->updateCommitData($author, $message, $committer); if ($this->shouldQueueFollowupTasks()) { $task = new PhabricatorWorkerTask(); $task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker'); $task->setData(array('commitID' => $commit->getID())); $task->save(); } }
public final function run() { do { $iterator = new PhabricatorTimelineIterator('cmittask', array('cmit')); foreach ($iterator as $event) { $data = $event->getData(); if (!$data) { // TODO: This event can't be processed, provide some way to // communicate that? continue; } $commit = id(new PhabricatorRepositoryCommit())->load($data['id']); if (!$commit) { // TODO: Same as above. continue; } // TODO: Cache these. $repository = id(new PhabricatorRepository())->load($commit->getRepositoryID()); if (!$repository) { // TODO: As above, although this almost certainly means the user just // deleted the repository and we're correct to ignore the event in // the timeline. continue; } $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; $task = new PhabricatorWorkerTask(); $task->setTaskClass($class); $task->setData(array('commitID' => $commit->getID())); $task->save(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; $task = new PhabricatorWorkerTask(); $task->setTaskClass($class); $task->setData(array('commitID' => $commit->getID())); $task->save(); break; default: throw new Exception("Unknown repository type."); } $this->stillWorking(); } sleep(1); $this->stillWorking(); } while (true); }
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit) { list($stdout) = $repository->execxLocalCommand('log --template %s --rev %s', '{author}\\n{desc}', $commit->getCommitIdentifier()); list($author, $message) = explode("\n", $stdout, 2); $author = phutil_utf8ize($author); $message = phutil_utf8ize($message); $message = trim($message); $this->updateCommitData($author, $message); if ($this->shouldQueueFollowupTasks()) { $task = new PhabricatorWorkerTask(); $task->setTaskClass('PhabricatorRepositoryMercurialCommitChangeParserWorker'); $task->setData(array('commitID' => $commit->getID())); $task->save(); } }
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit) { $uri = $repository->getDetail('remote-uri'); $log = $this->getSVNLogXMLObject($uri, $commit->getCommitIdentifier(), $verbose = false); $entry = $log->logentry[0]; $author = (string) $entry->author; $message = (string) $entry->msg; $this->updateCommitData($author, $message); if ($this->shouldQueueFollowupTasks()) { $task = new PhabricatorWorkerTask(); $task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker'); $task->setData(array('commitID' => $commit->getID())); $task->save(); } }
public function allocate() { $lease = $this->getPendingLease(); $data = array('type' => $this->getResourceType(), 'lease' => $lease->getID()); if ($this->synchronous) { $data['synchronous'] = true; $worker = new DrydockAllocatorWorker($data); $worker->executeTask(); } else { $task = new PhabricatorWorkerTask(); $task->setTaskClass('DrydockAllocatorWorker'); $task->setData($data); $task->save(); } return $lease; }
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit) { // NOTE: %B was introduced somewhat recently in git's history, so pull // commit message information with %s and %b instead. list($info) = $repository->execxLocalCommand("log -n 1 --encoding='UTF-8' --pretty=format:%%an%%x00%%s%%n%%n%%b %s", $commit->getCommitIdentifier()); list($author, $message) = explode("", $info); // Make sure these are valid UTF-8. $author = phutil_utf8ize($author); $message = phutil_utf8ize($message); $message = trim($message); $this->updateCommitData($author, $message); if ($this->shouldQueueFollowupTasks()) { $task = new PhabricatorWorkerTask(); $task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker'); $task->setData(array('commitID' => $commit->getID())); $task->save(); } }
protected function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit) { $affected_paths = PhabricatorOwnerPathQuery::loadAffectedPaths($repository, $commit); $affected_packages = PhabricatorOwnersPackage::loadAffectedPackages($repository, $affected_paths); if ($affected_packages) { $requests = id(new PhabricatorRepositoryAuditRequest())->loadAllWhere('commitPHID = %s', $commit->getPHID()); $requests = mpull($requests, null, 'getAuditorPHID'); foreach ($affected_packages as $package) { $request = idx($requests, $package->getPHID()); if ($request) { // Don't update request if it exists already. continue; } if ($package->getAuditingEnabled()) { $reasons = $this->checkAuditReasons($commit, $package); if ($reasons) { $audit_status = PhabricatorAuditStatusConstants::AUDIT_REQUIRED; } else { $audit_status = PhabricatorAuditStatusConstants::AUDIT_NOT_REQUIRED; } } else { $reasons = array(); $audit_status = PhabricatorAuditStatusConstants::NONE; } $relationship = new PhabricatorRepositoryAuditRequest(); $relationship->setAuditorPHID($package->getPHID()); $relationship->setCommitPHID($commit->getPHID()); $relationship->setAuditReasons($reasons); $relationship->setAuditStatus($audit_status); $relationship->save(); $requests[$package->getPHID()] = $relationship; } $commit->updateAuditStatus($requests); $commit->save(); } if ($this->shouldQueueFollowupTasks()) { $herald_task = new PhabricatorWorkerTask(); $herald_task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker'); $herald_task->setData(array('commitID' => $commit->getID())); $herald_task->save(); } }
public function delete() { $this->openTransaction(); if ($this->getDataID()) { $conn_w = $this->establishConnection('w'); $data_table = new PhabricatorWorkerTaskData(); queryfx($conn_w, 'DELETE FROM %T WHERE id = %d', $data_table->getTableName(), $this->getDataID()); } $result = parent::delete(); $this->saveTransaction(); return $result; }
public function forceSaveWithoutLease() { $is_new = !$this->getID(); if ($is_new) { $this->failureCount = 0; } if ($is_new && $this->getData() !== null) { $data = new PhabricatorWorkerTaskData(); $data->setData($this->getData()); $data->save(); $this->setDataID($data->getID()); } return parent::save(); }
public function getWaitBeforeRetry(PhabricatorWorkerTask $task) { return max($task->getFailureCount(), 1) * 60; }
} if ($reparse_change) { $classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker'; } break; } if ($reparse_herald) { $classes[] = 'PhabricatorRepositoryCommitHeraldWorker'; } if ($reparse_owners) { $classes[] = 'PhabricatorRepositoryCommitOwnersWorker'; } $spec = array('commitID' => $commit->getID(), 'only' => true); if ($is_all) { foreach ($classes as $class) { $task = new PhabricatorWorkerTask(); $task->setTaskClass($class); $task->setData($spec); $task->save(); $commit_name = 'r' . $callsign . $commit->getCommitIdentifier(); echo " Queued '{$class}' for commit '{$commit_name}'.\n"; } } else { foreach ($classes as $class) { $worker = newv($class, array($spec)); echo "Running '{$class}'...\n"; $worker->doWork(); } } } echo "\nDone.\n";
private function buildRetryListView(PhabricatorWorkerTask $task) { $view = new PHUIPropertyListView(); $data = id(new PhabricatorWorkerTaskData())->load($task->getDataID()); $task->setData($data->getData()); $worker = $task->getWorkerInstance(); $view->addProperty(pht('Failure Count'), $task->getFailureCount()); $retry_count = $worker->getMaximumRetryCount(); if ($retry_count === null) { $max_retries = phutil_tag('em', array(), pht('Retries Forever')); $retry_count = INF; } else { $max_retries = $retry_count; } $view->addProperty(pht('Maximum Retries'), $max_retries); $projection = clone $task; $projection->makeEphemeral(); $next = array(); for ($ii = $task->getFailureCount(); $ii < $retry_count; $ii++) { $projection->setFailureCount($ii); $next[] = $worker->getWaitBeforeRetry($projection); if (count($next) > 10) { break; } } if ($next) { $cumulative = 0; foreach ($next as $key => $duration) { if ($duration === null) { $duration = 60; } $cumulative += $duration; $next[$key] = phutil_format_relative_time($cumulative); } if ($ii != $retry_count) { $next[] = '...'; } $retries_in = implode(', ', $next); } else { $retries_in = pht('No More Retries'); } $view->addProperty(pht('Retries After'), $retries_in); return $view; }
protected function describeTask(PhabricatorWorkerTask $task) { return pht('Task %d (%s)', $task->getID(), $task->getTaskClass()); }
public function run() { $lease_ownership_name = $this->getLeaseOwnershipName(); $task_table = new PhabricatorWorkerTask(); $taskdata_table = new PhabricatorWorkerTaskData(); $sleep = 0; do { $conn_w = $task_table->establishConnection('w'); queryfx($conn_w, 'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15 WHERE leaseOwner IS NULL LIMIT 1', $task_table->getTableName(), $lease_ownership_name); $rows = $conn_w->getAffectedRows(); if (!$rows) { $rows = queryfx($conn_w, 'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15 WHERE leaseExpires < UNIX_TIMESTAMP() LIMIT 1', $task_table->getTableName(), $lease_ownership_name); $rows = $conn_w->getAffectedRows(); } if ($rows) { $data = queryfx_all($conn_w, 'SELECT task.*, taskdata.data _taskData, UNIX_TIMESTAMP() _serverTime FROM %T task LEFT JOIN %T taskdata ON taskdata.id = task.dataID WHERE leaseOwner = %s AND leaseExpires > UNIX_TIMESTAMP() LIMIT 1', $task_table->getTableName(), $taskdata_table->getTableName(), $lease_ownership_name); $tasks = $task_table->loadAllFromArray($data); $tasks = mpull($tasks, null, 'getID'); $task_data = array(); foreach ($data as $row) { $tasks[$row['id']]->setServerTime($row['_serverTime']); if ($row['_taskData']) { $task_data[$row['id']] = json_decode($row['_taskData'], true); } else { $task_data[$row['id']] = null; } } foreach ($tasks as $task) { // TODO: We should detect if we acquired a task with an expired lease // and log about it / bump up failure count. // TODO: We should detect if we acquired a task with an excessive // failure count and fail it permanently. $data = idx($task_data, $task->getID()); $class = $task->getTaskClass(); try { PhutilSymbolLoader::loadClass($class); if (!is_subclass_of($class, 'PhabricatorWorker')) { throw new Exception("Task class '{$class}' does not extend PhabricatorWorker."); } $worker = newv($class, array($data)); $lease = $worker->getRequiredLeaseTime(); if ($lease !== null) { $task->setLeaseDuration($lease); } $worker->executeTask(); $task->delete(); if ($data !== null) { queryfx($conn_w, 'DELETE FROM %T WHERE id = %d', $taskdata_table->getTableName(), $task->getDataID()); } } catch (Exception $ex) { $task->setFailureCount($task->getFailureCount() + 1); $task->save(); throw $ex; } } $sleep = 0; } else { $sleep = min($sleep + 1, 30); } $this->sleep($sleep); } while (true); }
public function getWaitBeforeRetry(PhabricatorWorkerTask $task) { return $task->getFailureCount() * 15; }
/** * See @{method:getMaximumRetryCount} for a description of Doorkeeper * retry defaults. */ public function getWaitBeforeRetry(PhabricatorWorkerTask $task) { $count = $task->getFailureCount(); return 5 * 60 * pow(8, $count); }
private function insertTask(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, $data = array()) { $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; break; default: throw new Exception("Unknown repository type '{$vcs}'!"); } $task = new PhabricatorWorkerTask(); $task->setTaskClass($class); $data['commitID'] = $commit->getID(); $task->setData($data); $task->save(); }
if (!$repository) { throw new Exception("No such repository exists!"); } $repositories = array($repository->getID() => $repository); echo "Loading commits in '{$callsign}' repository...\n"; $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere('repositoryID = %d', $repository->getID()); } echo "Inserting tasks for " . count($commits) . " commits"; foreach ($commits as $commit) { echo "."; $id = $commit->getID(); $repo = idx($repositories, $commit->getRepositoryID()); if (!$repo) { echo "\nWarning: Commit #{$id} has an invalid repository ID.\n"; } switch ($repo->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $task_class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $task_class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; break; default: throw new Exception("Unknown repository type!"); } $task = new PhabricatorWorkerTask(); $task->setTaskClass($task_class); $task->setData(array('commitID' => $commit->getID(), 'only' => true)); $task->save(); } echo "\nDone.\n";
* * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * After upgrading to/past D1723, the handling of messages queued for delivery * is a bit different. Running this script will take any messages that are * queued for delivery, but don't have a worker task created, and create that * worker task. Without the worker task, the message will just sit at "queued * for delivery" and nothing will happen to it. */ $root = dirname(dirname(dirname(__FILE__))); require_once $root . '/scripts/__init_script__.php'; $messages = id(new PhabricatorMetaMTAMail())->loadAllWhere('status = %s', PhabricatorMetaMTAMail::STATUS_QUEUE); foreach ($messages as $message) { if (!$message->getWorkerTaskID()) { $mailer_task = new PhabricatorWorkerTask(); $mailer_task->setTaskClass('PhabricatorMetaMTAWorker'); $mailer_task->setData($message->getID()); $mailer_task->save(); $message->setWorkerTaskID($mailer_task->getID()); $message->save(); $id = $message->getID(); echo "#{$id}\n"; } }
protected function didWriteData() { parent::didWriteData(); if (!$this->getWorkerTaskID()) { $mailer_task = new PhabricatorWorkerTask(); $mailer_task->setTaskClass('PhabricatorMetaMTAWorker'); $mailer_task->setData($this->getID()); $mailer_task->save(); $this->setWorkerTaskID($mailer_task->getID()); $this->save(); } }