protected function finishParse()
{
$commit = $this->commit;
PhabricatorSearchCommitIndexer::indexCommit($commit);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker');
$task->setData(array('commitID' => $commit->getID()));
$task->save();
}
}
public function processRequest()
{
$logs = id(new PhabricatorDaemonLog())->loadAllWhere('`status` != %s ORDER BY id DESC LIMIT 15', 'exit');
$request = $this->getRequest();
$user = $request->getUser();
$daemon_table = new PhabricatorDaemonLogListView();
$daemon_table->setUser($user);
$daemon_table->setDaemonLogs($logs);
$daemon_panel = new AphrontPanelView();
$daemon_panel->setHeader('Recently Launched Daemons');
$daemon_panel->appendChild($daemon_table);
$tasks = id(new PhabricatorWorkerTask())->loadAllWhere('leaseOwner IS NOT NULL');
$rows = array();
foreach ($tasks as $task) {
$rows[] = array($task->getID(), $task->getTaskClass(), $task->getLeaseOwner(), $task->getLeaseExpires() - time(), $task->getFailureCount(), phutil_render_tag('a', array('href' => '/daemon/task/' . $task->getID() . '/', 'class' => 'button small grey'), 'View Task'));
}
$leased_table = new AphrontTableView($rows);
$leased_table->setHeaders(array('ID', 'Class', 'Owner', 'Expires', 'Failures', ''));
$leased_table->setColumnClasses(array('n', 'wide', '', '', 'n', 'action'));
$leased_table->setNoDataString('No tasks are leased by workers.');
$leased_panel = new AphrontPanelView();
$leased_panel->setHeader('Leased Tasks');
$leased_panel->appendChild($leased_table);
$task_table = new PhabricatorWorkerTask();
$queued = queryfx_all($task_table->establishConnection('r'), 'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass
ORDER BY N DESC', $task_table->getTableName());
$rows = array();
foreach ($queued as $row) {
$rows[] = array(phutil_escape_html($row['taskClass']), number_format($row['N']));
}
$queued_table = new AphrontTableView($rows);
$queued_table->setHeaders(array('Class', 'Count'));
$queued_table->setColumnClasses(array('wide', 'n'));
$queued_table->setNoDataString('Task queue is empty.');
$queued_panel = new AphrontPanelView();
$queued_panel->setHeader('Queued Tasks');
$queued_panel->appendChild($queued_table);
$cursors = id(new PhabricatorTimelineCursor())->loadAll();
$rows = array();
foreach ($cursors as $cursor) {
$rows[] = array(phutil_escape_html($cursor->getName()), number_format($cursor->getPosition()));
}
$cursor_table = new AphrontTableView($rows);
$cursor_table->setHeaders(array('Name', 'Position'));
$cursor_table->setColumnClasses(array('wide', 'n'));
$cursor_table->setNoDataString('No timeline cursors exist.');
$cursor_panel = new AphrontPanelView();
$cursor_panel->setHeader('Timeline Cursors');
$cursor_panel->appendChild($cursor_table);
$nav = $this->buildSideNavView();
$nav->selectFilter('');
$nav->appendChild(array($daemon_panel, $cursor_panel, $queued_panel, $leased_panel));
return $this->buildApplicationPage($nav, array('title' => 'Console'));
}
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
// NOTE: %B was introduced somewhat recently in git's history, so pull
// commit message information with %s and %b instead.
// Even though we pass --encoding here, git doesn't always succeed, so
// we try a little harder, since git *does* tell us what the actual encoding
// is correctly (unless it doesn't; encoding is sometimes empty).
list($info) = $repository->execxLocalCommand('log -n 1 --encoding=%s --format=%s %s --', 'UTF-8', implode('%x00', array('%e', '%cn', '%ce', '%an', '%ae', '%s%n%n%b')), $commit->getCommitIdentifier());
$parts = explode("", $info);
$encoding = array_shift($parts);
// See note above - git doesn't always convert the encoding correctly.
$do_convert = false;
if (strlen($encoding) && strtoupper($encoding) != 'UTF-8') {
if (function_exists('mb_convert_encoding')) {
$do_convert = true;
}
}
foreach ($parts as $key => $part) {
if ($do_convert) {
$parts[$key] = mb_convert_encoding($part, 'UTF-8', $encoding);
}
$parts[$key] = phutil_utf8ize($part);
}
$committer_name = $parts[0];
$committer_email = $parts[1];
$author_name = $parts[2];
$author_email = $parts[3];
$message = $parts[4];
if (strlen($author_email)) {
$author = "{$author_name} <{$author_email}>";
} else {
$author = "{$author_name}";
}
if (strlen($committer_email)) {
$committer = "{$committer_name} <{$committer_email}>";
} else {
$committer = "{$committer_name}";
}
if ($committer == $author) {
$committer = null;
}
$this->updateCommitData($author, $message, $committer);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker');
$task->setData(array('commitID' => $commit->getID()));
$task->save();
}
}
public final function run()
{
do {
$iterator = new PhabricatorTimelineIterator('cmittask', array('cmit'));
foreach ($iterator as $event) {
$data = $event->getData();
if (!$data) {
// TODO: This event can't be processed, provide some way to
// communicate that?
continue;
}
$commit = id(new PhabricatorRepositoryCommit())->load($data['id']);
if (!$commit) {
// TODO: Same as above.
continue;
}
// TODO: Cache these.
$repository = id(new PhabricatorRepository())->load($commit->getRepositoryID());
if (!$repository) {
// TODO: As above, although this almost certainly means the user just
// deleted the repository and we're correct to ignore the event in
// the timeline.
continue;
}
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
$task = new PhabricatorWorkerTask();
$task->setTaskClass($class);
$task->setData(array('commitID' => $commit->getID()));
$task->save();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
$task = new PhabricatorWorkerTask();
$task->setTaskClass($class);
$task->setData(array('commitID' => $commit->getID()));
$task->save();
break;
default:
throw new Exception("Unknown repository type.");
}
$this->stillWorking();
}
sleep(1);
$this->stillWorking();
} while (true);
}
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
list($stdout) = $repository->execxLocalCommand('log --template %s --rev %s', '{author}\\n{desc}', $commit->getCommitIdentifier());
list($author, $message) = explode("\n", $stdout, 2);
$author = phutil_utf8ize($author);
$message = phutil_utf8ize($message);
$message = trim($message);
$this->updateCommitData($author, $message);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositoryMercurialCommitChangeParserWorker');
$task->setData(array('commitID' => $commit->getID()));
$task->save();
}
}
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
$uri = $repository->getDetail('remote-uri');
$log = $this->getSVNLogXMLObject($uri, $commit->getCommitIdentifier(), $verbose = false);
$entry = $log->logentry[0];
$author = (string) $entry->author;
$message = (string) $entry->msg;
$this->updateCommitData($author, $message);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker');
$task->setData(array('commitID' => $commit->getID()));
$task->save();
}
}
public function allocate()
{
$lease = $this->getPendingLease();
$data = array('type' => $this->getResourceType(), 'lease' => $lease->getID());
if ($this->synchronous) {
$data['synchronous'] = true;
$worker = new DrydockAllocatorWorker($data);
$worker->executeTask();
} else {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('DrydockAllocatorWorker');
$task->setData($data);
$task->save();
}
return $lease;
}
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
// NOTE: %B was introduced somewhat recently in git's history, so pull
// commit message information with %s and %b instead.
list($info) = $repository->execxLocalCommand("log -n 1 --encoding='UTF-8' --pretty=format:%%an%%x00%%s%%n%%n%%b %s", $commit->getCommitIdentifier());
list($author, $message) = explode("", $info);
// Make sure these are valid UTF-8.
$author = phutil_utf8ize($author);
$message = phutil_utf8ize($message);
$message = trim($message);
$this->updateCommitData($author, $message);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker');
$task->setData(array('commitID' => $commit->getID()));
$task->save();
}
}
protected function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
$affected_paths = PhabricatorOwnerPathQuery::loadAffectedPaths($repository, $commit);
$affected_packages = PhabricatorOwnersPackage::loadAffectedPackages($repository, $affected_paths);
if ($affected_packages) {
$requests = id(new PhabricatorRepositoryAuditRequest())->loadAllWhere('commitPHID = %s', $commit->getPHID());
$requests = mpull($requests, null, 'getAuditorPHID');
foreach ($affected_packages as $package) {
$request = idx($requests, $package->getPHID());
if ($request) {
// Don't update request if it exists already.
continue;
}
if ($package->getAuditingEnabled()) {
$reasons = $this->checkAuditReasons($commit, $package);
if ($reasons) {
$audit_status = PhabricatorAuditStatusConstants::AUDIT_REQUIRED;
} else {
$audit_status = PhabricatorAuditStatusConstants::AUDIT_NOT_REQUIRED;
}
} else {
$reasons = array();
$audit_status = PhabricatorAuditStatusConstants::NONE;
}
$relationship = new PhabricatorRepositoryAuditRequest();
$relationship->setAuditorPHID($package->getPHID());
$relationship->setCommitPHID($commit->getPHID());
$relationship->setAuditReasons($reasons);
$relationship->setAuditStatus($audit_status);
$relationship->save();
$requests[$package->getPHID()] = $relationship;
}
$commit->updateAuditStatus($requests);
$commit->save();
}
if ($this->shouldQueueFollowupTasks()) {
$herald_task = new PhabricatorWorkerTask();
$herald_task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker');
$herald_task->setData(array('commitID' => $commit->getID()));
$herald_task->save();
}
}
public function delete()
{
$this->openTransaction();
if ($this->getDataID()) {
$conn_w = $this->establishConnection('w');
$data_table = new PhabricatorWorkerTaskData();
queryfx($conn_w, 'DELETE FROM %T WHERE id = %d', $data_table->getTableName(), $this->getDataID());
}
$result = parent::delete();
$this->saveTransaction();
return $result;
}
public function forceSaveWithoutLease()
{
$is_new = !$this->getID();
if ($is_new) {
$this->failureCount = 0;
}
if ($is_new && $this->getData() !== null) {
$data = new PhabricatorWorkerTaskData();
$data->setData($this->getData());
$data->save();
$this->setDataID($data->getID());
}
return parent::save();
}
public function getWaitBeforeRetry(PhabricatorWorkerTask $task)
{
return max($task->getFailureCount(), 1) * 60;
}
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker';
}
break;
}
if ($reparse_herald) {
$classes[] = 'PhabricatorRepositoryCommitHeraldWorker';
}
if ($reparse_owners) {
$classes[] = 'PhabricatorRepositoryCommitOwnersWorker';
}
$spec = array('commitID' => $commit->getID(), 'only' => true);
if ($is_all) {
foreach ($classes as $class) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass($class);
$task->setData($spec);
$task->save();
$commit_name = 'r' . $callsign . $commit->getCommitIdentifier();
echo " Queued '{$class}' for commit '{$commit_name}'.\n";
}
} else {
foreach ($classes as $class) {
$worker = newv($class, array($spec));
echo "Running '{$class}'...\n";
$worker->doWork();
}
}
}
echo "\nDone.\n";
private function buildRetryListView(PhabricatorWorkerTask $task)
{
$view = new PHUIPropertyListView();
$data = id(new PhabricatorWorkerTaskData())->load($task->getDataID());
$task->setData($data->getData());
$worker = $task->getWorkerInstance();
$view->addProperty(pht('Failure Count'), $task->getFailureCount());
$retry_count = $worker->getMaximumRetryCount();
if ($retry_count === null) {
$max_retries = phutil_tag('em', array(), pht('Retries Forever'));
$retry_count = INF;
} else {
$max_retries = $retry_count;
}
$view->addProperty(pht('Maximum Retries'), $max_retries);
$projection = clone $task;
$projection->makeEphemeral();
$next = array();
for ($ii = $task->getFailureCount(); $ii < $retry_count; $ii++) {
$projection->setFailureCount($ii);
$next[] = $worker->getWaitBeforeRetry($projection);
if (count($next) > 10) {
break;
}
}
if ($next) {
$cumulative = 0;
foreach ($next as $key => $duration) {
if ($duration === null) {
$duration = 60;
}
$cumulative += $duration;
$next[$key] = phutil_format_relative_time($cumulative);
}
if ($ii != $retry_count) {
$next[] = '...';
}
$retries_in = implode(', ', $next);
} else {
$retries_in = pht('No More Retries');
}
$view->addProperty(pht('Retries After'), $retries_in);
return $view;
}
protected function describeTask(PhabricatorWorkerTask $task)
{
return pht('Task %d (%s)', $task->getID(), $task->getTaskClass());
}
public function run()
{
$lease_ownership_name = $this->getLeaseOwnershipName();
$task_table = new PhabricatorWorkerTask();
$taskdata_table = new PhabricatorWorkerTaskData();
$sleep = 0;
do {
$conn_w = $task_table->establishConnection('w');
queryfx($conn_w, 'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15
WHERE leaseOwner IS NULL LIMIT 1', $task_table->getTableName(), $lease_ownership_name);
$rows = $conn_w->getAffectedRows();
if (!$rows) {
$rows = queryfx($conn_w, 'UPDATE %T SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + 15
WHERE leaseExpires < UNIX_TIMESTAMP() LIMIT 1', $task_table->getTableName(), $lease_ownership_name);
$rows = $conn_w->getAffectedRows();
}
if ($rows) {
$data = queryfx_all($conn_w, 'SELECT task.*, taskdata.data _taskData, UNIX_TIMESTAMP() _serverTime
FROM %T task LEFT JOIN %T taskdata
ON taskdata.id = task.dataID
WHERE leaseOwner = %s AND leaseExpires > UNIX_TIMESTAMP()
LIMIT 1', $task_table->getTableName(), $taskdata_table->getTableName(), $lease_ownership_name);
$tasks = $task_table->loadAllFromArray($data);
$tasks = mpull($tasks, null, 'getID');
$task_data = array();
foreach ($data as $row) {
$tasks[$row['id']]->setServerTime($row['_serverTime']);
if ($row['_taskData']) {
$task_data[$row['id']] = json_decode($row['_taskData'], true);
} else {
$task_data[$row['id']] = null;
}
}
foreach ($tasks as $task) {
// TODO: We should detect if we acquired a task with an expired lease
// and log about it / bump up failure count.
// TODO: We should detect if we acquired a task with an excessive
// failure count and fail it permanently.
$data = idx($task_data, $task->getID());
$class = $task->getTaskClass();
try {
PhutilSymbolLoader::loadClass($class);
if (!is_subclass_of($class, 'PhabricatorWorker')) {
throw new Exception("Task class '{$class}' does not extend PhabricatorWorker.");
}
$worker = newv($class, array($data));
$lease = $worker->getRequiredLeaseTime();
if ($lease !== null) {
$task->setLeaseDuration($lease);
}
$worker->executeTask();
$task->delete();
if ($data !== null) {
queryfx($conn_w, 'DELETE FROM %T WHERE id = %d', $taskdata_table->getTableName(), $task->getDataID());
}
} catch (Exception $ex) {
$task->setFailureCount($task->getFailureCount() + 1);
$task->save();
throw $ex;
}
}
$sleep = 0;
} else {
$sleep = min($sleep + 1, 30);
}
$this->sleep($sleep);
} while (true);
}
public function getWaitBeforeRetry(PhabricatorWorkerTask $task)
{
return $task->getFailureCount() * 15;
}
/**
* See @{method:getMaximumRetryCount} for a description of Doorkeeper
* retry defaults.
*/
public function getWaitBeforeRetry(PhabricatorWorkerTask $task)
{
$count = $task->getFailureCount();
return 5 * 60 * pow(8, $count);
}
private function insertTask(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, $data = array())
{
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
break;
default:
throw new Exception("Unknown repository type '{$vcs}'!");
}
$task = new PhabricatorWorkerTask();
$task->setTaskClass($class);
$data['commitID'] = $commit->getID();
$task->setData($data);
$task->save();
}
if (!$repository) {
throw new Exception("No such repository exists!");
}
$repositories = array($repository->getID() => $repository);
echo "Loading commits in '{$callsign}' repository...\n";
$commits = id(new PhabricatorRepositoryCommit())->loadAllWhere('repositoryID = %d', $repository->getID());
}
echo "Inserting tasks for " . count($commits) . " commits";
foreach ($commits as $commit) {
echo ".";
$id = $commit->getID();
$repo = idx($repositories, $commit->getRepositoryID());
if (!$repo) {
echo "\nWarning: Commit #{$id} has an invalid repository ID.\n";
}
switch ($repo->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$task_class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$task_class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
break;
default:
throw new Exception("Unknown repository type!");
}
$task = new PhabricatorWorkerTask();
$task->setTaskClass($task_class);
$task->setData(array('commitID' => $commit->getID(), 'only' => true));
$task->save();
}
echo "\nDone.\n";
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* After upgrading to/past D1723, the handling of messages queued for delivery
* is a bit different. Running this script will take any messages that are
* queued for delivery, but don't have a worker task created, and create that
* worker task. Without the worker task, the message will just sit at "queued
* for delivery" and nothing will happen to it.
*/
$root = dirname(dirname(dirname(__FILE__)));
require_once $root . '/scripts/__init_script__.php';
$messages = id(new PhabricatorMetaMTAMail())->loadAllWhere('status = %s', PhabricatorMetaMTAMail::STATUS_QUEUE);
foreach ($messages as $message) {
if (!$message->getWorkerTaskID()) {
$mailer_task = new PhabricatorWorkerTask();
$mailer_task->setTaskClass('PhabricatorMetaMTAWorker');
$mailer_task->setData($message->getID());
$mailer_task->save();
$message->setWorkerTaskID($mailer_task->getID());
$message->save();
$id = $message->getID();
echo "#{$id}\n";
}
}
protected function didWriteData()
{
parent::didWriteData();
if (!$this->getWorkerTaskID()) {
$mailer_task = new PhabricatorWorkerTask();
$mailer_task->setTaskClass('PhabricatorMetaMTAWorker');
$mailer_task->setData($this->getID());
$mailer_task->save();
$this->setWorkerTaskID($mailer_task->getID());
$this->save();
}
}
