public function setKeys(array $keys, $ttl = null)
{
if (PhabricatorEnv::isReadOnly()) {
return;
}
if ($keys) {
$map = $this->digestKeys(array_keys($keys));
$conn_w = $this->establishConnection('w');
$sql = array();
foreach ($map as $key => $hash) {
$value = $keys[$key];
list($format, $storage_value) = $this->willWriteValue($key, $value);
$sql[] = qsprintf($conn_w, '(%s, %s, %s, %B, %d, %nd)', $hash, $key, $format, $storage_value, time(), $ttl ? time() + $ttl : null);
}
$guard = AphrontWriteGuard::beginScopedUnguardedWrites();
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T
(cacheKeyHash, cacheKey, cacheFormat, cacheData,
cacheCreated, cacheExpires) VALUES %Q
ON DUPLICATE KEY UPDATE
cacheKey = VALUES(cacheKey),
cacheFormat = VALUES(cacheFormat),
cacheData = VALUES(cacheData),
cacheCreated = VALUES(cacheCreated),
cacheExpires = VALUES(cacheExpires)', $this->getTableName(), $chunk);
}
unset($guard);
}
return $this;
}
public static function writeCaches(array $values)
{
if (PhabricatorEnv::isReadOnly()) {
return;
}
if (!$values) {
return;
}
$table = new self();
$conn_w = $table->establishConnection('w');
$sql = array();
foreach ($values as $value) {
$key = $value['key'];
$sql[] = qsprintf($conn_w, '(%s, %s, %s, %s, %s)', $value['userPHID'], PhabricatorHash::digestForIndex($key), $key, $value['value'], $value['type']->getUserCacheType());
}
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T (userPHID, cacheIndex, cacheKey, cacheData, cacheType)
VALUES %Q
ON DUPLICATE KEY UPDATE
cacheData = VALUES(cacheData),
cacheType = VALUES(cacheType)', $table->getTableName(), $chunk);
}
unset($unguarded);
}
protected function execute(ConduitAPIRequest $request)
{
$viewer = $request->getUser();
$repository_phid = $request->getValue('repositoryPHID');
$repository = id(new PhabricatorRepositoryQuery())->setViewer($viewer)->withPHIDs(array($repository_phid))->executeOne();
if (!$repository) {
throw new Exception(pht('No repository exists with PHID "%s".', $repository_phid));
}
$commit_name = $request->getValue('commit');
$commit = id(new DiffusionCommitQuery())->setViewer($viewer)->withRepository($repository)->withIdentifiers(array($commit_name))->executeOne();
if (!$commit) {
throw new Exception(pht('No commit exists with identifier "%s".', $commit_name));
}
$branch = PhabricatorRepositoryBranch::loadOrCreateBranch($repository->getID(), $request->getValue('branch'));
$coverage = $request->getValue('coverage');
$path_map = id(new DiffusionPathIDQuery(array_keys($coverage)))->loadPathIDs();
$conn = $repository->establishConnection('w');
$sql = array();
foreach ($coverage as $path => $coverage_info) {
$sql[] = qsprintf($conn, '(%d, %d, %d, %s)', $branch->getID(), $path_map[$path], $commit->getID(), $coverage_info);
}
$table_name = 'repository_coverage';
$conn->openTransaction();
queryfx($conn, 'DELETE FROM %T WHERE branchID = %d', $table_name, $branch->getID());
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx($conn, 'INSERT INTO %T (branchID, pathID, commitID, coverage) VALUES %Q', $table_name, $chunk);
}
$conn->saveTransaction();
}
public function testSQLChunking()
{
$fragments = array('a', 'a', 'b', 'b', 'ccc', 'dd', 'e');
$this->assertEqual(array('aa', 'bb', 'ccc', 'dd', 'e'), PhabricatorLiskDAO::chunkSQL($fragments, '', 2));
$fragments = array('a', 'a', 'a', 'XX', 'a', 'a', 'a', 'a');
$this->assertEqual(array('a, a, a', 'XX, a, a', 'a, a'), PhabricatorLiskDAO::chunkSQL($fragments, ', ', 8));
$fragments = array('xxxxxxxxxx', 'yyyyyyyyyy', 'a', 'b', 'c', 'zzzzzzzzzz');
$this->assertEqual(array('xxxxxxxxxx', 'yyyyyyyyyy', 'a, b, c', 'zzzzzzzzzz'), PhabricatorLiskDAO::chunkSQL($fragments, ', ', 8));
}
protected function deleteDocumentsByHash(array $hashes)
{
$atom_table = new DivinerLiveAtom();
$symbol_table = new DivinerLiveSymbol();
$conn_w = $symbol_table->establishConnection('w');
$strings = array();
foreach ($hashes as $hash) {
$strings[] = qsprintf($conn_w, '%s', $hash);
}
foreach (PhabricatorLiskDAO::chunkSQL($strings, ', ') as $chunk) {
queryfx($conn_w, 'UPDATE %T SET graphHash = NULL, nodeHash = NULL
WHERE graphHash IN (%Q)', $symbol_table->getTableName(), $chunk);
}
queryfx($conn_w, 'DELETE a FROM %T a LEFT JOIN %T s
ON a.symbolPHID = s.phid
WHERE s.graphHash IS NULL', $atom_table->getTableName(), $symbol_table->getTableName());
}
protected function execute(ConduitAPIRequest $request)
{
$viewer = $request->getUser();
$repository_phid = $request->getValue('repositoryPHID');
$repository = id(new PhabricatorRepositoryQuery())->setViewer($viewer)->withPHIDs(array($repository_phid))->executeOne();
if (!$repository) {
throw new Exception(pht('No repository exists with PHID "%s".', $repository_phid));
}
$commit_name = $request->getValue('commit');
$commit = id(new DiffusionCommitQuery())->setViewer($viewer)->withRepository($repository)->withIdentifiers(array($commit_name))->executeOne();
if (!$commit) {
throw new Exception(pht('No commit exists with identifier "%s".', $commit_name));
}
$branch = PhabricatorRepositoryBranch::loadOrCreateBranch($repository->getID(), $request->getValue('branch'));
$coverage = $request->getValue('coverage');
$path_map = id(new DiffusionPathIDQuery(array_keys($coverage)))->loadPathIDs();
$conn = $repository->establishConnection('w');
$sql = array();
foreach ($coverage as $path => $coverage_info) {
$sql[] = qsprintf($conn, '(%d, %d, %d, %s)', $branch->getID(), $path_map[$path], $commit->getID(), $coverage_info);
}
$table_name = 'repository_coverage';
$conn->openTransaction();
$mode = $request->getValue('mode');
switch ($mode) {
case '':
case 'overwrite':
// sets the coverage for the whole branch, deleting all previous
// coverage information
queryfx($conn, 'DELETE FROM %T WHERE branchID = %d', $table_name, $branch->getID());
break;
case 'update':
// sets the coverage for the provided files on the specified commit
break;
default:
$conn->killTransaction();
throw new Exception(pht('Invalid mode "%s".', $mode));
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx($conn, 'INSERT INTO %T (branchID, pathID, commitID, coverage) VALUES %Q' . ' ON DUPLICATE KEY UPDATE coverage=VALUES(coverage)', $table_name, $chunk);
}
$conn->saveTransaction();
}
private function writeCommitChanges(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, array $changes)
{
$repository_id = (int) $repository->getID();
$commit_id = (int) $commit->getID();
// NOTE: This SQL is being built manually instead of with qsprintf()
// because some SVN changes affect an enormous number of paths (millions)
// and this showed up as significantly slow on a profile at some point.
$changes_sql = array();
foreach ($changes as $change) {
$values = array($repository_id, (int) $change->getPathID(), $commit_id, nonempty((int) $change->getTargetPathID(), 'null'), nonempty((int) $change->getTargetCommitID(), 'null'), (int) $change->getChangeType(), (int) $change->getFileType(), (int) $change->getIsDirect(), (int) $change->getCommitSequence());
$changes_sql[] = '(' . implode(', ', $values) . ')';
}
$conn_w = $repository->establishConnection('w');
queryfx($conn_w, 'DELETE FROM %T WHERE commitID = %d', PhabricatorRepository::TABLE_PATHCHANGE, $commit_id);
foreach (PhabricatorLiskDAO::chunkSQL($changes_sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T
(repositoryID, pathID, commitID, targetPathID, targetCommitID,
changeType, fileType, isDirect, commitSequence)
VALUES %Q', PhabricatorRepository::TABLE_PATHCHANGE, $chunk);
}
}
<?php
$diff_table = new DifferentialDiff();
$conn_w = $diff_table->establishConnection('w');
$size = 1000;
$row_iter = id(new LiskMigrationIterator($diff_table))->setPageSize($size);
$chunk_iter = new PhutilChunkedIterator($row_iter, $size);
foreach ($chunk_iter as $chunk) {
$sql = array();
foreach ($chunk as $diff) {
$id = $diff->getID();
echo pht('Migrating diff ID %d...', $id) . "\n";
$phid = $diff->getPHID();
if (strlen($phid)) {
continue;
}
$type_diff = DifferentialDiffPHIDType::TYPECONST;
$new_phid = PhabricatorPHID::generateNewPHID($type_diff);
$sql[] = qsprintf($conn_w, '(%d, %s)', $id, $new_phid);
}
if (!$sql) {
continue;
}
foreach (PhabricatorLiskDAO::chunkSQL($sql, ', ') as $sql_chunk) {
queryfx($conn_w, 'INSERT IGNORE INTO %T (id, phid) VALUES %Q
ON DUPLICATE KEY UPDATE phid = VALUES(phid)', $diff_table->getTableName(), $sql_chunk);
}
}
echo pht('Done.') . "\n";
public function updateDatasourceTokens()
{
$table = self::TABLE_DATASOURCE_TOKEN;
$conn_w = $this->establishConnection('w');
$id = $this->getID();
$slugs = queryfx_all($conn_w, 'SELECT * FROM %T WHERE projectPHID = %s', id(new PhabricatorProjectSlug())->getTableName(), $this->getPHID());
$all_strings = ipull($slugs, 'slug');
$all_strings[] = $this->getName();
$all_strings = implode(' ', $all_strings);
$tokens = PhabricatorTypeaheadDatasource::tokenizeString($all_strings);
$sql = array();
foreach ($tokens as $token) {
$sql[] = qsprintf($conn_w, '(%d, %s)', $id, $token);
}
$this->openTransaction();
queryfx($conn_w, 'DELETE FROM %T WHERE projectID = %d', $table, $id);
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T (projectID, token) VALUES %Q', $table, $chunk);
}
$this->saveTransaction();
}
<?php
// Was PhabricatorEdgeConfig::TYPE_COLUMN_HAS_OBJECT
$type_has_object = 44;
$column = new PhabricatorProjectColumn();
$conn_w = $column->establishConnection('w');
$rows = queryfx_all($conn_w, 'SELECT src, dst FROM %T WHERE type = %d', PhabricatorEdgeConfig::TABLE_NAME_EDGE, $type_has_object);
$cols = array();
foreach ($rows as $row) {
$cols[$row['src']][] = $row['dst'];
}
$sql = array();
foreach ($cols as $col_phid => $obj_phids) {
echo pht("Migrating column '%s'...", $col_phid) . "\n";
$column = id(new PhabricatorProjectColumn())->loadOneWhere('phid = %s', $col_phid);
if (!$column) {
echo pht("Column '%s' does not exist.", $col_phid) . "\n";
continue;
}
$sequence = 0;
foreach ($obj_phids as $obj_phid) {
$sql[] = qsprintf($conn_w, '(%s, %s, %s, %d)', $column->getProjectPHID(), $column->getPHID(), $obj_phid, $sequence++);
}
}
echo pht('Inserting rows...') . "\n";
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T (boardPHID, columnPHID, objectPHID, sequence)
VALUES %Q', id(new PhabricatorProjectColumnPosition())->getTableName(), $chunk);
}
echo pht('Done.') . "\n";
private function sendNotifications()
{
$cursor = $this->getCursor();
$window_min = $cursor - phutil_units('16 hours in seconds');
$window_max = $cursor + phutil_units('16 hours in seconds');
$viewer = PhabricatorUser::getOmnipotentUser();
$events = id(new PhabricatorCalendarEventQuery())->setViewer($viewer)->withDateRange($window_min, $window_max)->withIsCancelled(false)->withIsImported(false)->setGenerateGhosts(true)->execute();
if (!$events) {
// No events are starting soon in any timezone, so there is nothing
// left to be done.
return;
}
$attendee_map = array();
foreach ($events as $key => $event) {
$notifiable_phids = array();
foreach ($event->getInvitees() as $invitee) {
if (!$invitee->isAttending()) {
continue;
}
$notifiable_phids[] = $invitee->getInviteePHID();
}
if (!$notifiable_phids) {
unset($events[$key]);
}
$attendee_map[$key] = array_fuse($notifiable_phids);
}
if (!$attendee_map) {
// None of the events have any notifiable attendees, so there is no
// one to notify of anything.
return;
}
$all_attendees = array();
foreach ($attendee_map as $key => $attendee_phids) {
foreach ($attendee_phids as $attendee_phid) {
$all_attendees[$attendee_phid] = $attendee_phid;
}
}
$user_map = id(new PhabricatorPeopleQuery())->setViewer($viewer)->withPHIDs($all_attendees)->withIsDisabled(false)->needUserSettings(true)->execute();
$user_map = mpull($user_map, null, 'getPHID');
if (!$user_map) {
// None of the attendees are valid users: they're all imported users
// or projects or invalid or some other kind of unnotifiable entity.
return;
}
$all_event_phids = array();
foreach ($events as $key => $event) {
foreach ($event->getNotificationPHIDs() as $phid) {
$all_event_phids[$phid] = $phid;
}
}
$table = new PhabricatorCalendarNotification();
$conn = $table->establishConnection('w');
$rows = queryfx_all($conn, 'SELECT * FROM %T WHERE eventPHID IN (%Ls) AND targetPHID IN (%Ls)', $table->getTableName(), $all_event_phids, $all_attendees);
$sent_map = array();
foreach ($rows as $row) {
$event_phid = $row['eventPHID'];
$target_phid = $row['targetPHID'];
$initial_epoch = $row['utcInitialEpoch'];
$sent_map[$event_phid][$target_phid][$initial_epoch] = $row;
}
$now = PhabricatorTime::getNow();
$notify_min = $now;
$notify_max = $now + $this->getNotifyWindow();
$notify_map = array();
foreach ($events as $key => $event) {
$initial_epoch = $event->getUTCInitialEpoch();
$event_phids = $event->getNotificationPHIDs();
// Select attendees who actually exist, and who we have not sent any
// notifications to yet.
$attendee_phids = $attendee_map[$key];
$users = array_select_keys($user_map, $attendee_phids);
foreach ($users as $user_phid => $user) {
foreach ($event_phids as $event_phid) {
if (isset($sent_map[$event_phid][$user_phid][$initial_epoch])) {
unset($users[$user_phid]);
continue 2;
}
}
}
if (!$users) {
continue;
}
// Discard attendees for whom the event start time isn't soon. Events
// may start at different times for different users, so we need to
// check every user's start time.
foreach ($users as $user_phid => $user) {
$user_datetime = $event->newStartDateTime()->setViewerTimezone($user->getTimezoneIdentifier());
$user_epoch = $user_datetime->getEpoch();
if ($user_epoch < $notify_min || $user_epoch > $notify_max) {
unset($users[$user_phid]);
continue;
}
$view = id(new PhabricatorCalendarEventNotificationView())->setViewer($user)->setEvent($event)->setDateTime($user_datetime)->setEpoch($user_epoch);
$notify_map[$user_phid][] = $view;
}
}
$mail_list = array();
$mark_list = array();
$now = PhabricatorTime::getNow();
foreach ($notify_map as $user_phid => $events) {
$user = $user_map[$user_phid];
$locale = PhabricatorEnv::beginScopedLocale($user->getTranslation());
$caught = null;
try {
$mail_list[] = $this->newMailMessage($user, $events);
} catch (Exception $ex) {
$caught = $ex;
}
unset($locale);
if ($caught) {
throw $ex;
}
foreach ($events as $view) {
$event = $view->getEvent();
foreach ($event->getNotificationPHIDs() as $phid) {
$mark_list[] = qsprintf($conn, '(%s, %s, %d, %d)', $phid, $user_phid, $event->getUTCInitialEpoch(), $now);
}
}
}
// Mark all the notifications we're about to send as delivered so we
// do not double-notify.
foreach (PhabricatorLiskDAO::chunkSQL($mark_list) as $chunk) {
queryfx($conn, 'INSERT IGNORE INTO %T
(eventPHID, targetPHID, utcInitialEpoch, didNotifyEpoch)
VALUES %Q', $table->getTableName(), $chunk);
}
foreach ($mail_list as $mail) {
$mail->saveAndSend();
}
}
/**
* Publish field indexes into index tables, so ApplicationSearch can search
* them.
*
* @return void
*/
public function rebuildIndexes(PhabricatorCustomFieldInterface $object)
{
$indexes = array();
$index_keys = array();
$phid = $object->getPHID();
$role = PhabricatorCustomField::ROLE_APPLICATIONSEARCH;
foreach ($this->fields as $field) {
if (!$field->shouldEnableForRole($role)) {
continue;
}
$index_keys[$field->getFieldIndex()] = true;
foreach ($field->buildFieldIndexes() as $index) {
$index->setObjectPHID($phid);
$indexes[$index->getTableName()][] = $index;
}
}
if (!$indexes) {
return;
}
$any_index = head(head($indexes));
$conn_w = $any_index->establishConnection('w');
foreach ($indexes as $table => $index_list) {
$sql = array();
foreach ($index_list as $index) {
$sql[] = $index->formatForInsert($conn_w);
}
$indexes[$table] = $sql;
}
$any_index->openTransaction();
foreach ($indexes as $table => $sql_list) {
queryfx($conn_w, 'DELETE FROM %T WHERE objectPHID = %s AND indexKey IN (%Ls)', $table, $phid, array_keys($index_keys));
if (!$sql_list) {
continue;
}
foreach (PhabricatorLiskDAO::chunkSQL($sql_list) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T (objectPHID, indexKey, indexValue) VALUES %Q', $table, $chunk);
}
}
$any_index->saveTransaction();
}
private function rebuildRepository(PhabricatorRepository $repo)
{
$console = PhutilConsole::getConsole();
$console->writeOut("%s\n", pht('Rebuilding "%s"...', $repo->getMonogram()));
$refs = id(new PhabricatorRepositoryRefCursorQuery())->setViewer($this->getViewer())->withRefTypes(array(PhabricatorRepositoryRefCursor::TYPE_BRANCH))->withRepositoryPHIDs(array($repo->getPHID()))->execute();
$graph = array();
foreach ($refs as $ref) {
if (!$repo->shouldTrackBranch($ref->getRefName())) {
continue;
}
$console->writeOut("%s\n", pht('Rebuilding branch "%s"...', $ref->getRefName()));
$commit = $ref->getCommitIdentifier();
if ($repo->isGit()) {
$stream = new PhabricatorGitGraphStream($repo, $commit);
} else {
$stream = new PhabricatorMercurialGraphStream($repo, $commit);
}
$discover = array($commit);
while ($discover) {
$target = array_pop($discover);
if (isset($graph[$target])) {
continue;
}
$graph[$target] = $stream->getParents($target);
foreach ($graph[$target] as $parent) {
$discover[] = $parent;
}
}
}
$console->writeOut("%s\n", pht('Found %s total commit(s); updating...', phutil_count($graph)));
$commit_table = id(new PhabricatorRepositoryCommit());
$commit_table_name = $commit_table->getTableName();
$conn_w = $commit_table->establishConnection('w');
$bar = id(new PhutilConsoleProgressBar())->setTotal(count($graph));
$need = array();
foreach ($graph as $child => $parents) {
foreach ($parents as $parent) {
$need[$parent] = $parent;
}
$need[$child] = $child;
}
$map = array();
foreach (array_chunk($need, 2048) as $chunk) {
$rows = queryfx_all($conn_w, 'SELECT id, commitIdentifier FROM %T
WHERE commitIdentifier IN (%Ls) AND repositoryID = %d', $commit_table_name, $chunk, $repo->getID());
foreach ($rows as $row) {
$map[$row['commitIdentifier']] = $row['id'];
}
}
$insert_sql = array();
$delete_sql = array();
foreach ($graph as $child => $parents) {
$names = $parents;
$names[] = $child;
foreach ($names as $name) {
if (empty($map[$name])) {
throw new Exception(pht('Unknown commit "%s"!', $name));
}
}
if (!$parents) {
// Write an explicit 0 to indicate "no parents" instead of "no data".
$insert_sql[] = qsprintf($conn_w, '(%d, 0)', $map[$child]);
} else {
foreach ($parents as $parent) {
$insert_sql[] = qsprintf($conn_w, '(%d, %d)', $map[$child], $map[$parent]);
}
}
$delete_sql[] = $map[$child];
$bar->update(1);
}
$commit_table->openTransaction();
foreach (PhabricatorLiskDAO::chunkSQL($delete_sql) as $chunk) {
queryfx($conn_w, 'DELETE FROM %T WHERE childCommitID IN (%Q)', PhabricatorRepository::TABLE_PARENTS, $chunk);
}
foreach (PhabricatorLiskDAO::chunkSQL($insert_sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T (childCommitID, parentCommitID) VALUES %Q', PhabricatorRepository::TABLE_PARENTS, $chunk);
}
$commit_table->saveTransaction();
$bar->done();
}
private static function recordAction(array $actors, PhabricatorSystemAction $action, $score)
{
$log = new PhabricatorSystemActionLog();
$conn_w = $log->establishConnection('w');
$sql = array();
foreach ($actors as $actor) {
$sql[] = qsprintf($conn_w, '(%s, %s, %s, %f, %d)', PhabricatorHash::digestForIndex($actor), $actor, $action->getActionConstant(), $score, time());
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx($conn_w, 'INSERT INTO %T (actorHash, actorIdentity, action, score, epoch)
VALUES %Q', $log->getTableName(), $chunk);
}
}
<?php
// Destroy duplicate drafts before storage adjustment adds a unique key to this
// table. See T1191. We retain the newest draft.
// (We can't easily do this in a single SQL statement because MySQL won't let us
// modify a table that's joined in a subquery.)
$table = new DifferentialDraft();
$conn_w = $table->establishConnection('w');
$duplicates = queryfx_all($conn_w, 'SELECT DISTINCT u.id id FROM %T u
JOIN %T v
ON u.objectPHID = v.objectPHID
AND u.authorPHID = v.authorPHID
AND u.draftKey = v.draftKey
AND u.id < v.id', $table->getTableName(), $table->getTableName());
$duplicates = ipull($duplicates, 'id');
foreach (PhabricatorLiskDAO::chunkSQL($duplicates) as $chunk) {
queryfx($conn_w, 'DELETE FROM %T WHERE id IN (%Q)', $table->getTableName(), $chunk);
}