/**
* Returns user's object ids updated since |timestamp|, keyed by libraryID,
* or count of all updated items if $countOnly is true
*
* @param int $libraryID User ID
* @param string $timestamp Unix timestamp of last sync time
* @param array $updatedLibraryIDs Libraries with updated data
* @return array|int
*/
public static function getUpdated($userID, $timestamp, $updatedLibraryIDs, $countOnly = false)
{
$table = self::$table;
$id = self::$idColumn;
$type = self::$objectType;
$types = self::$objectTypePlural;
$timestampCol = "serverDateModified";
// All joined groups have to be checked
$joinedGroupIDs = Zotero_Groups::getJoined($userID, $timestamp);
$joinedLibraryIDs = array();
foreach ($joinedGroupIDs as $groupID) {
$joinedLibraryIDs[] = Zotero_Groups::getLibraryIDFromGroupID($groupID);
}
// Separate libraries into shards for querying
$libraryIDs = array_unique(array_merge($joinedLibraryIDs, $updatedLibraryIDs));
$shardLibraryIDs = array();
foreach ($libraryIDs as $libraryID) {
$shardID = Zotero_Shards::getByLibraryID($libraryID);
if (!isset($shardLibraryIDs[$shardID])) {
$shardLibraryIDs[$shardID] = array('updated' => array(), 'joined' => array());
}
if (in_array($libraryID, $joinedLibraryIDs)) {
$shardLibraryIDs[$shardID]['joined'][] = $libraryID;
} else {
$shardLibraryIDs[$shardID]['updated'][] = $libraryID;
}
}
if ($countOnly) {
$count = 0;
$fieldList = "COUNT(*)";
} else {
$updatedByLibraryID = array();
$fieldList = "libraryID, {$id} AS id";
}
// Send query at each shard
foreach ($shardLibraryIDs as $shardID => $libraryIDs) {
$sql = "SELECT {$fieldList} FROM {$table} WHERE ";
if ($libraryIDs['updated']) {
$sql .= "(libraryID IN (" . implode(', ', array_fill(0, sizeOf($libraryIDs['updated']), '?')) . ")";
$params = $libraryIDs['updated'];
$sql .= " AND {$timestampCol} >= FROM_UNIXTIME(?))";
$params[] = $timestamp;
}
if ($libraryIDs['joined']) {
if ($libraryIDs['updated']) {
$sql .= " OR ";
} else {
$params = array();
}
$sql .= "libraryID IN (" . implode(', ', array_fill(0, sizeOf($libraryIDs['joined']), '?')) . ")";
$params = array_merge($params, $libraryIDs['joined']);
}
if ($countOnly) {
$count += Zotero_DB::valueQuery($sql, $params, $shardID);
} else {
$rows = Zotero_DB::query($sql, $params, $shardID);
if ($rows) {
// Separate ids by libraryID
foreach ($rows as $row) {
$updatedByLibraryID[$row['libraryID']][] = $row['id'];
}
}
}
}
return $countOnly ? $count : $updatedByLibraryID;
}
private static function processDownloadInternal($userID, $lastsync, DOMDocument $doc, $syncDownloadQueueID = null, $syncDownloadProcessID = null, $params = [])
{
$apiVersion = (int) $doc->documentElement->getAttribute('version');
if ($lastsync == 1) {
StatsD::increment("sync.process.download.full");
}
// TEMP
$cacheKeyExtra = (!empty($params['ft']) ? json_encode($params['ft']) : "") . (!empty($params['ftkeys']) ? json_encode($params['ftkeys']) : "");
try {
$cached = Zotero_Sync::getCachedDownload($userID, $lastsync, $apiVersion, $cacheKeyExtra);
if ($cached) {
$doc->loadXML($cached);
StatsD::increment("sync.process.download.cache.hit");
return;
}
} catch (Exception $e) {
$msg = $e->getMessage();
if (strpos($msg, "Too many connections") !== false) {
$msg = "'Too many connections' from MySQL";
} else {
$msg = "'{$msg}'";
}
Z_Core::logError("Warning: {$msg} getting cached download");
StatsD::increment("sync.process.download.cache.error");
}
set_time_limit(1800);
$profile = false;
if ($profile) {
$shardID = Zotero_Shards::getByUserID($userID);
Zotero_DB::profileStart(0);
}
if ($syncDownloadQueueID) {
self::addDownloadProcess($syncDownloadQueueID, $syncDownloadProcessID);
}
$updatedNode = $doc->createElement('updated');
$doc->documentElement->appendChild($updatedNode);
$userLibraryID = Zotero_Users::getLibraryIDFromUserID($userID);
$updatedCreators = array();
try {
Zotero_DB::beginTransaction();
// Blocks until any upload processes are done
$updateTimes = Zotero_Libraries::getUserLibraryUpdateTimes($userID);
$timestamp = Zotero_DB::getTransactionTimestampUnix();
$doc->documentElement->setAttribute('timestamp', $timestamp);
$doc->documentElement->setAttribute('userID', $userID);
$doc->documentElement->setAttribute('defaultLibraryID', $userLibraryID);
$updateKey = Zotero_Users::getUpdateKey($userID);
$doc->documentElement->setAttribute('updateKey', $updateKey);
// Get libraries with update times >= $timestamp
$updatedLibraryIDs = array();
foreach ($updateTimes as $libraryID => $timestamp) {
if ($timestamp >= $lastsync) {
$updatedLibraryIDs[] = $libraryID;
}
}
// Add new and updated groups
$joinedGroups = Zotero_Groups::getJoined($userID, (int) $lastsync);
$updatedIDs = array_unique(array_merge($joinedGroups, Zotero_Groups::getUpdated($userID, (int) $lastsync)));
if ($updatedIDs) {
$node = $doc->createElement('groups');
$showGroups = false;
foreach ($updatedIDs as $id) {
$group = new Zotero_Group();
$group->id = $id;
$xmlElement = $group->toXML($userID);
$newNode = dom_import_simplexml($xmlElement);
$newNode = $doc->importNode($newNode, true);
$node->appendChild($newNode);
$showGroups = true;
}
if ($showGroups) {
$updatedNode->appendChild($node);
}
}
// If there's updated data in any library or
// there are any new groups (in which case we need all their data)
$hasData = $updatedLibraryIDs || $joinedGroups;
if ($hasData) {
foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) {
$Name = $syncObject['singular'];
// 'Item'
$Names = $syncObject['plural'];
// 'Items'
$name = strtolower($Name);
// 'item'
$names = strtolower($Names);
// 'items'
$className = 'Zotero_' . $Names;
$updatedIDsByLibraryID = call_user_func(array($className, 'getUpdated'), $userID, $lastsync, $updatedLibraryIDs);
if ($updatedIDsByLibraryID) {
$node = $doc->createElement($names);
foreach ($updatedIDsByLibraryID as $libraryID => $ids) {
if ($name == 'creator') {
$updatedCreators[$libraryID] = $ids;
}
foreach ($ids as $id) {
if ($name == 'item') {
$obj = call_user_func(array($className, 'get'), $libraryID, $id);
$data = array('updatedCreators' => isset($updatedCreators[$libraryID]) ? $updatedCreators[$libraryID] : array());
$xmlElement = Zotero_Items::convertItemToXML($obj, $data, $apiVersion);
} else {
$instanceClass = 'Zotero_' . $Name;
$obj = new $instanceClass();
if (method_exists($instanceClass, '__construct')) {
$obj->__construct();
}
$obj->libraryID = $libraryID;
if ($name == 'setting') {
$obj->name = $id;
} else {
$obj->id = $id;
}
if ($name == 'tag') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, true);
} else {
if ($name == 'creator') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc);
if ($xmlElement->getAttribute('libraryID') == $userLibraryID) {
$xmlElement->removeAttribute('libraryID');
}
$node->appendChild($xmlElement);
} else {
if ($name == 'relation') {
// Skip new-style related items
if ($obj->predicate == 'dc:relation') {
continue;
}
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj);
if ($apiVersion <= 8) {
unset($xmlElement['libraryID']);
}
} else {
if ($name == 'setting') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc);
$node->appendChild($xmlElement);
} else {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj);
}
}
}
}
}
if ($xmlElement instanceof SimpleXMLElement) {
if ($xmlElement['libraryID'] == $userLibraryID) {
unset($xmlElement['libraryID']);
}
$newNode = dom_import_simplexml($xmlElement);
$newNode = $doc->importNode($newNode, true);
$node->appendChild($newNode);
}
}
}
if ($node->hasChildNodes()) {
$updatedNode->appendChild($node);
}
}
}
}
// Add full-text content if the client supports it
if (isset($params['ft'])) {
$libraries = Zotero_Libraries::getUserLibraries($userID);
$fulltextNode = false;
foreach ($libraries as $libraryID) {
if (!empty($params['ftkeys']) && $params['ftkeys'] === 'all') {
$ftlastsync = 1;
} else {
$ftlastsync = $lastsync;
}
if (!empty($params['ftkeys'][$libraryID])) {
$keys = $params['ftkeys'][$libraryID];
} else {
$keys = [];
}
$data = Zotero_FullText::getNewerInLibraryByTime($libraryID, $ftlastsync, $keys);
if ($data) {
if (!$fulltextNode) {
$fulltextNode = $doc->createElement('fulltexts');
}
foreach ($data as $itemData) {
if ($params['ft']) {
$empty = $itemData['empty'];
} else {
$empty = true;
}
$first = false;
$node = Zotero_FullText::itemDataToXML($itemData, $doc, $empty);
$fulltextNode->appendChild($node);
}
}
}
if ($fulltextNode) {
$updatedNode->appendChild($fulltextNode);
}
}
// Get earliest timestamp
$earliestModTime = Zotero_Users::getEarliestDataTimestamp($userID);
$doc->documentElement->setAttribute('earliest', $earliestModTime ? $earliestModTime : 0);
// Deleted objects
$deletedKeys = $hasData ? self::getDeletedObjectKeys($userID, $lastsync, true) : false;
$deletedIDs = self::getDeletedObjectIDs($userID, $lastsync, true);
if ($deletedKeys || $deletedIDs) {
$deletedNode = $doc->createElement('deleted');
// Add deleted data objects
if ($deletedKeys) {
foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) {
$Name = $syncObject['singular'];
// 'Item'
$Names = $syncObject['plural'];
// 'Items'
$name = strtolower($Name);
// 'item'
$names = strtolower($Names);
// 'items'
if (empty($deletedKeys[$names])) {
continue;
}
$typeNode = $doc->createElement($names);
foreach ($deletedKeys[$names] as $row) {
$node = $doc->createElement($name);
if ($row['libraryID'] != $userLibraryID || $name == 'setting') {
$node->setAttribute('libraryID', $row['libraryID']);
}
$node->setAttribute('key', $row['key']);
$typeNode->appendChild($node);
}
$deletedNode->appendChild($typeNode);
}
}
// Add deleted groups
if ($deletedIDs) {
$name = "group";
$names = "groups";
$typeNode = $doc->createElement($names);
$ids = $doc->createTextNode(implode(' ', $deletedIDs[$names]));
$typeNode->appendChild($ids);
$deletedNode->appendChild($typeNode);
}
$updatedNode->appendChild($deletedNode);
}
Zotero_DB::commit();
} catch (Exception $e) {
Zotero_DB::rollback(true);
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
throw $e;
}
function relaxNGErrorHandler($errno, $errstr)
{
Zotero_Sync::$validationError = $errstr;
}
set_error_handler('relaxNGErrorHandler');
$valid = $doc->relaxNGValidate(Z_ENV_MODEL_PATH . 'relax-ng/updated.rng');
restore_error_handler();
if (!$valid) {
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
throw new Exception(self::$validationError . "\n\nXML:\n\n" . $doc->saveXML());
}
// Cache response if response isn't empty
try {
if ($doc->documentElement->firstChild->hasChildNodes()) {
self::cacheDownload($userID, $updateKey, $lastsync, $apiVersion, $doc->saveXML(), $cacheKeyExtra);
}
} catch (Exception $e) {
Z_Core::logError("WARNING: " . $e);
}
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
if ($profile) {
$shardID = Zotero_Shards::getByUserID($userID);
Zotero_DB::profileEnd(0);
}
}