private static function processDownloadInternal($userID, $lastsync, DOMDocument $doc, $syncDownloadQueueID = null, $syncDownloadProcessID = null, $params = [])
{
$apiVersion = (int) $doc->documentElement->getAttribute('version');
if ($lastsync == 1) {
StatsD::increment("sync.process.download.full");
}
// TEMP
$cacheKeyExtra = (!empty($params['ft']) ? json_encode($params['ft']) : "") . (!empty($params['ftkeys']) ? json_encode($params['ftkeys']) : "");
try {
$cached = Zotero_Sync::getCachedDownload($userID, $lastsync, $apiVersion, $cacheKeyExtra);
if ($cached) {
$doc->loadXML($cached);
StatsD::increment("sync.process.download.cache.hit");
return;
}
} catch (Exception $e) {
$msg = $e->getMessage();
if (strpos($msg, "Too many connections") !== false) {
$msg = "'Too many connections' from MySQL";
} else {
$msg = "'{$msg}'";
}
Z_Core::logError("Warning: {$msg} getting cached download");
StatsD::increment("sync.process.download.cache.error");
}
set_time_limit(1800);
$profile = false;
if ($profile) {
$shardID = Zotero_Shards::getByUserID($userID);
Zotero_DB::profileStart(0);
}
if ($syncDownloadQueueID) {
self::addDownloadProcess($syncDownloadQueueID, $syncDownloadProcessID);
}
$updatedNode = $doc->createElement('updated');
$doc->documentElement->appendChild($updatedNode);
$userLibraryID = Zotero_Users::getLibraryIDFromUserID($userID);
$updatedCreators = array();
try {
Zotero_DB::beginTransaction();
// Blocks until any upload processes are done
$updateTimes = Zotero_Libraries::getUserLibraryUpdateTimes($userID);
$timestamp = Zotero_DB::getTransactionTimestampUnix();
$doc->documentElement->setAttribute('timestamp', $timestamp);
$doc->documentElement->setAttribute('userID', $userID);
$doc->documentElement->setAttribute('defaultLibraryID', $userLibraryID);
$updateKey = Zotero_Users::getUpdateKey($userID);
$doc->documentElement->setAttribute('updateKey', $updateKey);
// Get libraries with update times >= $timestamp
$updatedLibraryIDs = array();
foreach ($updateTimes as $libraryID => $timestamp) {
if ($timestamp >= $lastsync) {
$updatedLibraryIDs[] = $libraryID;
}
}
// Add new and updated groups
$joinedGroups = Zotero_Groups::getJoined($userID, (int) $lastsync);
$updatedIDs = array_unique(array_merge($joinedGroups, Zotero_Groups::getUpdated($userID, (int) $lastsync)));
if ($updatedIDs) {
$node = $doc->createElement('groups');
$showGroups = false;
foreach ($updatedIDs as $id) {
$group = new Zotero_Group();
$group->id = $id;
$xmlElement = $group->toXML($userID);
$newNode = dom_import_simplexml($xmlElement);
$newNode = $doc->importNode($newNode, true);
$node->appendChild($newNode);
$showGroups = true;
}
if ($showGroups) {
$updatedNode->appendChild($node);
}
}
// If there's updated data in any library or
// there are any new groups (in which case we need all their data)
$hasData = $updatedLibraryIDs || $joinedGroups;
if ($hasData) {
foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) {
$Name = $syncObject['singular'];
// 'Item'
$Names = $syncObject['plural'];
// 'Items'
$name = strtolower($Name);
// 'item'
$names = strtolower($Names);
// 'items'
$className = 'Zotero_' . $Names;
$updatedIDsByLibraryID = call_user_func(array($className, 'getUpdated'), $userID, $lastsync, $updatedLibraryIDs);
if ($updatedIDsByLibraryID) {
$node = $doc->createElement($names);
foreach ($updatedIDsByLibraryID as $libraryID => $ids) {
if ($name == 'creator') {
$updatedCreators[$libraryID] = $ids;
}
foreach ($ids as $id) {
if ($name == 'item') {
$obj = call_user_func(array($className, 'get'), $libraryID, $id);
$data = array('updatedCreators' => isset($updatedCreators[$libraryID]) ? $updatedCreators[$libraryID] : array());
$xmlElement = Zotero_Items::convertItemToXML($obj, $data, $apiVersion);
} else {
$instanceClass = 'Zotero_' . $Name;
$obj = new $instanceClass();
if (method_exists($instanceClass, '__construct')) {
$obj->__construct();
}
$obj->libraryID = $libraryID;
if ($name == 'setting') {
$obj->name = $id;
} else {
$obj->id = $id;
}
if ($name == 'tag') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, true);
} else {
if ($name == 'creator') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc);
if ($xmlElement->getAttribute('libraryID') == $userLibraryID) {
$xmlElement->removeAttribute('libraryID');
}
$node->appendChild($xmlElement);
} else {
if ($name == 'relation') {
// Skip new-style related items
if ($obj->predicate == 'dc:relation') {
continue;
}
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj);
if ($apiVersion <= 8) {
unset($xmlElement['libraryID']);
}
} else {
if ($name == 'setting') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc);
$node->appendChild($xmlElement);
} else {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj);
}
}
}
}
}
if ($xmlElement instanceof SimpleXMLElement) {
if ($xmlElement['libraryID'] == $userLibraryID) {
unset($xmlElement['libraryID']);
}
$newNode = dom_import_simplexml($xmlElement);
$newNode = $doc->importNode($newNode, true);
$node->appendChild($newNode);
}
}
}
if ($node->hasChildNodes()) {
$updatedNode->appendChild($node);
}
}
}
}
// Add full-text content if the client supports it
if (isset($params['ft'])) {
$libraries = Zotero_Libraries::getUserLibraries($userID);
$fulltextNode = false;
foreach ($libraries as $libraryID) {
if (!empty($params['ftkeys']) && $params['ftkeys'] === 'all') {
$ftlastsync = 1;
} else {
$ftlastsync = $lastsync;
}
if (!empty($params['ftkeys'][$libraryID])) {
$keys = $params['ftkeys'][$libraryID];
} else {
$keys = [];
}
$data = Zotero_FullText::getNewerInLibraryByTime($libraryID, $ftlastsync, $keys);
if ($data) {
if (!$fulltextNode) {
$fulltextNode = $doc->createElement('fulltexts');
}
foreach ($data as $itemData) {
if ($params['ft']) {
$empty = $itemData['empty'];
} else {
$empty = true;
}
$first = false;
$node = Zotero_FullText::itemDataToXML($itemData, $doc, $empty);
$fulltextNode->appendChild($node);
}
}
}
if ($fulltextNode) {
$updatedNode->appendChild($fulltextNode);
}
}
// Get earliest timestamp
$earliestModTime = Zotero_Users::getEarliestDataTimestamp($userID);
$doc->documentElement->setAttribute('earliest', $earliestModTime ? $earliestModTime : 0);
// Deleted objects
$deletedKeys = $hasData ? self::getDeletedObjectKeys($userID, $lastsync, true) : false;
$deletedIDs = self::getDeletedObjectIDs($userID, $lastsync, true);
if ($deletedKeys || $deletedIDs) {
$deletedNode = $doc->createElement('deleted');
// Add deleted data objects
if ($deletedKeys) {
foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) {
$Name = $syncObject['singular'];
// 'Item'
$Names = $syncObject['plural'];
// 'Items'
$name = strtolower($Name);
// 'item'
$names = strtolower($Names);
// 'items'
if (empty($deletedKeys[$names])) {
continue;
}
$typeNode = $doc->createElement($names);
foreach ($deletedKeys[$names] as $row) {
$node = $doc->createElement($name);
if ($row['libraryID'] != $userLibraryID || $name == 'setting') {
$node->setAttribute('libraryID', $row['libraryID']);
}
$node->setAttribute('key', $row['key']);
$typeNode->appendChild($node);
}
$deletedNode->appendChild($typeNode);
}
}
// Add deleted groups
if ($deletedIDs) {
$name = "group";
$names = "groups";
$typeNode = $doc->createElement($names);
$ids = $doc->createTextNode(implode(' ', $deletedIDs[$names]));
$typeNode->appendChild($ids);
$deletedNode->appendChild($typeNode);
}
$updatedNode->appendChild($deletedNode);
}
Zotero_DB::commit();
} catch (Exception $e) {
Zotero_DB::rollback(true);
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
throw $e;
}
function relaxNGErrorHandler($errno, $errstr)
{
Zotero_Sync::$validationError = $errstr;
}
set_error_handler('relaxNGErrorHandler');
$valid = $doc->relaxNGValidate(Z_ENV_MODEL_PATH . 'relax-ng/updated.rng');
restore_error_handler();
if (!$valid) {
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
throw new Exception(self::$validationError . "\n\nXML:\n\n" . $doc->saveXML());
}
// Cache response if response isn't empty
try {
if ($doc->documentElement->firstChild->hasChildNodes()) {
self::cacheDownload($userID, $updateKey, $lastsync, $apiVersion, $doc->saveXML(), $cacheKeyExtra);
}
} catch (Exception $e) {
Z_Core::logError("WARNING: " . $e);
}
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
if ($profile) {
$shardID = Zotero_Shards::getByUserID($userID);
Zotero_DB::profileEnd(0);
}
}
public function updated()
{
if (empty($_REQUEST['lastsync'])) {
$this->error(400, 'NO_LAST_SYNC_TIME', 'Last sync time not provided');
}
$lastsync = false;
if (is_numeric($_REQUEST['lastsync'])) {
$lastsync = (int) $_REQUEST['lastsync'];
} else {
$this->error(400, 'INVALID_LAST_SYNC_TIME', 'Last sync time is invalid');
}
$this->sessionCheck();
if (isset($_SERVER['HTTP_X_ZOTERO_VERSION'])) {
require_once '../model/ToolkitVersionComparator.inc.php';
if (ToolkitVersionComparator::compare($_SERVER['HTTP_X_ZOTERO_VERSION'], "2.0.4") < 0) {
$futureUsers = Z_Core::$MC->get('futureUsers');
if (!$futureUsers) {
$futureUsers = Zotero_DB::columnQuery("SELECT userID FROM futureUsers");
Z_Core::$MC->set('futureUsers', $futureUsers, 1800);
}
if (in_array($this->userID, $futureUsers)) {
Z_Core::logError("Blocking sync for future user " . $this->userID . " with version " . $_SERVER['HTTP_X_ZOTERO_VERSION']);
$upgradeMessage = "Due to improvements made to sync functionality, you must upgrade to Zotero 2.0.6 or later (via Firefox's Tools menu -> Add-ons -> Extensions -> Find Updates or from zotero.org) to continue syncing your Zotero library.";
$this->error(400, 'UPGRADE_REQUIRED', $upgradeMessage);
}
}
}
$doc = new DOMDocument();
$domResponse = dom_import_simplexml($this->responseXML);
$domResponse = $doc->importNode($domResponse, true);
$doc->appendChild($domResponse);
try {
$result = Zotero_Sync::getSessionDownloadResult($this->sessionID);
} catch (Exception $e) {
$this->handleUpdatedError($e);
}
// XML response
if (is_string($result)) {
$this->clearWaitTime($this->sessionID);
$this->responseXML = new SimpleXMLElement($result);
$this->end();
}
// Queued
if ($result === false) {
$queued = $this->responseXML->addChild('locked');
$queued['wait'] = $this->getWaitTime($this->sessionID);
$this->end();
}
// Not queued
if ($result == -1) {
// See if we're locked
Zotero_DB::beginTransaction();
if (Zotero_Sync::userIsWriteLocked($this->userID) || !empty($_REQUEST['upload']) && Zotero_Sync::userIsReadLocked($this->userID)) {
Zotero_DB::commit();
$locked = $this->responseXML->addChild('locked');
$locked['wait'] = $this->getWaitTime($this->sessionID);
$this->end();
}
Zotero_DB::commit();
$queue = true;
if (Z_ENV_TESTING_SITE && !empty($_GET['noqueue'])) {
$queue = false;
}
// TEMP
$cacheKeyExtra = (!empty($_POST['ft']) ? json_encode($_POST['ft']) : "") . (!empty($_POST['ftkeys']) ? json_encode($_POST['ftkeys']) : "");
// If we have a cached response, return that
try {
$startedTimestamp = microtime(true);
$cached = Zotero_Sync::getCachedDownload($this->userID, $lastsync, $this->apiVersion, $cacheKeyExtra);
// Not locked, so clear wait index
$this->clearWaitTime($this->sessionID);
if ($cached) {
$this->responseXML = simplexml_load_string($cached, "SimpleXMLElement", LIBXML_COMPACT | LIBXML_PARSEHUGE);
// TEMP
if (!$this->responseXML) {
error_log("Invalid cached XML data -- stripping control characters");
// Strip control characters in XML data
$cached = preg_replace('/[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F\\x7F]/', '', $cached);
$this->responseXML = simplexml_load_string($cached, "SimpleXMLElement", LIBXML_COMPACT | LIBXML_PARSEHUGE);
}
$duration = round((double) microtime(true) - $startedTimestamp, 2);
Zotero_Sync::logDownload($this->userID, round($lastsync), strlen($cached), $this->ipAddress ? $this->ipAddress : 0, 0, $duration, $duration, (int) (!$this->responseXML));
StatsD::increment("sync.process.download.cache.hit");
if (!$this->responseXML) {
$msg = "Error parsing cached XML for user " . $this->userID;
error_log($msg);
$this->handleUpdatedError(new Exception($msg));
}
$this->end();
}
} catch (Exception $e) {
$msg = $e->getMessage();
if (strpos($msg, "Too many connections") !== false) {
$msg = "'Too many connections' from MySQL";
} else {
$msg = "'{$msg}'";
}
Z_Core::logError("Warning: {$msg} getting cached download");
StatsD::increment("sync.process.download.cache.error");
}
try {
$num = Zotero_Items::countUpdated($this->userID, $lastsync, 5);
} catch (Exception $e) {
// We can get a MySQL lock timeout here if the upload starts
// after the write lock check above but before we get here
$this->handleUpdatedError($e);
}
// If nothing updated, or if just a few objects and processing is enabled, process synchronously
if ($num == 0 || $num < 5 && Z_CONFIG::$PROCESSORS_ENABLED) {
$queue = false;
}
$params = [];
if (isset($_POST['ft'])) {
$params['ft'] = $_POST['ft'];
}
if (isset($_POST['ftkeys'])) {
$queue = true;
$params['ftkeys'] = $_POST['ftkeys'];
}
if ($queue) {
Zotero_Sync::queueDownload($this->userID, $this->sessionID, $lastsync, $this->apiVersion, $num, $params);
try {
Zotero_Processors::notifyProcessors('download');
} catch (Exception $e) {
Z_Core::logError($e);
}
$locked = $this->responseXML->addChild('locked');
$locked['wait'] = 1000;
} else {
try {
Zotero_Sync::processDownload($this->userID, $lastsync, $doc, $params);
$this->responseXML = simplexml_import_dom($doc);
StatsD::increment("sync.process.download.immediate.success");
} catch (Exception $e) {
StatsD::increment("sync.process.download.immediate.error");
$this->handleUpdatedError($e);
}
}
$this->end();
}
throw new Exception("Unexpected session result {$result}");
}
