private static function processDownloadInternal($userID, $lastsync, DOMDocument $doc, $syncDownloadQueueID = null, $syncDownloadProcessID = null, $params = [])
{
$apiVersion = (int) $doc->documentElement->getAttribute('version');
if ($lastsync == 1) {
StatsD::increment("sync.process.download.full");
}
// TEMP
$cacheKeyExtra = (!empty($params['ft']) ? json_encode($params['ft']) : "") . (!empty($params['ftkeys']) ? json_encode($params['ftkeys']) : "");
try {
$cached = Zotero_Sync::getCachedDownload($userID, $lastsync, $apiVersion, $cacheKeyExtra);
if ($cached) {
$doc->loadXML($cached);
StatsD::increment("sync.process.download.cache.hit");
return;
}
} catch (Exception $e) {
$msg = $e->getMessage();
if (strpos($msg, "Too many connections") !== false) {
$msg = "'Too many connections' from MySQL";
} else {
$msg = "'{$msg}'";
}
Z_Core::logError("Warning: {$msg} getting cached download");
StatsD::increment("sync.process.download.cache.error");
}
set_time_limit(1800);
$profile = false;
if ($profile) {
$shardID = Zotero_Shards::getByUserID($userID);
Zotero_DB::profileStart(0);
}
if ($syncDownloadQueueID) {
self::addDownloadProcess($syncDownloadQueueID, $syncDownloadProcessID);
}
$updatedNode = $doc->createElement('updated');
$doc->documentElement->appendChild($updatedNode);
$userLibraryID = Zotero_Users::getLibraryIDFromUserID($userID);
$updatedCreators = array();
try {
Zotero_DB::beginTransaction();
// Blocks until any upload processes are done
$updateTimes = Zotero_Libraries::getUserLibraryUpdateTimes($userID);
$timestamp = Zotero_DB::getTransactionTimestampUnix();
$doc->documentElement->setAttribute('timestamp', $timestamp);
$doc->documentElement->setAttribute('userID', $userID);
$doc->documentElement->setAttribute('defaultLibraryID', $userLibraryID);
$updateKey = Zotero_Users::getUpdateKey($userID);
$doc->documentElement->setAttribute('updateKey', $updateKey);
// Get libraries with update times >= $timestamp
$updatedLibraryIDs = array();
foreach ($updateTimes as $libraryID => $timestamp) {
if ($timestamp >= $lastsync) {
$updatedLibraryIDs[] = $libraryID;
}
}
// Add new and updated groups
$joinedGroups = Zotero_Groups::getJoined($userID, (int) $lastsync);
$updatedIDs = array_unique(array_merge($joinedGroups, Zotero_Groups::getUpdated($userID, (int) $lastsync)));
if ($updatedIDs) {
$node = $doc->createElement('groups');
$showGroups = false;
foreach ($updatedIDs as $id) {
$group = new Zotero_Group();
$group->id = $id;
$xmlElement = $group->toXML($userID);
$newNode = dom_import_simplexml($xmlElement);
$newNode = $doc->importNode($newNode, true);
$node->appendChild($newNode);
$showGroups = true;
}
if ($showGroups) {
$updatedNode->appendChild($node);
}
}
// If there's updated data in any library or
// there are any new groups (in which case we need all their data)
$hasData = $updatedLibraryIDs || $joinedGroups;
if ($hasData) {
foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) {
$Name = $syncObject['singular'];
// 'Item'
$Names = $syncObject['plural'];
// 'Items'
$name = strtolower($Name);
// 'item'
$names = strtolower($Names);
// 'items'
$className = 'Zotero_' . $Names;
$updatedIDsByLibraryID = call_user_func(array($className, 'getUpdated'), $userID, $lastsync, $updatedLibraryIDs);
if ($updatedIDsByLibraryID) {
$node = $doc->createElement($names);
foreach ($updatedIDsByLibraryID as $libraryID => $ids) {
if ($name == 'creator') {
$updatedCreators[$libraryID] = $ids;
}
foreach ($ids as $id) {
if ($name == 'item') {
$obj = call_user_func(array($className, 'get'), $libraryID, $id);
$data = array('updatedCreators' => isset($updatedCreators[$libraryID]) ? $updatedCreators[$libraryID] : array());
$xmlElement = Zotero_Items::convertItemToXML($obj, $data, $apiVersion);
} else {
$instanceClass = 'Zotero_' . $Name;
$obj = new $instanceClass();
if (method_exists($instanceClass, '__construct')) {
$obj->__construct();
}
$obj->libraryID = $libraryID;
if ($name == 'setting') {
$obj->name = $id;
} else {
$obj->id = $id;
}
if ($name == 'tag') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, true);
} else {
if ($name == 'creator') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc);
if ($xmlElement->getAttribute('libraryID') == $userLibraryID) {
$xmlElement->removeAttribute('libraryID');
}
$node->appendChild($xmlElement);
} else {
if ($name == 'relation') {
// Skip new-style related items
if ($obj->predicate == 'dc:relation') {
continue;
}
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj);
if ($apiVersion <= 8) {
unset($xmlElement['libraryID']);
}
} else {
if ($name == 'setting') {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc);
$node->appendChild($xmlElement);
} else {
$xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj);
}
}
}
}
}
if ($xmlElement instanceof SimpleXMLElement) {
if ($xmlElement['libraryID'] == $userLibraryID) {
unset($xmlElement['libraryID']);
}
$newNode = dom_import_simplexml($xmlElement);
$newNode = $doc->importNode($newNode, true);
$node->appendChild($newNode);
}
}
}
if ($node->hasChildNodes()) {
$updatedNode->appendChild($node);
}
}
}
}
// Add full-text content if the client supports it
if (isset($params['ft'])) {
$libraries = Zotero_Libraries::getUserLibraries($userID);
$fulltextNode = false;
foreach ($libraries as $libraryID) {
if (!empty($params['ftkeys']) && $params['ftkeys'] === 'all') {
$ftlastsync = 1;
} else {
$ftlastsync = $lastsync;
}
if (!empty($params['ftkeys'][$libraryID])) {
$keys = $params['ftkeys'][$libraryID];
} else {
$keys = [];
}
$data = Zotero_FullText::getNewerInLibraryByTime($libraryID, $ftlastsync, $keys);
if ($data) {
if (!$fulltextNode) {
$fulltextNode = $doc->createElement('fulltexts');
}
foreach ($data as $itemData) {
if ($params['ft']) {
$empty = $itemData['empty'];
} else {
$empty = true;
}
$first = false;
$node = Zotero_FullText::itemDataToXML($itemData, $doc, $empty);
$fulltextNode->appendChild($node);
}
}
}
if ($fulltextNode) {
$updatedNode->appendChild($fulltextNode);
}
}
// Get earliest timestamp
$earliestModTime = Zotero_Users::getEarliestDataTimestamp($userID);
$doc->documentElement->setAttribute('earliest', $earliestModTime ? $earliestModTime : 0);
// Deleted objects
$deletedKeys = $hasData ? self::getDeletedObjectKeys($userID, $lastsync, true) : false;
$deletedIDs = self::getDeletedObjectIDs($userID, $lastsync, true);
if ($deletedKeys || $deletedIDs) {
$deletedNode = $doc->createElement('deleted');
// Add deleted data objects
if ($deletedKeys) {
foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) {
$Name = $syncObject['singular'];
// 'Item'
$Names = $syncObject['plural'];
// 'Items'
$name = strtolower($Name);
// 'item'
$names = strtolower($Names);
// 'items'
if (empty($deletedKeys[$names])) {
continue;
}
$typeNode = $doc->createElement($names);
foreach ($deletedKeys[$names] as $row) {
$node = $doc->createElement($name);
if ($row['libraryID'] != $userLibraryID || $name == 'setting') {
$node->setAttribute('libraryID', $row['libraryID']);
}
$node->setAttribute('key', $row['key']);
$typeNode->appendChild($node);
}
$deletedNode->appendChild($typeNode);
}
}
// Add deleted groups
if ($deletedIDs) {
$name = "group";
$names = "groups";
$typeNode = $doc->createElement($names);
$ids = $doc->createTextNode(implode(' ', $deletedIDs[$names]));
$typeNode->appendChild($ids);
$deletedNode->appendChild($typeNode);
}
$updatedNode->appendChild($deletedNode);
}
Zotero_DB::commit();
} catch (Exception $e) {
Zotero_DB::rollback(true);
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
throw $e;
}
function relaxNGErrorHandler($errno, $errstr)
{
Zotero_Sync::$validationError = $errstr;
}
set_error_handler('relaxNGErrorHandler');
$valid = $doc->relaxNGValidate(Z_ENV_MODEL_PATH . 'relax-ng/updated.rng');
restore_error_handler();
if (!$valid) {
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
throw new Exception(self::$validationError . "\n\nXML:\n\n" . $doc->saveXML());
}
// Cache response if response isn't empty
try {
if ($doc->documentElement->firstChild->hasChildNodes()) {
self::cacheDownload($userID, $updateKey, $lastsync, $apiVersion, $doc->saveXML(), $cacheKeyExtra);
}
} catch (Exception $e) {
Z_Core::logError("WARNING: " . $e);
}
if ($syncDownloadQueueID) {
self::removeDownloadProcess($syncDownloadProcessID);
}
if ($profile) {
$shardID = Zotero_Shards::getByUserID($userID);
Zotero_DB::profileEnd(0);
}
}
/**
* Converts a Zotero_Item object to a SimpleXMLElement Atom object
*
* @param object $item Zotero_Item object
* @param string $content
* @return SimpleXMLElement Item data as SimpleXML element
*/
public static function convertItemToAtom(Zotero_Item $item, $queryParams, $apiVersion = null, $permissions = null, $sharedData = null)
{
$content = $queryParams['content'];
$contentIsHTML = sizeOf($content) == 1 && $content[0] == 'html';
$contentParamString = urlencode(implode(',', $content));
$style = $queryParams['style'];
$entry = '<entry xmlns="' . Zotero_Atom::$nsAtom . '" xmlns:zapi="' . Zotero_Atom::$nsZoteroAPI . '"/>';
$xml = new SimpleXMLElement($entry);
$title = $item->getDisplayTitle(true);
$title = $title ? $title : '[Untitled]';
$xml->title = $title;
$author = $xml->addChild('author');
$createdByUserID = null;
switch (Zotero_Libraries::getType($item->libraryID)) {
case 'group':
$createdByUserID = $item->createdByUserID;
break;
}
if ($createdByUserID) {
$author->name = Zotero_Users::getUsername($createdByUserID);
$author->uri = Zotero_URI::getUserURI($createdByUserID);
} else {
$author->name = Zotero_Libraries::getName($item->libraryID);
$author->uri = Zotero_URI::getLibraryURI($item->libraryID);
}
$id = Zotero_URI::getItemURI($item);
/*if (!$contentIsHTML) {
$id .= "?content=$content";
}*/
$xml->id = $id;
$xml->published = Zotero_Date::sqlToISO8601($item->getField('dateAdded'));
$xml->updated = Zotero_Date::sqlToISO8601($item->getField('dateModified'));
$link = $xml->addChild("link");
$link['rel'] = "self";
$link['type'] = "application/atom+xml";
$href = Zotero_Atom::getItemURI($item);
if (!$contentIsHTML) {
$href .= "?content={$contentParamString}";
}
$link['href'] = $href;
$parent = $item->getSource();
if ($parent) {
// TODO: handle group items?
$parentItem = Zotero_Items::get($item->libraryID, $parent);
$link = $xml->addChild("link");
$link['rel'] = "up";
$link['type'] = "application/atom+xml";
$href = Zotero_Atom::getItemURI($parentItem);
if (!$contentIsHTML) {
$href .= "?content={$contentParamString}";
}
$link['href'] = $href;
}
$link = $xml->addChild('link');
$link['rel'] = 'alternate';
$link['type'] = 'text/html';
$link['href'] = Zotero_URI::getItemURI($item);
// If appropriate permissions and the file is stored in ZFS, get file request link
if ($permissions && $permissions->canAccess($item->libraryID, 'files')) {
$details = Zotero_S3::getDownloadDetails($item);
if ($details) {
$link = $xml->addChild('link');
$link['rel'] = 'enclosure';
$type = $item->attachmentMIMEType;
if ($type) {
$link['type'] = $type;
}
$link['href'] = $details['url'];
if (!empty($details['filename'])) {
$link['title'] = $details['filename'];
}
if (!empty($details['size'])) {
$link['length'] = $details['size'];
}
}
}
$xml->addChild('zapi:key', $item->key, Zotero_Atom::$nsZoteroAPI);
$xml->addChild('zapi:itemType', Zotero_ItemTypes::getName($item->itemTypeID), Zotero_Atom::$nsZoteroAPI);
if ($item->isRegularItem()) {
$val = $item->creatorSummary;
if ($val !== '') {
$xml->addChild('zapi:creatorSummary', htmlspecialchars($val), Zotero_Atom::$nsZoteroAPI);
}
$val = substr($item->getField('date', true, true, true), 0, 4);
if ($val !== '' && $val !== '0000') {
$xml->addChild('zapi:year', $val, Zotero_Atom::$nsZoteroAPI);
}
}
if (!$parent && $item->isRegularItem()) {
if ($permissions && !$permissions->canAccess($item->libraryID, 'notes')) {
$numChildren = $item->numAttachments();
} else {
$numChildren = $item->numChildren();
}
$xml->addChild('zapi:numChildren', $numChildren, Zotero_Atom::$nsZoteroAPI);
}
$xml->addChild('zapi:numTags', $item->numTags(), Zotero_Atom::$nsZoteroAPI);
$xml->content = '';
//
// DOM XML from here on out
//
$contentNode = dom_import_simplexml($xml->content);
$domDoc = $contentNode->ownerDocument;
$multiFormat = sizeOf($content) > 1;
// Create a root XML document for multi-format responses
if ($multiFormat) {
$contentNode->setAttribute('type', 'application/xml');
/*$multicontent = $domDoc->createElementNS(
Zotero_Atom::$nsZoteroAPI, 'multicontent'
);
$contentNode->appendChild($multicontent);*/
}
foreach ($content as $type) {
// Set the target to either the main <content>
// or a <multicontent> <content>
if (!$multiFormat) {
$target = $contentNode;
} else {
$target = $domDoc->createElementNS(Zotero_Atom::$nsZoteroAPI, 'subcontent');
$contentNode->appendChild($target);
}
$target->setAttributeNS(Zotero_Atom::$nsZoteroAPI, "zapi:type", $type);
if ($type == 'html') {
if (!$multiFormat) {
$target->setAttribute('type', 'xhtml');
}
$div = $domDoc->createElement('div');
$div->setAttribute('xmlns', Zotero_Atom::$nsXHTML);
$target->appendChild($div);
$html = $item->toHTML(true);
$subNode = dom_import_simplexml($html);
$importedNode = $domDoc->importNode($subNode, true);
$div->appendChild($importedNode);
} else {
if ($type == 'citation') {
if (!$multiFormat) {
$target->setAttribute('type', 'xhtml');
}
if (isset($sharedData[$type][$item->libraryID . "/" . $item->key])) {
$html = $sharedData[$type][$item->libraryID . "/" . $item->key];
} else {
if ($sharedData !== null) {
error_log("Citation not found in sharedData -- retrieving individually");
}
$html = Zotero_Cite::getCitationFromCiteServer($item, $style);
}
$html = new SimpleXMLElement($html);
$html['xmlns'] = Zotero_Atom::$nsXHTML;
$subNode = dom_import_simplexml($html);
$importedNode = $domDoc->importNode($subNode, true);
$target->appendChild($importedNode);
} else {
if ($type == 'bib') {
if (!$multiFormat) {
$target->setAttribute('type', 'xhtml');
}
if (isset($sharedData[$type][$item->libraryID . "/" . $item->key])) {
$html = $sharedData[$type][$item->libraryID . "/" . $item->key];
} else {
if ($sharedData !== null) {
error_log("Bibliography not found in sharedData -- retrieving individually");
}
$html = Zotero_Cite::getBibliographyFromCitationServer(array($item), $style);
}
$html = new SimpleXMLElement($html);
$html['xmlns'] = Zotero_Atom::$nsXHTML;
$subNode = dom_import_simplexml($html);
$importedNode = $domDoc->importNode($subNode, true);
$target->appendChild($importedNode);
} else {
if ($type == 'json') {
$target->setAttributeNS(Zotero_Atom::$nsZoteroAPI, "zapi:etag", $item->etag);
$textNode = $domDoc->createTextNode($item->toJSON(false, $queryParams['pprint'], true));
$target->appendChild($textNode);
} else {
if ($type == 'csljson') {
$arr = $item->toCSLItem();
$mask = JSON_HEX_TAG | JSON_HEX_AMP;
if ($queryParams['pprint']) {
$json = Zotero_Utilities::json_encode_pretty($arr, $mask);
} else {
$json = json_encode($arr, $mask);
}
// Until JSON_UNESCAPED_SLASHES is available
$json = str_replace('\\/', '/', $json);
$textNode = $domDoc->createTextNode($json);
$target->appendChild($textNode);
} else {
if ($type == 'full') {
if (!$multiFormat) {
$target->setAttribute('type', 'xhtml');
}
$fullXML = Zotero_Items::convertItemToXML($item, array(), $apiVersion);
$fullXML->addAttribute("xmlns", Zotero_Atom::$nsZoteroTransfer);
$subNode = dom_import_simplexml($fullXML);
$importedNode = $domDoc->importNode($subNode, true);
$target->appendChild($importedNode);
} else {
if (in_array($type, Zotero_Translate::$exportFormats)) {
$export = Zotero_Translate::doExport(array($item), $type);
$target->setAttribute('type', $export['mimeType']);
// Insert XML into document
if (preg_match('/\\+xml$/', $export['mimeType'])) {
// Strip prolog
$body = preg_replace('/^<\\?xml.+\\n/', "", $export['body']);
$subNode = $domDoc->createDocumentFragment();
$subNode->appendXML($body);
$target->appendChild($subNode);
} else {
$textNode = $domDoc->createTextNode($export['body']);
$target->appendChild($textNode);
}
}
}
}
}
}
}
}
}
return $xml;
}
