private static function processDownloadInternal($userID, $lastsync, DOMDocument $doc, $syncDownloadQueueID = null, $syncDownloadProcessID = null, $params = []) { $apiVersion = (int) $doc->documentElement->getAttribute('version'); if ($lastsync == 1) { StatsD::increment("sync.process.download.full"); } // TEMP $cacheKeyExtra = (!empty($params['ft']) ? json_encode($params['ft']) : "") . (!empty($params['ftkeys']) ? json_encode($params['ftkeys']) : ""); try { $cached = Zotero_Sync::getCachedDownload($userID, $lastsync, $apiVersion, $cacheKeyExtra); if ($cached) { $doc->loadXML($cached); StatsD::increment("sync.process.download.cache.hit"); return; } } catch (Exception $e) { $msg = $e->getMessage(); if (strpos($msg, "Too many connections") !== false) { $msg = "'Too many connections' from MySQL"; } else { $msg = "'{$msg}'"; } Z_Core::logError("Warning: {$msg} getting cached download"); StatsD::increment("sync.process.download.cache.error"); } set_time_limit(1800); $profile = false; if ($profile) { $shardID = Zotero_Shards::getByUserID($userID); Zotero_DB::profileStart(0); } if ($syncDownloadQueueID) { self::addDownloadProcess($syncDownloadQueueID, $syncDownloadProcessID); } $updatedNode = $doc->createElement('updated'); $doc->documentElement->appendChild($updatedNode); $userLibraryID = Zotero_Users::getLibraryIDFromUserID($userID); $updatedCreators = array(); try { Zotero_DB::beginTransaction(); // Blocks until any upload processes are done $updateTimes = Zotero_Libraries::getUserLibraryUpdateTimes($userID); $timestamp = Zotero_DB::getTransactionTimestampUnix(); $doc->documentElement->setAttribute('timestamp', $timestamp); $doc->documentElement->setAttribute('userID', $userID); $doc->documentElement->setAttribute('defaultLibraryID', $userLibraryID); $updateKey = Zotero_Users::getUpdateKey($userID); $doc->documentElement->setAttribute('updateKey', $updateKey); // Get libraries with update times >= $timestamp $updatedLibraryIDs = array(); foreach ($updateTimes as $libraryID => $timestamp) { if ($timestamp >= $lastsync) { $updatedLibraryIDs[] = $libraryID; } } // Add new and updated groups $joinedGroups = Zotero_Groups::getJoined($userID, (int) $lastsync); $updatedIDs = array_unique(array_merge($joinedGroups, Zotero_Groups::getUpdated($userID, (int) $lastsync))); if ($updatedIDs) { $node = $doc->createElement('groups'); $showGroups = false; foreach ($updatedIDs as $id) { $group = new Zotero_Group(); $group->id = $id; $xmlElement = $group->toXML($userID); $newNode = dom_import_simplexml($xmlElement); $newNode = $doc->importNode($newNode, true); $node->appendChild($newNode); $showGroups = true; } if ($showGroups) { $updatedNode->appendChild($node); } } // If there's updated data in any library or // there are any new groups (in which case we need all their data) $hasData = $updatedLibraryIDs || $joinedGroups; if ($hasData) { foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) { $Name = $syncObject['singular']; // 'Item' $Names = $syncObject['plural']; // 'Items' $name = strtolower($Name); // 'item' $names = strtolower($Names); // 'items' $className = 'Zotero_' . $Names; $updatedIDsByLibraryID = call_user_func(array($className, 'getUpdated'), $userID, $lastsync, $updatedLibraryIDs); if ($updatedIDsByLibraryID) { $node = $doc->createElement($names); foreach ($updatedIDsByLibraryID as $libraryID => $ids) { if ($name == 'creator') { $updatedCreators[$libraryID] = $ids; } foreach ($ids as $id) { if ($name == 'item') { $obj = call_user_func(array($className, 'get'), $libraryID, $id); $data = array('updatedCreators' => isset($updatedCreators[$libraryID]) ? $updatedCreators[$libraryID] : array()); $xmlElement = Zotero_Items::convertItemToXML($obj, $data, $apiVersion); } else { $instanceClass = 'Zotero_' . $Name; $obj = new $instanceClass(); if (method_exists($instanceClass, '__construct')) { $obj->__construct(); } $obj->libraryID = $libraryID; if ($name == 'setting') { $obj->name = $id; } else { $obj->id = $id; } if ($name == 'tag') { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, true); } else { if ($name == 'creator') { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc); if ($xmlElement->getAttribute('libraryID') == $userLibraryID) { $xmlElement->removeAttribute('libraryID'); } $node->appendChild($xmlElement); } else { if ($name == 'relation') { // Skip new-style related items if ($obj->predicate == 'dc:relation') { continue; } $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj); if ($apiVersion <= 8) { unset($xmlElement['libraryID']); } } else { if ($name == 'setting') { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc); $node->appendChild($xmlElement); } else { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj); } } } } } if ($xmlElement instanceof SimpleXMLElement) { if ($xmlElement['libraryID'] == $userLibraryID) { unset($xmlElement['libraryID']); } $newNode = dom_import_simplexml($xmlElement); $newNode = $doc->importNode($newNode, true); $node->appendChild($newNode); } } } if ($node->hasChildNodes()) { $updatedNode->appendChild($node); } } } } // Add full-text content if the client supports it if (isset($params['ft'])) { $libraries = Zotero_Libraries::getUserLibraries($userID); $fulltextNode = false; foreach ($libraries as $libraryID) { if (!empty($params['ftkeys']) && $params['ftkeys'] === 'all') { $ftlastsync = 1; } else { $ftlastsync = $lastsync; } if (!empty($params['ftkeys'][$libraryID])) { $keys = $params['ftkeys'][$libraryID]; } else { $keys = []; } $data = Zotero_FullText::getNewerInLibraryByTime($libraryID, $ftlastsync, $keys); if ($data) { if (!$fulltextNode) { $fulltextNode = $doc->createElement('fulltexts'); } foreach ($data as $itemData) { if ($params['ft']) { $empty = $itemData['empty']; } else { $empty = true; } $first = false; $node = Zotero_FullText::itemDataToXML($itemData, $doc, $empty); $fulltextNode->appendChild($node); } } } if ($fulltextNode) { $updatedNode->appendChild($fulltextNode); } } // Get earliest timestamp $earliestModTime = Zotero_Users::getEarliestDataTimestamp($userID); $doc->documentElement->setAttribute('earliest', $earliestModTime ? $earliestModTime : 0); // Deleted objects $deletedKeys = $hasData ? self::getDeletedObjectKeys($userID, $lastsync, true) : false; $deletedIDs = self::getDeletedObjectIDs($userID, $lastsync, true); if ($deletedKeys || $deletedIDs) { $deletedNode = $doc->createElement('deleted'); // Add deleted data objects if ($deletedKeys) { foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) { $Name = $syncObject['singular']; // 'Item' $Names = $syncObject['plural']; // 'Items' $name = strtolower($Name); // 'item' $names = strtolower($Names); // 'items' if (empty($deletedKeys[$names])) { continue; } $typeNode = $doc->createElement($names); foreach ($deletedKeys[$names] as $row) { $node = $doc->createElement($name); if ($row['libraryID'] != $userLibraryID || $name == 'setting') { $node->setAttribute('libraryID', $row['libraryID']); } $node->setAttribute('key', $row['key']); $typeNode->appendChild($node); } $deletedNode->appendChild($typeNode); } } // Add deleted groups if ($deletedIDs) { $name = "group"; $names = "groups"; $typeNode = $doc->createElement($names); $ids = $doc->createTextNode(implode(' ', $deletedIDs[$names])); $typeNode->appendChild($ids); $deletedNode->appendChild($typeNode); } $updatedNode->appendChild($deletedNode); } Zotero_DB::commit(); } catch (Exception $e) { Zotero_DB::rollback(true); if ($syncDownloadQueueID) { self::removeDownloadProcess($syncDownloadProcessID); } throw $e; } function relaxNGErrorHandler($errno, $errstr) { Zotero_Sync::$validationError = $errstr; } set_error_handler('relaxNGErrorHandler'); $valid = $doc->relaxNGValidate(Z_ENV_MODEL_PATH . 'relax-ng/updated.rng'); restore_error_handler(); if (!$valid) { if ($syncDownloadQueueID) { self::removeDownloadProcess($syncDownloadProcessID); } throw new Exception(self::$validationError . "\n\nXML:\n\n" . $doc->saveXML()); } // Cache response if response isn't empty try { if ($doc->documentElement->firstChild->hasChildNodes()) { self::cacheDownload($userID, $updateKey, $lastsync, $apiVersion, $doc->saveXML(), $cacheKeyExtra); } } catch (Exception $e) { Z_Core::logError("WARNING: " . $e); } if ($syncDownloadQueueID) { self::removeDownloadProcess($syncDownloadProcessID); } if ($profile) { $shardID = Zotero_Shards::getByUserID($userID); Zotero_DB::profileEnd(0); } }
/** * Converts a Zotero_Item object to a SimpleXMLElement Atom object * * @param object $item Zotero_Item object * @param string $content * @return SimpleXMLElement Item data as SimpleXML element */ public static function convertItemToAtom(Zotero_Item $item, $queryParams, $apiVersion = null, $permissions = null, $sharedData = null) { $content = $queryParams['content']; $contentIsHTML = sizeOf($content) == 1 && $content[0] == 'html'; $contentParamString = urlencode(implode(',', $content)); $style = $queryParams['style']; $entry = '<entry xmlns="' . Zotero_Atom::$nsAtom . '" xmlns:zapi="' . Zotero_Atom::$nsZoteroAPI . '"/>'; $xml = new SimpleXMLElement($entry); $title = $item->getDisplayTitle(true); $title = $title ? $title : '[Untitled]'; $xml->title = $title; $author = $xml->addChild('author'); $createdByUserID = null; switch (Zotero_Libraries::getType($item->libraryID)) { case 'group': $createdByUserID = $item->createdByUserID; break; } if ($createdByUserID) { $author->name = Zotero_Users::getUsername($createdByUserID); $author->uri = Zotero_URI::getUserURI($createdByUserID); } else { $author->name = Zotero_Libraries::getName($item->libraryID); $author->uri = Zotero_URI::getLibraryURI($item->libraryID); } $id = Zotero_URI::getItemURI($item); /*if (!$contentIsHTML) { $id .= "?content=$content"; }*/ $xml->id = $id; $xml->published = Zotero_Date::sqlToISO8601($item->getField('dateAdded')); $xml->updated = Zotero_Date::sqlToISO8601($item->getField('dateModified')); $link = $xml->addChild("link"); $link['rel'] = "self"; $link['type'] = "application/atom+xml"; $href = Zotero_Atom::getItemURI($item); if (!$contentIsHTML) { $href .= "?content={$contentParamString}"; } $link['href'] = $href; $parent = $item->getSource(); if ($parent) { // TODO: handle group items? $parentItem = Zotero_Items::get($item->libraryID, $parent); $link = $xml->addChild("link"); $link['rel'] = "up"; $link['type'] = "application/atom+xml"; $href = Zotero_Atom::getItemURI($parentItem); if (!$contentIsHTML) { $href .= "?content={$contentParamString}"; } $link['href'] = $href; } $link = $xml->addChild('link'); $link['rel'] = 'alternate'; $link['type'] = 'text/html'; $link['href'] = Zotero_URI::getItemURI($item); // If appropriate permissions and the file is stored in ZFS, get file request link if ($permissions && $permissions->canAccess($item->libraryID, 'files')) { $details = Zotero_S3::getDownloadDetails($item); if ($details) { $link = $xml->addChild('link'); $link['rel'] = 'enclosure'; $type = $item->attachmentMIMEType; if ($type) { $link['type'] = $type; } $link['href'] = $details['url']; if (!empty($details['filename'])) { $link['title'] = $details['filename']; } if (!empty($details['size'])) { $link['length'] = $details['size']; } } } $xml->addChild('zapi:key', $item->key, Zotero_Atom::$nsZoteroAPI); $xml->addChild('zapi:itemType', Zotero_ItemTypes::getName($item->itemTypeID), Zotero_Atom::$nsZoteroAPI); if ($item->isRegularItem()) { $val = $item->creatorSummary; if ($val !== '') { $xml->addChild('zapi:creatorSummary', htmlspecialchars($val), Zotero_Atom::$nsZoteroAPI); } $val = substr($item->getField('date', true, true, true), 0, 4); if ($val !== '' && $val !== '0000') { $xml->addChild('zapi:year', $val, Zotero_Atom::$nsZoteroAPI); } } if (!$parent && $item->isRegularItem()) { if ($permissions && !$permissions->canAccess($item->libraryID, 'notes')) { $numChildren = $item->numAttachments(); } else { $numChildren = $item->numChildren(); } $xml->addChild('zapi:numChildren', $numChildren, Zotero_Atom::$nsZoteroAPI); } $xml->addChild('zapi:numTags', $item->numTags(), Zotero_Atom::$nsZoteroAPI); $xml->content = ''; // // DOM XML from here on out // $contentNode = dom_import_simplexml($xml->content); $domDoc = $contentNode->ownerDocument; $multiFormat = sizeOf($content) > 1; // Create a root XML document for multi-format responses if ($multiFormat) { $contentNode->setAttribute('type', 'application/xml'); /*$multicontent = $domDoc->createElementNS( Zotero_Atom::$nsZoteroAPI, 'multicontent' ); $contentNode->appendChild($multicontent);*/ } foreach ($content as $type) { // Set the target to either the main <content> // or a <multicontent> <content> if (!$multiFormat) { $target = $contentNode; } else { $target = $domDoc->createElementNS(Zotero_Atom::$nsZoteroAPI, 'subcontent'); $contentNode->appendChild($target); } $target->setAttributeNS(Zotero_Atom::$nsZoteroAPI, "zapi:type", $type); if ($type == 'html') { if (!$multiFormat) { $target->setAttribute('type', 'xhtml'); } $div = $domDoc->createElement('div'); $div->setAttribute('xmlns', Zotero_Atom::$nsXHTML); $target->appendChild($div); $html = $item->toHTML(true); $subNode = dom_import_simplexml($html); $importedNode = $domDoc->importNode($subNode, true); $div->appendChild($importedNode); } else { if ($type == 'citation') { if (!$multiFormat) { $target->setAttribute('type', 'xhtml'); } if (isset($sharedData[$type][$item->libraryID . "/" . $item->key])) { $html = $sharedData[$type][$item->libraryID . "/" . $item->key]; } else { if ($sharedData !== null) { error_log("Citation not found in sharedData -- retrieving individually"); } $html = Zotero_Cite::getCitationFromCiteServer($item, $style); } $html = new SimpleXMLElement($html); $html['xmlns'] = Zotero_Atom::$nsXHTML; $subNode = dom_import_simplexml($html); $importedNode = $domDoc->importNode($subNode, true); $target->appendChild($importedNode); } else { if ($type == 'bib') { if (!$multiFormat) { $target->setAttribute('type', 'xhtml'); } if (isset($sharedData[$type][$item->libraryID . "/" . $item->key])) { $html = $sharedData[$type][$item->libraryID . "/" . $item->key]; } else { if ($sharedData !== null) { error_log("Bibliography not found in sharedData -- retrieving individually"); } $html = Zotero_Cite::getBibliographyFromCitationServer(array($item), $style); } $html = new SimpleXMLElement($html); $html['xmlns'] = Zotero_Atom::$nsXHTML; $subNode = dom_import_simplexml($html); $importedNode = $domDoc->importNode($subNode, true); $target->appendChild($importedNode); } else { if ($type == 'json') { $target->setAttributeNS(Zotero_Atom::$nsZoteroAPI, "zapi:etag", $item->etag); $textNode = $domDoc->createTextNode($item->toJSON(false, $queryParams['pprint'], true)); $target->appendChild($textNode); } else { if ($type == 'csljson') { $arr = $item->toCSLItem(); $mask = JSON_HEX_TAG | JSON_HEX_AMP; if ($queryParams['pprint']) { $json = Zotero_Utilities::json_encode_pretty($arr, $mask); } else { $json = json_encode($arr, $mask); } // Until JSON_UNESCAPED_SLASHES is available $json = str_replace('\\/', '/', $json); $textNode = $domDoc->createTextNode($json); $target->appendChild($textNode); } else { if ($type == 'full') { if (!$multiFormat) { $target->setAttribute('type', 'xhtml'); } $fullXML = Zotero_Items::convertItemToXML($item, array(), $apiVersion); $fullXML->addAttribute("xmlns", Zotero_Atom::$nsZoteroTransfer); $subNode = dom_import_simplexml($fullXML); $importedNode = $domDoc->importNode($subNode, true); $target->appendChild($importedNode); } else { if (in_array($type, Zotero_Translate::$exportFormats)) { $export = Zotero_Translate::doExport(array($item), $type); $target->setAttribute('type', $export['mimeType']); // Insert XML into document if (preg_match('/\\+xml$/', $export['mimeType'])) { // Strip prolog $body = preg_replace('/^<\\?xml.+\\n/', "", $export['body']); $subNode = $domDoc->createDocumentFragment(); $subNode->appendXML($body); $target->appendChild($subNode); } else { $textNode = $domDoc->createTextNode($export['body']); $target->appendChild($textNode); } } } } } } } } } return $xml; }