/** * Handle uploaded data, overwriting existing data */ public function upload() { $this->sessionCheck(); // Another session is either queued or writing — upload data won't be valid, // so client should wait and return to /updated with 'upload' flag Zotero_DB::beginTransaction(); if (Zotero_Sync::userIsReadLocked($this->userID) || Zotero_Sync::userIsWriteLocked($this->userID)) { Zotero_DB::commit(); $locked = $this->responseXML->addChild('locked'); $locked['wait'] = $this->getWaitTime($this->sessionID); $this->end(); } Zotero_DB::commit(); $this->clearWaitTime($this->sessionID); if (empty($_REQUEST['updateKey'])) { $this->error(400, 'INVALID_UPLOAD_DATA', 'Update key not provided'); } if ($_REQUEST['updateKey'] != Zotero_Users::getUpdateKey($this->userID)) { $this->e409("Server data has changed since last retrieval"); } // TODO: change to POST if (empty($_REQUEST['data'])) { $this->error(400, 'MISSING_UPLOAD_DATA', 'Uploaded data not provided'); } $xmldata =& $_REQUEST['data']; try { $doc = new DOMDocument(); $doc->loadXML($xmldata, LIBXML_PARSEHUGE); // For huge uploads, make sure notes aren't bigger than SimpleXML can parse if (strlen($xmldata) > 7000000) { $xpath = new DOMXPath($doc); $results = $xpath->query('/data/items/item/note[string-length(text()) > ' . Zotero_Notes::$MAX_NOTE_LENGTH . ']'); if ($results->length) { $noteElem = $results->item(0); $text = $noteElem->textContent; $libraryID = $noteElem->parentNode->getAttribute('libraryID'); $key = $noteElem->parentNode->getAttribute('key'); // UTF-8 (0xC2 0xA0) isn't trimmed by default $whitespace = chr(0x20) . chr(0x9) . chr(0xa) . chr(0xd) . chr(0x0) . chr(0xb) . chr(0xc2) . chr(0xa0); $excerpt = iconv("UTF-8", "UTF-8//IGNORE", Zotero_Notes::noteToTitle(trim($text), true)); $excerpt = trim($excerpt, $whitespace); // If tag-stripped version is empty, just return raw HTML if ($excerpt == '') { $excerpt = iconv("UTF-8", "UTF-8//IGNORE", preg_replace('/\\s+/', ' ', mb_substr(trim($text), 0, Zotero_Notes::$MAX_TITLE_LENGTH))); $excerpt = html_entity_decode($excerpt); $excerpt = trim($excerpt, $whitespace); } $msg = "=Note '" . $excerpt . "...' too long"; if ($key) { $msg .= " for item '" . $libraryID . "/" . $key . "'"; } throw new Exception($msg, Z_ERROR_NOTE_TOO_LONG); } } } catch (Exception $e) { $this->handleUploadError($e, $xmldata); } function relaxNGErrorHandler($errno, $errstr) { //Z_Core::logError($errstr); } set_error_handler('relaxNGErrorHandler'); set_time_limit(60); if (!$doc->relaxNGValidate(Z_ENV_MODEL_PATH . 'relax-ng/upload.rng')) { $id = substr(md5(uniqid(rand(), true)), 0, 10); $str = date("D M j G:i:s T Y") . "\n"; $str .= "IP address: " . $_SERVER['REMOTE_ADDR'] . "\n"; if (isset($_SERVER['HTTP_X_ZOTERO_VERSION'])) { $str .= "Version: " . $_SERVER['HTTP_X_ZOTERO_VERSION'] . "\n"; } $str .= "Error: RELAX NG validation failed\n\n"; $str .= $xmldata; if (!file_put_contents(Z_CONFIG::$SYNC_ERROR_PATH . $id, $str)) { error_log("Unable to save error report to " . Z_CONFIG::$SYNC_ERROR_PATH . $id); } $this->error(500, 'INVALID_UPLOAD_DATA', "Uploaded data not well-formed (Report ID: {$id})"); } restore_error_handler(); try { $xml = simplexml_import_dom($doc); $queue = true; if (Z_ENV_TESTING_SITE && !empty($_GET['noqueue'])) { $queue = false; } if ($queue) { $affectedLibraries = Zotero_Sync::parseAffectedLibraries($xmldata); // Relations-only uploads don't have affected libraries if (!$affectedLibraries) { $affectedLibraries = array(Zotero_Users::getLibraryIDFromUserID($this->userID)); } Zotero_Sync::queueUpload($this->userID, $this->sessionID, $xmldata, $affectedLibraries); try { Zotero_Processors::notifyProcessors('upload'); Zotero_Processors::notifyProcessors('error'); usleep(750000); } catch (Exception $e) { Z_Core::logError($e); } // Give processor a chance to finish while we're still here $this->uploadstatus(); } else { set_time_limit(210); $timestamp = Zotero_Sync::processUpload($this->userID, $xml); $this->responseXML['timestamp'] = $timestamp; $this->responseXML->addChild('uploaded'); $this->end(); } } catch (Exception $e) { $this->handleUploadError($e, $xmldata); } }
private static function processDownloadInternal($userID, $lastsync, DOMDocument $doc, $syncDownloadQueueID = null, $syncDownloadProcessID = null, $params = []) { $apiVersion = (int) $doc->documentElement->getAttribute('version'); if ($lastsync == 1) { StatsD::increment("sync.process.download.full"); } // TEMP $cacheKeyExtra = (!empty($params['ft']) ? json_encode($params['ft']) : "") . (!empty($params['ftkeys']) ? json_encode($params['ftkeys']) : ""); try { $cached = Zotero_Sync::getCachedDownload($userID, $lastsync, $apiVersion, $cacheKeyExtra); if ($cached) { $doc->loadXML($cached); StatsD::increment("sync.process.download.cache.hit"); return; } } catch (Exception $e) { $msg = $e->getMessage(); if (strpos($msg, "Too many connections") !== false) { $msg = "'Too many connections' from MySQL"; } else { $msg = "'{$msg}'"; } Z_Core::logError("Warning: {$msg} getting cached download"); StatsD::increment("sync.process.download.cache.error"); } set_time_limit(1800); $profile = false; if ($profile) { $shardID = Zotero_Shards::getByUserID($userID); Zotero_DB::profileStart(0); } if ($syncDownloadQueueID) { self::addDownloadProcess($syncDownloadQueueID, $syncDownloadProcessID); } $updatedNode = $doc->createElement('updated'); $doc->documentElement->appendChild($updatedNode); $userLibraryID = Zotero_Users::getLibraryIDFromUserID($userID); $updatedCreators = array(); try { Zotero_DB::beginTransaction(); // Blocks until any upload processes are done $updateTimes = Zotero_Libraries::getUserLibraryUpdateTimes($userID); $timestamp = Zotero_DB::getTransactionTimestampUnix(); $doc->documentElement->setAttribute('timestamp', $timestamp); $doc->documentElement->setAttribute('userID', $userID); $doc->documentElement->setAttribute('defaultLibraryID', $userLibraryID); $updateKey = Zotero_Users::getUpdateKey($userID); $doc->documentElement->setAttribute('updateKey', $updateKey); // Get libraries with update times >= $timestamp $updatedLibraryIDs = array(); foreach ($updateTimes as $libraryID => $timestamp) { if ($timestamp >= $lastsync) { $updatedLibraryIDs[] = $libraryID; } } // Add new and updated groups $joinedGroups = Zotero_Groups::getJoined($userID, (int) $lastsync); $updatedIDs = array_unique(array_merge($joinedGroups, Zotero_Groups::getUpdated($userID, (int) $lastsync))); if ($updatedIDs) { $node = $doc->createElement('groups'); $showGroups = false; foreach ($updatedIDs as $id) { $group = new Zotero_Group(); $group->id = $id; $xmlElement = $group->toXML($userID); $newNode = dom_import_simplexml($xmlElement); $newNode = $doc->importNode($newNode, true); $node->appendChild($newNode); $showGroups = true; } if ($showGroups) { $updatedNode->appendChild($node); } } // If there's updated data in any library or // there are any new groups (in which case we need all their data) $hasData = $updatedLibraryIDs || $joinedGroups; if ($hasData) { foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) { $Name = $syncObject['singular']; // 'Item' $Names = $syncObject['plural']; // 'Items' $name = strtolower($Name); // 'item' $names = strtolower($Names); // 'items' $className = 'Zotero_' . $Names; $updatedIDsByLibraryID = call_user_func(array($className, 'getUpdated'), $userID, $lastsync, $updatedLibraryIDs); if ($updatedIDsByLibraryID) { $node = $doc->createElement($names); foreach ($updatedIDsByLibraryID as $libraryID => $ids) { if ($name == 'creator') { $updatedCreators[$libraryID] = $ids; } foreach ($ids as $id) { if ($name == 'item') { $obj = call_user_func(array($className, 'get'), $libraryID, $id); $data = array('updatedCreators' => isset($updatedCreators[$libraryID]) ? $updatedCreators[$libraryID] : array()); $xmlElement = Zotero_Items::convertItemToXML($obj, $data, $apiVersion); } else { $instanceClass = 'Zotero_' . $Name; $obj = new $instanceClass(); if (method_exists($instanceClass, '__construct')) { $obj->__construct(); } $obj->libraryID = $libraryID; if ($name == 'setting') { $obj->name = $id; } else { $obj->id = $id; } if ($name == 'tag') { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, true); } else { if ($name == 'creator') { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc); if ($xmlElement->getAttribute('libraryID') == $userLibraryID) { $xmlElement->removeAttribute('libraryID'); } $node->appendChild($xmlElement); } else { if ($name == 'relation') { // Skip new-style related items if ($obj->predicate == 'dc:relation') { continue; } $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj); if ($apiVersion <= 8) { unset($xmlElement['libraryID']); } } else { if ($name == 'setting') { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj, $doc); $node->appendChild($xmlElement); } else { $xmlElement = call_user_func(array($className, "convert{$Name}ToXML"), $obj); } } } } } if ($xmlElement instanceof SimpleXMLElement) { if ($xmlElement['libraryID'] == $userLibraryID) { unset($xmlElement['libraryID']); } $newNode = dom_import_simplexml($xmlElement); $newNode = $doc->importNode($newNode, true); $node->appendChild($newNode); } } } if ($node->hasChildNodes()) { $updatedNode->appendChild($node); } } } } // Add full-text content if the client supports it if (isset($params['ft'])) { $libraries = Zotero_Libraries::getUserLibraries($userID); $fulltextNode = false; foreach ($libraries as $libraryID) { if (!empty($params['ftkeys']) && $params['ftkeys'] === 'all') { $ftlastsync = 1; } else { $ftlastsync = $lastsync; } if (!empty($params['ftkeys'][$libraryID])) { $keys = $params['ftkeys'][$libraryID]; } else { $keys = []; } $data = Zotero_FullText::getNewerInLibraryByTime($libraryID, $ftlastsync, $keys); if ($data) { if (!$fulltextNode) { $fulltextNode = $doc->createElement('fulltexts'); } foreach ($data as $itemData) { if ($params['ft']) { $empty = $itemData['empty']; } else { $empty = true; } $first = false; $node = Zotero_FullText::itemDataToXML($itemData, $doc, $empty); $fulltextNode->appendChild($node); } } } if ($fulltextNode) { $updatedNode->appendChild($fulltextNode); } } // Get earliest timestamp $earliestModTime = Zotero_Users::getEarliestDataTimestamp($userID); $doc->documentElement->setAttribute('earliest', $earliestModTime ? $earliestModTime : 0); // Deleted objects $deletedKeys = $hasData ? self::getDeletedObjectKeys($userID, $lastsync, true) : false; $deletedIDs = self::getDeletedObjectIDs($userID, $lastsync, true); if ($deletedKeys || $deletedIDs) { $deletedNode = $doc->createElement('deleted'); // Add deleted data objects if ($deletedKeys) { foreach (Zotero_DataObjects::$classicObjectTypes as $syncObject) { $Name = $syncObject['singular']; // 'Item' $Names = $syncObject['plural']; // 'Items' $name = strtolower($Name); // 'item' $names = strtolower($Names); // 'items' if (empty($deletedKeys[$names])) { continue; } $typeNode = $doc->createElement($names); foreach ($deletedKeys[$names] as $row) { $node = $doc->createElement($name); if ($row['libraryID'] != $userLibraryID || $name == 'setting') { $node->setAttribute('libraryID', $row['libraryID']); } $node->setAttribute('key', $row['key']); $typeNode->appendChild($node); } $deletedNode->appendChild($typeNode); } } // Add deleted groups if ($deletedIDs) { $name = "group"; $names = "groups"; $typeNode = $doc->createElement($names); $ids = $doc->createTextNode(implode(' ', $deletedIDs[$names])); $typeNode->appendChild($ids); $deletedNode->appendChild($typeNode); } $updatedNode->appendChild($deletedNode); } Zotero_DB::commit(); } catch (Exception $e) { Zotero_DB::rollback(true); if ($syncDownloadQueueID) { self::removeDownloadProcess($syncDownloadProcessID); } throw $e; } function relaxNGErrorHandler($errno, $errstr) { Zotero_Sync::$validationError = $errstr; } set_error_handler('relaxNGErrorHandler'); $valid = $doc->relaxNGValidate(Z_ENV_MODEL_PATH . 'relax-ng/updated.rng'); restore_error_handler(); if (!$valid) { if ($syncDownloadQueueID) { self::removeDownloadProcess($syncDownloadProcessID); } throw new Exception(self::$validationError . "\n\nXML:\n\n" . $doc->saveXML()); } // Cache response if response isn't empty try { if ($doc->documentElement->firstChild->hasChildNodes()) { self::cacheDownload($userID, $updateKey, $lastsync, $apiVersion, $doc->saveXML(), $cacheKeyExtra); } } catch (Exception $e) { Z_Core::logError("WARNING: " . $e); } if ($syncDownloadQueueID) { self::removeDownloadProcess($syncDownloadProcessID); } if ($profile) { $shardID = Zotero_Shards::getByUserID($userID); Zotero_DB::profileEnd(0); } }
/** * Handle uploaded data, overwriting existing data */ public function upload() { $this->sessionCheck(); // Another session is either queued or writing — upload data won't be valid, // so client should wait and return to /updated with 'upload' flag Zotero_DB::beginTransaction(); if (Zotero_Sync::userIsReadLocked($this->userID) || Zotero_Sync::userIsWriteLocked($this->userID)) { Zotero_DB::commit(); $locked = $this->responseXML->addChild('locked'); $locked['wait'] = $this->getWaitTime($this->sessionID); $this->end(); } Zotero_DB::commit(); $this->clearWaitTime($this->sessionID); if (empty($_REQUEST['updateKey'])) { $this->error(400, 'INVALID_UPLOAD_DATA', 'Update key not provided'); } if ($_REQUEST['updateKey'] != Zotero_Users::getUpdateKey($this->userID)) { $this->e409("Server data has changed since last retrieval"); } // TODO: change to POST if (empty($_REQUEST['data'])) { $this->error(400, 'MISSING_UPLOAD_DATA', 'Uploaded data not provided'); } $xmldata =& $_REQUEST['data']; $doc = new DOMDocument(); $doc->loadXML($xmldata); function relaxNGErrorHandler($errno, $errstr) { //Z_Core::logError($errstr); } set_error_handler('relaxNGErrorHandler'); set_time_limit(60); if (!$doc->relaxNGValidate(Z_ENV_MODEL_PATH . 'relax-ng/upload.rng')) { $id = substr(md5(uniqid(rand(), true)), 0, 10); $str = date("D M j G:i:s T Y") . "\n"; $str .= "IP address: " . $_SERVER['REMOTE_ADDR'] . "\n"; if (isset($_SERVER['HTTP_X_ZOTERO_VERSION'])) { $str .= "Version: " . $_SERVER['HTTP_X_ZOTERO_VERSION'] . "\n"; } $str .= "Error: RELAX NG validation failed\n\n"; $str .= $xmldata; file_put_contents(Z_CONFIG::$SYNC_ERROR_PATH . $id, $str); $this->error(500, 'INVALID_UPLOAD_DATA', "Uploaded data not well-formed (Report ID: {$id})"); } restore_error_handler(); try { $xml = simplexml_import_dom($doc); $queue = true; if (Z_ENV_TESTING_SITE && !empty($_GET['noqueue'])) { $queue = false; } if ($queue) { $affectedLibraries = Zotero_Sync::parseAffectedLibraries($xmldata); // Relations-only uploads don't have affected libraries if (!$affectedLibraries) { $affectedLibraries = array(Zotero_Users::getLibraryIDFromUserID($this->userID)); } Zotero_Sync::queueUpload($this->userID, $this->sessionID, $xmldata, $affectedLibraries); try { Zotero_Processors::notifyProcessors('upload'); Zotero_Processors::notifyProcessors('error'); usleep(750000); } catch (Exception $e) { Z_Core::logError($e); } // Give processor a chance to finish while we're still here $this->uploadstatus(); } else { set_time_limit(210); $timestamp = Zotero_Sync::processUpload($this->userID, $xml); $this->responseXML['timestamp'] = $timestamp; $this->responseXML->addChild('uploaded'); Zotero_Processors::notifyProcessors('index'); $this->end(); } } catch (Exception $e) { $this->handleUploadError($e, $xmldata); } }