/** * Converts a Zotero_Item object to a SimpleXMLElement Atom object * * Note: Increment Z_CONFIG::$CACHE_VERSION_ATOM_ENTRY when changing * the response. * * @param object $item Zotero_Item object * @param string $content * @return SimpleXMLElement Item data as SimpleXML element */ public static function convertItemToAtom(Zotero_Item $item, $queryParams, $permissions, $sharedData = null) { $t = microtime(true); // Uncached stuff or parts of the cache key $version = $item->version; $parent = $item->getSource(); $isRegularItem = !$parent && $item->isRegularItem(); $downloadDetails = $permissions->canAccess($item->libraryID, 'files') ? Zotero_Storage::getDownloadDetails($item) : false; if ($isRegularItem) { $numChildren = $permissions->canAccess($item->libraryID, 'notes') ? $item->numChildren() : $item->numAttachments(); } // <id> changes based on group visibility in v1 if ($queryParams['v'] < 2) { $id = Zotero_URI::getItemURI($item, false, true); } else { $id = Zotero_URI::getItemURI($item); } $libraryType = Zotero_Libraries::getType($item->libraryID); // Any query parameters that have an effect on the output // need to be added here $allowedParams = array('content', 'style', 'css', 'linkwrap'); $cachedParams = Z_Array::filterKeys($queryParams, $allowedParams); $cacheVersion = 2; $cacheKey = "atomEntry_" . $item->libraryID . "/" . $item->id . "_" . md5($version . json_encode($cachedParams) . ($downloadDetails ? 'hasFile' : '') . ($libraryType == 'group' ? 'id' . $id : '')) . "_" . $queryParams['v'] . "_" . $cacheVersion . (isset(Z_CONFIG::$CACHE_VERSION_ATOM_ENTRY) ? "_" . Z_CONFIG::$CACHE_VERSION_ATOM_ENTRY : "") . (in_array('bib', $queryParams['content']) && isset(Z_CONFIG::$CACHE_VERSION_BIB) ? "_" . Z_CONFIG::$CACHE_VERSION_BIB : ""); $xmlstr = Z_Core::$MC->get($cacheKey); if ($xmlstr) { try { // TEMP: Strip control characters $xmlstr = Zotero_Utilities::cleanString($xmlstr, true); $doc = new DOMDocument(); $doc->loadXML($xmlstr); $xpath = new DOMXpath($doc); $xpath->registerNamespace('atom', Zotero_Atom::$nsAtom); $xpath->registerNamespace('zapi', Zotero_Atom::$nsZoteroAPI); $xpath->registerNamespace('xhtml', Zotero_Atom::$nsXHTML); // Make sure numChildren reflects the current permissions if ($isRegularItem) { $xpath->query('/atom:entry/zapi:numChildren')->item(0)->nodeValue = $numChildren; } // To prevent PHP from messing with namespace declarations, // we have to extract, remove, and then add back <content> // subelements. Otherwise the subelements become, say, // <default:span xmlns="http://www.w3.org/1999/xhtml"> instead // of just <span xmlns="http://www.w3.org/1999/xhtml">, and // xmlns:default="http://www.w3.org/1999/xhtml" gets added to // the parent <entry>. While you might reasonably think that // // echo $xml->saveXML(); // // and // // $xml = new SimpleXMLElement($xml->saveXML()); // echo $xml->saveXML(); // // would be identical, you would be wrong. $multiFormat = !!$xpath->query('/atom:entry/atom:content/zapi:subcontent')->length; $contentNodes = array(); if ($multiFormat) { $contentNodes = $xpath->query('/atom:entry/atom:content/zapi:subcontent'); } else { $contentNodes = $xpath->query('/atom:entry/atom:content'); } foreach ($contentNodes as $contentNode) { $contentParts = array(); while ($contentNode->hasChildNodes()) { $contentParts[] = $doc->saveXML($contentNode->firstChild); $contentNode->removeChild($contentNode->firstChild); } foreach ($contentParts as $part) { if (!trim($part)) { continue; } // Strip the namespace and add it back via SimpleXMLElement, // which keeps it from being changed later if (preg_match('%^<[^>]+xmlns="http://www.w3.org/1999/xhtml"%', $part)) { $part = preg_replace('%^(<[^>]+)xmlns="http://www.w3.org/1999/xhtml"%', '$1', $part); $html = new SimpleXMLElement($part); $html['xmlns'] = "http://www.w3.org/1999/xhtml"; $subNode = dom_import_simplexml($html); $importedNode = $doc->importNode($subNode, true); $contentNode->appendChild($importedNode); } else { if (preg_match('%^<[^>]+xmlns="http://zotero.org/ns/transfer"%', $part)) { $part = preg_replace('%^(<[^>]+)xmlns="http://zotero.org/ns/transfer"%', '$1', $part); $html = new SimpleXMLElement($part); $html['xmlns'] = "http://zotero.org/ns/transfer"; $subNode = dom_import_simplexml($html); $importedNode = $doc->importNode($subNode, true); $contentNode->appendChild($importedNode); } else { $docFrag = $doc->createDocumentFragment(); $docFrag->appendXML($part); $contentNode->appendChild($docFrag); } } } } $xml = simplexml_import_dom($doc); StatsD::timing("api.items.itemToAtom.cached", (microtime(true) - $t) * 1000); StatsD::increment("memcached.items.itemToAtom.hit"); // Skip the cache every 10 times for now, to ensure cache sanity if (Z_Core::probability(10)) { $xmlstr = $xml->saveXML(); } else { return $xml; } } catch (Exception $e) { error_log($xmlstr); error_log("WARNING: " . $e); } } $content = $queryParams['content']; $contentIsHTML = sizeOf($content) == 1 && $content[0] == 'html'; $contentParamString = urlencode(implode(',', $content)); $style = $queryParams['style']; $entry = '<?xml version="1.0" encoding="UTF-8"?>' . '<entry xmlns="' . Zotero_Atom::$nsAtom . '" xmlns:zapi="' . Zotero_Atom::$nsZoteroAPI . '"/>'; $xml = new SimpleXMLElement($entry); $title = $item->getDisplayTitle(true); $title = $title ? $title : '[Untitled]'; $xml->title = $title; $author = $xml->addChild('author'); $createdByUserID = null; $lastModifiedByUserID = null; switch (Zotero_Libraries::getType($item->libraryID)) { case 'group': $createdByUserID = $item->createdByUserID; // Used for zapi:lastModifiedByUser below $lastModifiedByUserID = $item->lastModifiedByUserID; break; } if ($createdByUserID) { $author->name = Zotero_Users::getUsername($createdByUserID); $author->uri = Zotero_URI::getUserURI($createdByUserID); } else { $author->name = Zotero_Libraries::getName($item->libraryID); $author->uri = Zotero_URI::getLibraryURI($item->libraryID); } $xml->id = $id; $xml->published = Zotero_Date::sqlToISO8601($item->dateAdded); $xml->updated = Zotero_Date::sqlToISO8601($item->dateModified); $link = $xml->addChild("link"); $link['rel'] = "self"; $link['type'] = "application/atom+xml"; $href = Zotero_API::getItemURI($item); if (!$contentIsHTML) { $href .= "?content={$contentParamString}"; } $link['href'] = $href; if ($parent) { // TODO: handle group items? $parentItem = Zotero_Items::get($item->libraryID, $parent); $link = $xml->addChild("link"); $link['rel'] = "up"; $link['type'] = "application/atom+xml"; $href = Zotero_API::getItemURI($parentItem); if (!$contentIsHTML) { $href .= "?content={$contentParamString}"; } $link['href'] = $href; } $link = $xml->addChild('link'); $link['rel'] = 'alternate'; $link['type'] = 'text/html'; $link['href'] = Zotero_URI::getItemURI($item, true); // If appropriate permissions and the file is stored in ZFS, get file request link if ($downloadDetails) { $details = $downloadDetails; $link = $xml->addChild('link'); $link['rel'] = 'enclosure'; $type = $item->attachmentMIMEType; if ($type) { $link['type'] = $type; } $link['href'] = $details['url']; if (!empty($details['filename'])) { $link['title'] = $details['filename']; } if (isset($details['size'])) { $link['length'] = $details['size']; } } $xml->addChild('zapi:key', $item->key, Zotero_Atom::$nsZoteroAPI); $xml->addChild('zapi:version', $item->version, Zotero_Atom::$nsZoteroAPI); if ($lastModifiedByUserID) { $xml->addChild('zapi:lastModifiedByUser', Zotero_Users::getUsername($lastModifiedByUserID), Zotero_Atom::$nsZoteroAPI); } $xml->addChild('zapi:itemType', Zotero_ItemTypes::getName($item->itemTypeID), Zotero_Atom::$nsZoteroAPI); if ($isRegularItem) { $val = $item->creatorSummary; if ($val !== '') { $xml->addChild('zapi:creatorSummary', htmlspecialchars($val), Zotero_Atom::$nsZoteroAPI); } $val = $item->getField('date', true, true, true); if ($val !== '') { if ($queryParams['v'] < 3) { $val = substr($val, 0, 4); if ($val !== '0000') { $xml->addChild('zapi:year', $val, Zotero_Atom::$nsZoteroAPI); } } else { $sqlDate = Zotero_Date::multipartToSQL($val); if (substr($sqlDate, 0, 4) !== '0000') { $xml->addChild('zapi:parsedDate', Zotero_Date::sqlToISO8601($sqlDate), Zotero_Atom::$nsZoteroAPI); } } } $xml->addChild('zapi:numChildren', $numChildren, Zotero_Atom::$nsZoteroAPI); } if ($queryParams['v'] < 3) { $xml->addChild('zapi:numTags', $item->numTags(), Zotero_Atom::$nsZoteroAPI); } $xml->content = ''; // // DOM XML from here on out // $contentNode = dom_import_simplexml($xml->content); $domDoc = $contentNode->ownerDocument; $multiFormat = sizeOf($content) > 1; // Create a root XML document for multi-format responses if ($multiFormat) { $contentNode->setAttribute('type', 'application/xml'); /*$multicontent = $domDoc->createElementNS( Zotero_Atom::$nsZoteroAPI, 'multicontent' ); $contentNode->appendChild($multicontent);*/ } foreach ($content as $type) { // Set the target to either the main <content> // or a <multicontent> <content> if (!$multiFormat) { $target = $contentNode; } else { $target = $domDoc->createElementNS(Zotero_Atom::$nsZoteroAPI, 'subcontent'); $contentNode->appendChild($target); } $target->setAttributeNS(Zotero_Atom::$nsZoteroAPI, "zapi:type", $type); if ($type == 'html') { if (!$multiFormat) { $target->setAttribute('type', 'xhtml'); } $div = $domDoc->createElementNS(Zotero_Atom::$nsXHTML, 'div'); $target->appendChild($div); $html = $item->toHTML(true); $subNode = dom_import_simplexml($html); $importedNode = $domDoc->importNode($subNode, true); $div->appendChild($importedNode); } else { if ($type == 'citation') { if (!$multiFormat) { $target->setAttribute('type', 'xhtml'); } if (isset($sharedData[$type][$item->libraryID . "/" . $item->key])) { $html = $sharedData[$type][$item->libraryID . "/" . $item->key]; } else { if ($sharedData !== null) { //error_log("Citation not found in sharedData -- retrieving individually"); } $html = Zotero_Cite::getCitationFromCiteServer($item, $queryParams); } $html = new SimpleXMLElement($html); $html['xmlns'] = Zotero_Atom::$nsXHTML; $subNode = dom_import_simplexml($html); $importedNode = $domDoc->importNode($subNode, true); $target->appendChild($importedNode); } else { if ($type == 'bib') { if (!$multiFormat) { $target->setAttribute('type', 'xhtml'); } if (isset($sharedData[$type][$item->libraryID . "/" . $item->key])) { $html = $sharedData[$type][$item->libraryID . "/" . $item->key]; } else { if ($sharedData !== null) { //error_log("Bibliography not found in sharedData -- retrieving individually"); } $html = Zotero_Cite::getBibliographyFromCitationServer(array($item), $queryParams); } $html = new SimpleXMLElement($html); $html['xmlns'] = Zotero_Atom::$nsXHTML; $subNode = dom_import_simplexml($html); $importedNode = $domDoc->importNode($subNode, true); $target->appendChild($importedNode); } else { if ($type == 'json') { if ($queryParams['v'] < 2) { $target->setAttributeNS(Zotero_Atom::$nsZoteroAPI, "zapi:etag", $item->etag); } $textNode = $domDoc->createTextNode($item->toJSON(false, $queryParams, true)); $target->appendChild($textNode); } else { if ($type == 'csljson') { $arr = $item->toCSLItem(); $json = Zotero_Utilities::formatJSON($arr); $textNode = $domDoc->createTextNode($json); $target->appendChild($textNode); } else { if (in_array($type, Zotero_Translate::$exportFormats)) { $export = Zotero_Translate::doExport(array($item), $type); $target->setAttribute('type', $export['mimeType']); // Insert XML into document if (preg_match('/\\+xml$/', $export['mimeType'])) { // Strip prolog $body = preg_replace('/^<\\?xml.+\\n/', "", $export['body']); $subNode = $domDoc->createDocumentFragment(); $subNode->appendXML($body); $target->appendChild($subNode); } else { $textNode = $domDoc->createTextNode($export['body']); $target->appendChild($textNode); } } } } } } } } // TEMP if ($xmlstr) { $uncached = $xml->saveXML(); if ($xmlstr != $uncached) { $uncached = str_replace('<zapi:year></zapi:year>', '<zapi:year/>', $uncached); $uncached = str_replace('<content zapi:type="none"></content>', '<content zapi:type="none"/>', $uncached); $uncached = str_replace('<zapi:subcontent zapi:type="coins" type="text/html"></zapi:subcontent>', '<zapi:subcontent zapi:type="coins" type="text/html"/>', $uncached); $uncached = str_replace('<title></title>', '<title/>', $uncached); $uncached = str_replace('<note></note>', '<note/>', $uncached); $uncached = str_replace('<path></path>', '<path/>', $uncached); $uncached = str_replace('<td></td>', '<td/>', $uncached); if ($xmlstr != $uncached) { error_log("Cached Atom item entry does not match"); error_log(" Cached: " . $xmlstr); error_log("Uncached: " . $uncached); Z_Core::$MC->set($cacheKey, $uncached, 3600); // 1 hour for now } } } else { $xmlstr = $xml->saveXML(); Z_Core::$MC->set($cacheKey, $xmlstr, 3600); // 1 hour for now StatsD::timing("api.items.itemToAtom.uncached", (microtime(true) - $t) * 1000); StatsD::increment("memcached.items.itemToAtom.miss"); } return $xml; }
/** * Download file from S3, extract it if necessary, and return a temporary URL * pointing to the main file */ public static function getTemporaryURL(Zotero_Item $item, $localOnly = false) { $extURLPrefix = Z_CONFIG::$ATTACHMENT_SERVER_URL; if ($extURLPrefix[strlen($extURLPrefix) - 1] != "/") { $extURLPrefix .= "/"; } $info = Zotero_Storage::getLocalFileItemInfo($item); $storageFileID = $info['storageFileID']; $filename = $info['filename']; $mtime = $info['mtime']; $zip = $info['zip']; $realFilename = preg_replace("/^storage:/", "", $item->attachmentPath); $realFilename = self::decodeRelativeDescriptorString($realFilename); $realEncodedFilename = rawurlencode($realFilename); $docroot = Z_CONFIG::$ATTACHMENT_SERVER_DOCROOT; // Check memcached to see if file is already extracted $key = "attachmentServerString_" . $storageFileID . "_" . $mtime; if ($randomStr = Z_Core::$MC->get($key)) { Z_Core::debug("Got attachment path '{$randomStr}/{$realEncodedFilename}' from memcached"); return $extURLPrefix . "{$randomStr}/{$realEncodedFilename}"; } $localAddr = gethostbyname(gethostname()); // See if this is an attachment host $index = false; $skipHost = false; for ($i = 0, $len = sizeOf(Z_CONFIG::$ATTACHMENT_SERVER_HOSTS); $i < $len; $i++) { $hostAddr = gethostbyname(Z_CONFIG::$ATTACHMENT_SERVER_HOSTS[$i]); if ($hostAddr != $localAddr) { continue; } // Make a HEAD request on the local static port to make sure // this host is actually functional $url = "http://" . Z_CONFIG::$ATTACHMENT_SERVER_HOSTS[$i] . ":" . Z_CONFIG::$ATTACHMENT_SERVER_STATIC_PORT . "/"; Z_Core::debug("Making HEAD request to {$url}"); $ch = curl_init($url); curl_setopt($ch, CURLOPT_NOBODY, 1); curl_setopt($ch, CURLOPT_HTTPHEADER, array("Expect:")); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 1); curl_setopt($ch, CURLOPT_TIMEOUT, 2); curl_setopt($ch, CURLOPT_HEADER, 0); // do not return HTTP headers curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); $response = curl_exec($ch); $code = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($code != 200) { $skipHost = Z_CONFIG::$ATTACHMENT_SERVER_HOSTS[$i]; if ($code == 0) { Z_Core::logError("Error connecting to local attachments server"); } else { Z_Core::logError("Local attachments server returned {$code}"); } break; } $index = $i + 1; break; } // If not, make an internal root request to trigger the extraction on // one of them and retrieve the temporary URL if ($index === false) { // Prevent redirect madness if target server doesn't think it's an // attachment server if ($localOnly) { throw new Exception("Internal attachments request hit a non-attachment server"); } $prefix = 'http://' . Z_CONFIG::$API_SUPER_USERNAME . ":" . Z_CONFIG::$API_SUPER_PASSWORD . "@"; $path = Zotero_API::getItemURI($item) . "/file/view?int=1"; $path = preg_replace('/^[^:]+:\\/\\/[^\\/]+/', '', $path); $context = stream_context_create(array('http' => array('follow_location' => 0))); $url = false; $hosts = Z_CONFIG::$ATTACHMENT_SERVER_HOSTS; // Try in random order shuffle($hosts); foreach ($hosts as $host) { // Don't try the local host again if we know it's not working if ($host == $skipHost) { continue; } $intURL = $prefix . $host . ":" . Z_CONFIG::$ATTACHMENT_SERVER_DYNAMIC_PORT . $path; Z_Core::debug("Making GET request to {$host}"); if (file_get_contents($intURL, false, $context) !== false) { foreach ($http_response_header as $header) { if (preg_match('/^Location:\\s*(.+)$/', $header, $matches)) { if (strpos($matches[1], $extURLPrefix) !== 0) { throw new Exception("Redirect location '" . $matches[1] . "'" . " does not begin with {$extURLPrefix}"); } return $matches[1]; } } } } return false; } // If this is an attachment host, do the download/extraction inline // and generate a random number with an embedded host id. // // The reverse proxy routes incoming file requests to the proper hosts // using the embedded id. // // A cron job deletes old attachment directories $randomStr = rand(1000000, 2147483647); // Seventh number is the host id $randomStr = substr($randomStr, 0, 6) . $index . substr($randomStr, 6); // Download file $dir = $docroot . $randomStr . "/"; $downloadDir = $zip ? $dir . "ztmp/" : $dir; Z_Core::debug("Downloading attachment to {$dir}"); if (!mkdir($downloadDir, 0777, true)) { throw new Exception("Unable to create directory '{$downloadDir}'"); } if ($zip) { $response = Zotero_Storage::downloadFile($info, $downloadDir); } else { $response = Zotero_Storage::downloadFile($info, $downloadDir, $realFilename); } if ($response) { if ($zip) { $success = self::extractZip($downloadDir . $info['filename'], $dir); unlink($downloadDir . $info['filename']); rmdir($downloadDir); // Make sure charset is just a string with no spaces or newlines if (preg_match('/^[^\\s]+/', trim($item->attachmentCharset), $matches)) { $charset = $matches[0]; } else { $charset = 'Off'; } file_put_contents($dir . ".htaccess", "AddDefaultCharset " . $charset); } else { $success = true; if (preg_match('/^[^\\s]+/', trim($item->attachmentContentType), $matches)) { $contentType = $matches[0]; $charset = trim($item->attachmentCharset); if (substr($charset, 0, 5) == 'text/' && preg_match('/^[^\\s]+/', $charset, $matches)) { $contentType .= '; ' . $matches[0]; } file_put_contents($dir . ".htaccess", "ForceType " . $contentType); } } } if (!$response || !$success) { return false; } Z_Core::$MC->set($key, $randomStr, self::$cacheTime); return $extURLPrefix . "{$randomStr}/" . $realEncodedFilename; }
public function toResponseJSON($requestParams=[], Zotero_Permissions $permissions, $sharedData=null) { $t = microtime(true); if (!$this->loaded['primaryData']) { $this->loadPrimaryData(); } if (!$this->loaded['itemData']) { $this->loadItemData(); } // Uncached stuff or parts of the cache key $version = $this->version; $parent = $this->getSource(); $isRegularItem = !$parent && $this->isRegularItem(); $downloadDetails = $permissions->canAccess($this->libraryID, 'files') ? Zotero_Storage::getDownloadDetails($this) : false; if ($isRegularItem) { $numChildren = $permissions->canAccess($this->libraryID, 'notes') ? $this->numChildren() : $this->numAttachments(); } $libraryType = Zotero_Libraries::getType($this->libraryID); // Any query parameters that have an effect on the output // need to be added here $allowedParams = [ 'include', 'style', 'css', 'linkwrap' ]; $cachedParams = Z_Array::filterKeys($requestParams, $allowedParams); $cacheVersion = 1; $cacheKey = "jsonEntry_" . $this->libraryID . "/" . $this->id . "_" . md5( $version . json_encode($cachedParams) . ($downloadDetails ? 'hasFile' : '') // For groups, include the group WWW URL, which can change . ($libraryType == 'group' ? Zotero_URI::getItemURI($this, true) : '') ) . "_" . $requestParams['v'] // For code-based changes . "_" . $cacheVersion // For data-based changes . (isset(Z_CONFIG::$CACHE_VERSION_RESPONSE_JSON_ITEM) ? "_" . Z_CONFIG::$CACHE_VERSION_RESPONSE_JSON_ITEM : "") // If there's bib content, include the bib cache version . ((in_array('bib', $requestParams['include']) && isset(Z_CONFIG::$CACHE_VERSION_BIB)) ? "_" . Z_CONFIG::$CACHE_VERSION_BIB : ""); $cached = Z_Core::$MC->get($cacheKey); if (false && $cached) { // Make sure numChildren reflects the current permissions if ($isRegularItem) { $json = json_decode($cached); $json['numChildren'] = $numChildren; $cached = json_encode($json); } //StatsD::timing("api.items.itemToResponseJSON.cached", (microtime(true) - $t) * 1000); //StatsD::increment("memcached.items.itemToResponseJSON.hit"); // Skip the cache every 10 times for now, to ensure cache sanity if (!Z_Core::probability(10)) { return $cached; } } $json = [ 'key' => $this->key, 'version' => $version, 'library' => Zotero_Libraries::toJSON($this->libraryID) ]; $json['links'] = [ 'self' => [ 'href' => Zotero_API::getItemURI($this), 'type' => 'application/json' ], 'alternate' => [ 'href' => Zotero_URI::getItemURI($this, true), 'type' => 'text/html' ] ]; if ($parent) { $parentItem = Zotero_Items::get($this->libraryID, $parent); $json['links']['up'] = [ 'href' => Zotero_API::getItemURI($parentItem), 'type' => 'application/json' ]; } // If appropriate permissions and the file is stored in ZFS, get file request link if ($downloadDetails) { $details = $downloadDetails; $type = $this->attachmentMIMEType; if ($type) { $json['links']['enclosure'] = [ 'type' => $type ]; } $json['links']['enclosure']['href'] = $details['url']; if (!empty($details['filename'])) { $json['links']['enclosure']['title'] = $details['filename']; } if (isset($details['size'])) { $json['links']['enclosure']['length'] = $details['size']; } } // 'meta' $json['meta'] = new stdClass; if (Zotero_Libraries::getType($this->libraryID) == 'group') { $createdByUserID = $this->createdByUserID; $lastModifiedByUserID = $this->lastModifiedByUserID; if ($createdByUserID) { $json['meta']->createdByUser = Zotero_Users::toJSON($createdByUserID); } if ($lastModifiedByUserID && $lastModifiedByUserID != $createdByUserID) { $json['meta']->lastModifiedByUser = Zotero_Users::toJSON($lastModifiedByUserID); } } if ($isRegularItem) { $val = $this->getCreatorSummary(); if ($val !== '') { $json['meta']->creatorSummary = $val; } $val = $this->getField('date', true, true, true); if ($val !== '') { $sqlDate = Zotero_Date::multipartToSQL($val); if (substr($sqlDate, 0, 4) !== '0000') { $json['meta']->parsedDate = Zotero_Date::sqlToISO8601($sqlDate); } } $json['meta']->numChildren = $numChildren; } // 'include' $include = $requestParams['include']; foreach ($include as $type) { if ($type == 'html') { $json[$type] = trim($this->toHTML()); } else if ($type == 'citation') { if (isset($sharedData[$type][$this->libraryID . "/" . $this->key])) { $html = $sharedData[$type][$this->libraryID . "/" . $this->key]; } else { if ($sharedData !== null) { //error_log("Citation not found in sharedData -- retrieving individually"); } $html = Zotero_Cite::getCitationFromCiteServer($this, $requestParams); } $json[$type] = $html; } else if ($type == 'bib') { if (isset($sharedData[$type][$this->libraryID . "/" . $this->key])) { $html = $sharedData[$type][$this->libraryID . "/" . $this->key]; } else { if ($sharedData !== null) { //error_log("Bibliography not found in sharedData -- retrieving individually"); } $html = Zotero_Cite::getBibliographyFromCitationServer([$this], $requestParams); // Strip prolog $html = preg_replace('/^<\?xml.+\n/', "", $html); $html = trim($html); } $json[$type] = $html; } else if ($type == 'data') { $json[$type] = $this->toJSON(true, $requestParams, true); } else if ($type == 'csljson') { $json[$type] = $this->toCSLItem(); } else if (in_array($type, Zotero_Translate::$exportFormats)) { $export = Zotero_Translate::doExport([$this], $type); $json[$type] = $export['body']; unset($export); } } // TEMP if ($cached) { $uncached = Zotero_Utilities::formatJSON($json); if ($cached != $uncached) { error_log("Cached JSON item entry does not match"); error_log(" Cached: " . $cached); error_log("Uncached: " . $uncached); //Z_Core::$MC->set($cacheKey, $uncached, 3600); // 1 hour for now } } else { /*Z_Core::$MC->set($cacheKey, $xmlstr, 3600); // 1 hour for now StatsD::timing("api.items.itemToAtom.uncached", (microtime(true) - $t) * 1000); StatsD::increment("memcached.items.itemToAtom.miss");*/ } return $json; }
public function storagetransferbucket() { // DISABLED $this->e404(); if (!$this->permissions->isSuper()) { $this->e404(); } $this->allowMethods(array('POST')); Zotero_Storage::transferBucket('zoterofilestorage', 'zoterofilestoragetest'); exit; }
public static function patchFile($item, $info, $algorithm, $patch) { switch ($algorithm) { case 'bsdiff': case 'xdelta': case 'vcdiff': break; case 'xdiff': if (!function_exists('xdiff_file_patch_binary')) { throw new Exception("=xdiff not available"); } break; default: throw new Exception("Invalid algorithm '{$algorithm}'", Z_ERROR_INVALID_INPUT); } $originalInfo = Zotero_Storage::getLocalFileItemInfo($item); $basePath = "/tmp/zfsupload/"; $path = $basePath . $info->hash . "_" . uniqid() . "/"; mkdir($path, 0777, true); $cleanup = function () use($basePath, $path) { unlink("original"); unlink("patch"); unlink("new"); chdir($basePath); rmdir($path); }; $e = null; try { // Download file from S3 to temp directory if (!Zotero_Storage::downloadFile($originalInfo, $path, "original")) { throw new Exception("Error downloading original file"); } chdir($path); // Save body to temp file file_put_contents("patch", $patch); // Patch file switch ($algorithm) { case 'bsdiff': exec('bspatch original new patch 2>&1', $output, $ret); if ($ret) { throw new Exception("Error applying patch ({$ret}): " . implode("\n", $output)); } if (!file_exists("new")) { throw new Exception("Error applying patch ({$ret})"); } break; case 'xdelta': case 'vcdiff': exec('xdelta3 -d -s original patch new 2>&1', $output, $ret); if ($ret) { if ($ret == 2) { throw new Exception("Invalid delta", Z_ERROR_INVALID_INPUT); } throw new Exception("Error applying patch ({$ret}): " . implode("\n", $output)); } if (!file_exists("new")) { throw new Exception("Error applying patch ({$ret})"); } break; case 'xdiff': $ret = xdiff_file_patch_binary("original", "patch", "new"); if (!$ret) { throw new Exception("Error applying patch"); } break; } // Check MD5 hash if (md5_file("new") != $info->hash) { $cleanup(); throw new HTTPException("Patched file does not match hash", 409); } // Check file size if (filesize("new") != $info->size) { $cleanup(); throw new HTTPException("Patched file size does not match " . "(" . filesize("new") . " != {$info->size})", 409); } // If ZIP, make sure it's a ZIP if ($info->zip && file_get_contents("new", false, null, 0, 4) != "PK" . chr(03) . chr(04)) { $cleanup(); throw new HTTPException("Patched file is not a ZIP file", 409); } // Upload to S3 $t = $info->contentType . ($info->contentType && $info->charset ? "; charset={$info->charset}" : ""); $storageFileID = Zotero_Storage::uploadFile($info, "new", $t); } catch (Exception $e) { //$cleanup(); throw $e; } return $storageFileID; }
/** * Handle S3 request * * Permission-checking provided by items() */ private function _handleFileRequest($item) { if (!$this->permissions->canAccess($this->objectLibraryID, 'files')) { $this->e403(); } $this->allowMethods(array('HEAD', 'GET', 'POST', 'PATCH')); if (!$item->isAttachment()) { $this->e400("Item is not an attachment"); } // File info for 4.0 client sync // // Use of HEAD method was discontinued after 2.0.8/2.1b1 due to // compatibility problems with proxies and security software if ($this->method == 'GET' && $this->fileMode == 'info') { $info = Zotero_Storage::getLocalFileItemInfo($item); if (!$info) { $this->e404(); } StatsD::increment("storage.info", 1); /* header("Last-Modified: " . gmdate('r', $info['uploaded'])); header("Content-Type: " . $info['type']); */ header("Content-Length: " . $info['size']); header("ETag: " . $info['hash']); header("X-Zotero-Filename: " . $info['filename']); header("X-Zotero-Modification-Time: " . $info['mtime']); header("X-Zotero-Compressed: " . ($info['zip'] ? 'Yes' : 'No')); header_remove("X-Powered-By"); $this->end(); } else { if ($this->method == 'GET') { $info = Zotero_Storage::getLocalFileItemInfo($item); if (!$info) { $this->e404(); } // File viewing if ($this->fileView) { $url = Zotero_Attachments::getTemporaryURL($item, !empty($_GET['int'])); if (!$url) { $this->e500(); } StatsD::increment("storage.view", 1); $this->redirect($url); exit; } // File download $url = Zotero_Storage::getDownloadURL($item, 60); if (!$url) { $this->e404(); } // Provide some headers to let 5.0 client skip download header("Zotero-File-Modification-Time: {$info['mtime']}"); header("Zotero-File-MD5: {$info['hash']}"); header("Zotero-File-Size: {$info['size']}"); header("Zotero-File-Compressed: " . ($info['zip'] ? 'Yes' : 'No')); StatsD::increment("storage.download", 1); Zotero_Storage::logDownload($item, $this->userID, IPAddress::getIP()); $this->redirect($url); exit; } else { if ($this->method == 'POST' || $this->method == 'PATCH') { if (!$item->isImportedAttachment()) { $this->e400("Cannot upload file for linked file/URL attachment item"); } $libraryID = $item->libraryID; $type = Zotero_Libraries::getType($libraryID); if ($type == 'group') { $groupID = Zotero_Groups::getGroupIDFromLibraryID($libraryID); $group = Zotero_Groups::get($groupID); if (!$group->userCanEditFiles($this->userID)) { $this->e403("You do not have file editing access"); } } else { $group = null; } // If not the client, require If-Match or If-None-Match if (!$this->httpAuth) { if (empty($_SERVER['HTTP_IF_MATCH']) && empty($_SERVER['HTTP_IF_NONE_MATCH'])) { $this->e428("If-Match/If-None-Match header not provided"); } if (!empty($_SERVER['HTTP_IF_MATCH'])) { if (!preg_match('/^"?([a-f0-9]{32})"?$/', $_SERVER['HTTP_IF_MATCH'], $matches)) { $this->e400("Invalid ETag in If-Match header"); } if (!$item->attachmentStorageHash) { $this->e412("ETag set but file does not exist"); } if ($item->attachmentStorageHash != $matches[1]) { $this->libraryVersion = $item->version; $this->libraryVersionOnFailure = true; $this->e412("ETag does not match current version of file"); } } else { if ($_SERVER['HTTP_IF_NONE_MATCH'] != "*") { $this->e400("Invalid value for If-None-Match header"); } if (Zotero_Storage::getLocalFileItemInfo($item)) { $this->libraryVersion = $item->version; $this->libraryVersionOnFailure = true; $this->e412("If-None-Match: * set but file exists"); } } } // // Upload authorization // if (!isset($_POST['update']) && !isset($_REQUEST['upload'])) { $info = new Zotero_StorageFileInfo(); // Validate upload metadata if (empty($_REQUEST['md5'])) { $this->e400('MD5 hash not provided'); } if (!preg_match('/[abcdefg0-9]{32}/', $_REQUEST['md5'])) { $this->e400('Invalid MD5 hash'); } if (!isset($_REQUEST['filename']) || $_REQUEST['filename'] === "") { $this->e400('Filename not provided'); } // Multi-file upload // // For ZIP files, the filename and hash of the ZIP file are different from those // of the main file. We use the former for S3, and we store the latter in the // upload log to set the attachment metadata with them on file registration. if (!empty($_REQUEST['zipMD5'])) { if (!preg_match('/[abcdefg0-9]{32}/', $_REQUEST['zipMD5'])) { $this->e400('Invalid ZIP MD5 hash'); } if (empty($_REQUEST['zipFilename'])) { $this->e400('ZIP filename not provided'); } $info->zip = true; $info->hash = $_REQUEST['zipMD5']; $info->filename = $_REQUEST['zipFilename']; $info->itemFilename = $_REQUEST['filename']; $info->itemHash = $_REQUEST['md5']; } else { if (!empty($_REQUEST['zipFilename'])) { $this->e400('ZIP MD5 hash not provided'); } else { $info->zip = !empty($_REQUEST['zip']); $info->filename = $_REQUEST['filename']; $info->hash = $_REQUEST['md5']; } } if (empty($_REQUEST['mtime'])) { $this->e400('File modification time not provided'); } $info->mtime = $_REQUEST['mtime']; if (!isset($_REQUEST['filesize'])) { $this->e400('File size not provided'); } $info->size = $_REQUEST['filesize']; if (!is_numeric($info->size)) { $this->e400("Invalid file size"); } $info->contentType = isset($_REQUEST['contentType']) ? $_REQUEST['contentType'] : null; if (!preg_match("/^[a-zA-Z0-9\\-\\/]+\$/", $info->contentType)) { $info->contentType = null; } $info->charset = isset($_REQUEST['charset']) ? $_REQUEST['charset'] : null; if (!preg_match("/^[a-zA-Z0-9\\-]+\$/", $info->charset)) { $info->charset = null; } $contentTypeHeader = $info->contentType . ($info->contentType && $info->charset ? "; charset=" . $info->charset : ""); // Reject file if it would put account over quota if ($group) { $quota = Zotero_Storage::getEffectiveUserQuota($group->ownerUserID); $usage = Zotero_Storage::getUserUsage($group->ownerUserID); } else { $quota = Zotero_Storage::getEffectiveUserQuota($this->objectUserID); $usage = Zotero_Storage::getUserUsage($this->objectUserID); } $total = $usage['total']; $fileSizeMB = round($info->size / 1024 / 1024, 1); if ($total + $fileSizeMB > $quota) { StatsD::increment("storage.upload.quota", 1); $this->e413("File would exceed quota ({$total} + {$fileSizeMB} > {$quota})"); } Zotero_DB::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); Zotero_DB::beginTransaction(); // See if file exists with this filename $localInfo = Zotero_Storage::getLocalFileInfo($info); if ($localInfo) { $storageFileID = $localInfo['storageFileID']; // Verify file size if ($localInfo['size'] != $info->size) { throw new Exception("Specified file size incorrect for existing file " . $info->hash . "/" . $info->filename . " ({$localInfo['size']} != {$info->size})"); } } else { $oldStorageFileID = Zotero_Storage::getFileByHash($info->hash, $info->zip); if ($oldStorageFileID) { // Verify file size $localInfo = Zotero_Storage::getFileInfoByID($oldStorageFileID); if ($localInfo['size'] != $info->size) { throw new Exception("Specified file size incorrect for duplicated file " . $info->hash . "/" . $info->filename . " ({$localInfo['size']} != {$info->size})"); } // Create new file on S3 with new name $storageFileID = Zotero_Storage::duplicateFile($oldStorageFileID, $info->filename, $info->zip, $contentTypeHeader); } } // If we already have a file, add/update storageFileItems row and stop if (!empty($storageFileID)) { Zotero_Storage::updateFileItemInfo($item, $storageFileID, $info, $this->httpAuth); Zotero_DB::commit(); StatsD::increment("storage.upload.existing", 1); if ($this->httpAuth) { $this->queryParams['format'] = null; header('Content-Type: application/xml'); echo "<exists/>"; } else { $this->queryParams['format'] = null; header('Content-Type: application/json'); $this->libraryVersion = $item->version; echo json_encode(array('exists' => 1)); } $this->end(); } Zotero_DB::commit(); // Add request to upload queue $uploadKey = Zotero_Storage::queueUpload($this->userID, $info); // User over queue limit if (!$uploadKey) { header('Retry-After: ' . Zotero_Storage::$uploadQueueTimeout); if ($this->httpAuth) { $this->e413("Too many queued uploads"); } else { $this->e429("Too many queued uploads"); } } StatsD::increment("storage.upload.new", 1); // Output XML for client requests (which use HTTP Auth) if ($this->httpAuth) { $params = Zotero_Storage::generateUploadPOSTParams($item, $info, true); $this->queryParams['format'] = null; header('Content-Type: application/xml'); $xml = new SimpleXMLElement('<upload/>'); $xml->url = Zotero_Storage::getUploadBaseURL(); $xml->key = $uploadKey; foreach ($params as $key => $val) { $xml->params->{$key} = $val; } echo $xml->asXML(); } else { if (!empty($_REQUEST['params']) && $_REQUEST['params'] == "1") { $params = array("url" => Zotero_Storage::getUploadBaseURL(), "params" => array()); foreach (Zotero_Storage::generateUploadPOSTParams($item, $info) as $key => $val) { $params['params'][$key] = $val; } } else { $params = Zotero_Storage::getUploadPOSTData($item, $info); } $params['uploadKey'] = $uploadKey; $this->queryParams['format'] = null; header('Content-Type: application/json'); echo json_encode($params); } exit; } // // API partial upload and post-upload file registration // if (isset($_REQUEST['upload'])) { $uploadKey = $_REQUEST['upload']; if (!$uploadKey) { $this->e400("Upload key not provided"); } $info = Zotero_Storage::getUploadInfo($uploadKey); if (!$info) { $this->e400("Upload key not found"); } // Partial upload if ($this->method == 'PATCH') { if (empty($_REQUEST['algorithm'])) { throw new Exception("Algorithm not specified", Z_ERROR_INVALID_INPUT); } $storageFileID = Zotero_Storage::patchFile($item, $info, $_REQUEST['algorithm'], $this->body); } else { $remoteInfo = Zotero_Storage::getRemoteFileInfo($info); if (!$remoteInfo) { error_log("Remote file {$info->hash}/{$info->filename} not found"); $this->e400("Remote file not found"); } if ($remoteInfo->size != $info->size) { error_log("Uploaded file size does not match " . "({$remoteInfo->size} != {$info->size}) " . "for file {$info->hash}/{$info->filename}"); } } // Set an automatic shared lock in getLocalFileInfo() to prevent // two simultaneous transactions from adding a file Zotero_DB::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); Zotero_DB::beginTransaction(); if (!isset($storageFileID)) { // Check if file already exists, which can happen if two identical // files are uploaded simultaneously $fileInfo = Zotero_Storage::getLocalFileInfo($info); if ($fileInfo) { $storageFileID = $fileInfo['storageFileID']; } else { $storageFileID = Zotero_Storage::addFile($info); } } Zotero_Storage::updateFileItemInfo($item, $storageFileID, $info); Zotero_Storage::logUpload($this->userID, $item, $uploadKey, IPAddress::getIP()); Zotero_DB::commit(); header("HTTP/1.1 204 No Content"); header("Last-Modified-Version: " . $item->version); exit; } // // Client post-upload file registration // if (isset($_POST['update'])) { $this->allowMethods(array('POST')); if (empty($_POST['mtime'])) { throw new Exception('File modification time not provided'); } $uploadKey = $_POST['update']; $info = Zotero_Storage::getUploadInfo($uploadKey); if (!$info) { $this->e400("Upload key not found"); } $remoteInfo = Zotero_Storage::getRemoteFileInfo($info); if (!$remoteInfo) { $this->e400("Remote file not found"); } if (!isset($info->size)) { throw new Exception("Size information not available"); } $info->mtime = $_POST['mtime']; // Set an automatic shared lock in getLocalFileInfo() to prevent // two simultaneous transactions from adding a file Zotero_DB::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); Zotero_DB::beginTransaction(); // Check if file already exists, which can happen if two identical // files are uploaded simultaneously $fileInfo = Zotero_Storage::getLocalFileInfo($info); if ($fileInfo) { $storageFileID = $fileInfo['storageFileID']; } else { $storageFileID = Zotero_Storage::addFile($info); } Zotero_Storage::updateFileItemInfo($item, $storageFileID, $info, true); Zotero_Storage::logUpload($this->userID, $item, $uploadKey, IPAddress::getIP()); Zotero_DB::commit(); header("HTTP/1.1 204 No Content"); exit; } throw new Exception("Invalid request", Z_ERROR_INVALID_INPUT); } } } exit; }