/** * Handle S3 request * * Permission-checking provided by items() */ private function _handleFileRequest($item) { if (!$this->permissions->canAccess($this->objectLibraryID, 'files')) { $this->e403(); } $this->allowMethods(array('HEAD', 'GET', 'POST', 'PATCH')); if (!$item->isAttachment()) { $this->e400("Item is not an attachment"); } // File info for client sync // // Use of HEAD method is deprecated after 2.0.8/2.1b1 due to // compatibility problems with proxies and security software if ($this->method == 'HEAD' || $this->method == 'GET' && $this->fileMode == 'info') { $info = Zotero_S3::getLocalFileItemInfo($item); if (!$info) { $this->e404(); } /* header("Last-Modified: " . gmdate('r', $info['uploaded'])); header("Content-Type: " . $info['type']); */ header("Content-Length: " . $info['size']); header("ETag: " . $info['hash']); header("X-Zotero-Filename: " . $info['filename']); header("X-Zotero-Modification-Time: " . $info['mtime']); header("X-Zotero-Compressed: " . ($info['zip'] ? 'Yes' : 'No')); header_remove("X-Powered-By"); } else { if ($this->method == 'GET' || $this->method == 'POST' && $this->fileView) { if ($this->fileView) { $info = Zotero_S3::getLocalFileItemInfo($item); if (!$info) { $this->e404(); } // For zip files, redirect to files domain if ($info['zip']) { $url = Zotero_Attachments::getTemporaryURL($item, !empty($_GET['int'])); if (!$url) { $this->e500(); } header("Location: {$url}"); exit; } } // For single files, redirect to S3 $url = Zotero_S3::getDownloadURL($item, 60); if (!$url) { $this->e404(); } Zotero_S3::logDownload($item, $this->userID, IPAddress::getIP()); header("Location: {$url}"); exit; } else { if ($this->method == 'POST' || $this->method == 'PATCH') { if (!$item->isImportedAttachment()) { $this->e400("Cannot upload file for linked file/URL attachment item"); } $libraryID = $item->libraryID; $type = Zotero_Libraries::getType($libraryID); if ($type == 'group') { $groupID = Zotero_Groups::getGroupIDFromLibraryID($libraryID); $group = Zotero_Groups::get($groupID); if (!$group->userCanEditFiles($this->userID)) { $this->e403("You do not have file editing access"); } } else { $group = null; } // If not the client, require If-Match or If-None-Match if (!$this->httpAuth) { if (empty($_SERVER['HTTP_IF_MATCH']) && empty($_SERVER['HTTP_IF_NONE_MATCH'])) { $this->e428("If-Match/If-None-Match header not provided"); } if (!empty($_SERVER['HTTP_IF_MATCH'])) { if (!preg_match('/^"?([a-f0-9]{32})"?$/', $_SERVER['HTTP_IF_MATCH'], $matches)) { $this->e400("Invalid ETag in If-Match header"); } if (!$item->attachmentStorageHash) { $info = Zotero_S3::getLocalFileItemInfo($item); $this->e412("ETag set but file does not exist"); } if ($item->attachmentStorageHash != $matches[1]) { $this->e412("ETag does not match current version of file"); } } else { if ($_SERVER['HTTP_IF_NONE_MATCH'] != "*") { $this->e400("Invalid value for If-None-Match header"); } if ($this->attachmentStorageHash) { $this->e412("If-None-Match: * set but file exists"); } } } // // Upload authorization // if (!isset($_POST['update']) && !isset($_REQUEST['upload'])) { $info = new Zotero_StorageFileInfo(); // Validate upload metadata if (empty($_REQUEST['md5'])) { $this->e400('MD5 hash not provided'); } $info->hash = $_REQUEST['md5']; if (!preg_match('/[abcdefg0-9]{32}/', $info->hash)) { $this->e400('Invalid MD5 hash'); } if (empty($_REQUEST['mtime'])) { $this->e400('File modification time not provided'); } $info->mtime = $_REQUEST['mtime']; if (!isset($_REQUEST['filename']) || $_REQUEST['filename'] === "") { $this->e400('File name not provided'); } $info->filename = $_REQUEST['filename']; if (!isset($_REQUEST['filesize'])) { $this->e400('File size not provided'); } $info->size = $_REQUEST['filesize']; if (!is_numeric($info->size)) { $this->e400("Invalid file size"); } $info->contentType = isset($_REQUEST['contentType']) ? $_REQUEST['contentType'] : ""; if (!preg_match("/^[a-zA-Z0-9\\-\\/]+\$/", $info->contentType)) { $info->contentType = ""; } $info->charset = isset($_REQUEST['charset']) ? $_REQUEST['charset'] : ""; if (!preg_match("/^[a-zA-Z0-9\\-]+\$/", $info->charset)) { $info->charset = ""; } $contentTypeHeader = $info->contentType . ($info->contentType && $info->charset ? "; charset=" . $info->charset : ""); $info->zip = !empty($_REQUEST['zip']); // Reject file if it would put account over quota if ($group) { $quota = Zotero_S3::getEffectiveUserQuota($group->ownerUserID); $usage = Zotero_S3::getUserUsage($group->ownerUserID); } else { $quota = Zotero_S3::getEffectiveUserQuota($this->objectUserID); $usage = Zotero_S3::getUserUsage($this->objectUserID); } $total = $usage['total']; $fileSizeMB = round($info->size / 1024 / 1024, 1); if ($total + $fileSizeMB > $quota) { $this->e413("File would exceed quota ({$total} + {$fileSizeMB} > {$quota})"); } Zotero_DB::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); Zotero_DB::beginTransaction(); // See if file exists with this filename $localInfo = Zotero_S3::getLocalFileInfo($info); if ($localInfo) { $storageFileID = $localInfo['storageFileID']; // Verify file size if ($localInfo['size'] != $info->size) { throw new Exception("Specified file size incorrect for existing file " . $info->hash . "/" . $info->filename . " ({$localInfo['size']} != {$info->size})"); } } else { $oldStorageFileID = Zotero_S3::getFileByHash($info->hash, $info->zip); if ($oldStorageFileID) { // Verify file size $localInfo = Zotero_S3::getFileInfoByID($oldStorageFileID); if ($localInfo['size'] != $info->size) { throw new Exception("Specified file size incorrect for duplicated file " . $info->hash . "/" . $info->filename . " ({$localInfo['size']} != {$info->size})"); } // Create new file on S3 with new name $storageFileID = Zotero_S3::duplicateFile($oldStorageFileID, $info->filename, $info->zip, $contentTypeHeader); if (!$storageFileID) { $this->e500("File duplication failed"); } } } // If we already have a file, add/update storageFileItems row and stop if (!empty($storageFileID)) { Zotero_S3::updateFileItemInfo($item, $storageFileID, $info); Zotero_DB::commit(); if ($this->httpAuth) { header('Content-Type: application/xml'); echo "<exists/>"; } else { header('Content-Type: application/json'); echo json_encode(array('exists' => 1)); } exit; } Zotero_DB::commit(); // Add request to upload queue $uploadKey = Zotero_S3::queueUpload($this->userID, $info); // User over queue limit if (!$uploadKey) { header('Retry-After: ' . Zotero_S3::$uploadQueueTimeout); if ($this->httpAuth) { $this->e413("Too many queued uploads"); } else { $this->e429("Too many queued uploads"); } } // Output XML for client requests (which use HTTP Auth) if ($this->httpAuth) { $params = Zotero_S3::generateUploadPOSTParams($item, $info, true); header('Content-Type: application/xml'); $xml = new SimpleXMLElement('<upload/>'); $xml->url = Zotero_S3::getUploadBaseURL(); $xml->key = $uploadKey; foreach ($params as $key => $val) { $xml->params->{$key} = $val; } echo $xml->asXML(); } else { if (!empty($_REQUEST['params']) && $_REQUEST['params'] == "1") { $params = array("url" => Zotero_S3::getUploadBaseURL(), "params" => array()); foreach (Zotero_S3::generateUploadPOSTParams($item, $info) as $key => $val) { $params['params'][$key] = $val; } } else { $params = Zotero_S3::getUploadPOSTData($item, $info); } $params['uploadKey'] = $uploadKey; header('Content-Type: application/json'); echo json_encode($params); } exit; } // // API partial upload and post-upload file registration // if (isset($_REQUEST['upload'])) { $uploadKey = $_REQUEST['upload']; if (!$uploadKey) { $this->e400("Upload key not provided"); } $info = Zotero_S3::getUploadInfo($uploadKey); if (!$info) { $this->e400("Upload key not found"); } // Partial upload if ($this->method == 'PATCH') { if (empty($_REQUEST['algorithm'])) { throw new Exception("Algorithm not specified", Z_ERROR_INVALID_INPUT); } $storageFileID = Zotero_S3::patchFile($item, $info, $_REQUEST['algorithm'], $this->body); } else { $remoteInfo = Zotero_S3::getRemoteFileInfo($info); if (!$remoteInfo) { error_log("Remote file {$info->hash}/{$info->filename} not found"); $this->e400("Remote file not found"); } if ($remoteInfo['size'] != $info->size) { error_log("Uploaded file size does not match ({$remoteInfo['size']} != {$info->size}) for file {$info->hash}/{$info->filename}"); } } // Set an automatic shared lock in getLocalFileInfo() to prevent // two simultaneous transactions from adding a file Zotero_DB::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); Zotero_DB::beginTransaction(); if (!isset($storageFileID)) { // Check if file already exists, which can happen if two identical // files are uploaded simultaneously $fileInfo = Zotero_S3::getLocalFileInfo($info); if ($fileInfo) { $storageFileID = $fileInfo['storageFileID']; } else { $storageFileID = Zotero_S3::addFile($info); } } Zotero_S3::updateFileItemInfo($item, $storageFileID, $info); Zotero_S3::logUpload($this->userID, $item, $uploadKey, IPAddress::getIP()); Zotero_DB::commit(); header("HTTP/1.1 204 No Content"); exit; } // // Client post-upload file registration // if (isset($_POST['update'])) { $this->allowMethods(array('POST')); if (empty($_POST['mtime'])) { throw new Exception('File modification time not provided'); } $uploadKey = $_POST['update']; $info = Zotero_S3::getUploadInfo($uploadKey); if (!$info) { $this->e400("Upload key not found"); } $remoteInfo = Zotero_S3::getRemoteFileInfo($info); if (!$remoteInfo) { $this->e400("Remote file not found"); } if (!isset($info->size)) { throw new Exception("Size information not available"); } $info->mtime = $_POST['mtime']; // Set an automatic shared lock in getLocalFileInfo() to prevent // two simultaneous transactions from adding a file Zotero_DB::query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); Zotero_DB::beginTransaction(); // Check if file already exists, which can happen if two identical // files are uploaded simultaneously $fileInfo = Zotero_S3::getLocalFileInfo($info); if ($fileInfo) { $storageFileID = $fileInfo['storageFileID']; } else { $storageFileID = Zotero_S3::addFile($info); } Zotero_S3::updateFileItemInfo($item, $storageFileID, $info); Zotero_S3::logUpload($this->userID, $item, $uploadKey, IPAddress::getIP()); Zotero_DB::commit(); header("HTTP/1.1 204 No Content"); exit; } } } } exit; }
public static function patchFile($item, $info, $algorithm, $patch) { switch ($algorithm) { case 'bsdiff': case 'xdelta': case 'vcdiff': break; case 'xdiff': if (!function_exists('xdiff_file_patch_binary')) { throw new Exception("=xdiff not available"); } break; default: throw new Exception("Invalid algorithm '{$algorithm}'", Z_ERROR_INVALID_INPUT); } $originalInfo = Zotero_S3::getLocalFileItemInfo($item); $basePath = "/tmp/zfsupload/"; $path = $basePath . $info->hash . "_" . uniqid() . "/"; mkdir($path, 0777, true); $cleanup = function () use($basePath, $path) { unlink("original"); unlink("patch"); unlink("new"); chdir($basePath); rmdir($path); }; $e = null; try { // Download file from S3 to temp directory if (!Zotero_S3::downloadFile($originalInfo, $path, "original")) { throw new Exception("Error downloading original file"); } chdir($path); // Save body to temp file file_put_contents("patch", $patch); // Patch file switch ($algorithm) { case 'bsdiff': exec('bspatch original new patch 2>&1', $output, $ret); if ($ret) { throw new Exception("Error applying patch ({$ret}): " . implode("\n", $output)); } if (!file_exists("new")) { throw new Exception("Error applying patch ({$ret})"); } break; case 'xdelta': case 'vcdiff': exec('xdelta3 -d -s original patch new 2>&1', $output, $ret); if ($ret) { if ($ret == 2) { Z_HTTP::e400("Invalid delta"); } throw new Exception("Error applying patch ({$ret}): " . implode("\n", $output)); } if (!file_exists("new")) { throw new Exception("Error applying patch ({$ret})"); } break; case 'xdiff': $ret = xdiff_file_patch_binary("original", "patch", "new"); if (!$ret) { throw new Exception("Error applying patch"); } break; } // Check MD5 hash if (md5_file("new") != $info->hash) { $cleanup(); Z_HTTP::e409("Patched file does not match hash"); } // Check file size if (filesize("new") != $info->size) { $cleanup(); Z_HTTP::e409("Patched file size does not match (" . filesize("new") . " != {$info->size})"); } // If ZIP, make sure it's a ZIP if ($info->zip && file_get_contents("new", false, null, 0, 4) != "PK" . chr(03) . chr(04)) { $cleanup(); Z_HTTP::e409("Patched file is not a ZIP file"); } // Upload to S3 $t = $info->contentType . ($info->contentType && $info->charset ? "; charset={$info->charset}" : ""); $storageFileID = Zotero_S3::uploadFile($info, "new", $t); } catch (Exception $e) { //$cleanup(); throw $e; } return $storageFileID; }
/** * Download ZIP file from S3, extract it, and return a temporary URL * pointing to the main file */ public static function getTemporaryURL(Zotero_Item $item, $localOnly = false) { $extURLPrefix = Z_CONFIG::$ATTACHMENT_SERVER_URL; if ($extURLPrefix[strlen($extURLPrefix) - 1] != "/") { $extURLPrefix .= "/"; } $info = Zotero_S3::getLocalFileItemInfo($item); $storageFileID = $info['storageFileID']; $filename = $info['filename']; $mtime = $info['mtime']; if (!$info['zip']) { throw new Exception("Not a zip attachment"); } $realFilename = preg_replace("/^storage:/", "", $item->attachmentPath); $realFilename = self::decodeRelativeDescriptorString($realFilename); $realFilename = urlencode($realFilename); $docroot = Z_CONFIG::$ATTACHMENT_SERVER_DOCROOT; //Z_Core::$debug = true; // Check memcached to see if file is already extracted $key = "attachmentServerString_" . $storageFileID . "_" . $mtime; if ($randomStr = Z_Core::$MC->get($key)) { Z_Core::debug("Got attachment path '{$randomStr}/{$realFilename}' from memcached"); return $extURLPrefix . "{$randomStr}/{$realFilename}"; } $localAddr = gethostbyname(gethostname()); // See if this is an attachment host $index = false; $skipHost = false; for ($i = 0, $len = sizeOf(Z_CONFIG::$ATTACHMENT_SERVER_HOSTS); $i < $len; $i++) { $hostAddr = gethostbyname(Z_CONFIG::$ATTACHMENT_SERVER_HOSTS[$i]); if ($hostAddr == $localAddr) { // Make a HEAD request on the local static port to make sure // this host is actually functional $url = "http://" . Z_CONFIG::$ATTACHMENT_SERVER_HOSTS[$i] . ":" . Z_CONFIG::$ATTACHMENT_SERVER_STATIC_PORT . "/"; Z_Core::debug("Making HEAD request to {$url}"); $ch = curl_init($url); curl_setopt($ch, CURLOPT_NOBODY, 1); curl_setopt($ch, CURLOPT_HTTPHEADER, array("Expect:")); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 1); curl_setopt($ch, CURLOPT_TIMEOUT, 2); curl_setopt($ch, CURLOPT_HEADER, 0); // do not return HTTP headers curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); $response = curl_exec($ch); $code = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($code != 200) { $skipHost = Z_CONFIG::$ATTACHMENT_SERVER_HOSTS[$i]; if ($code == 0) { Z_Core::logError("Error connecting to local attachments server"); } else { Z_Core::logError("Local attachments server returned {$code}"); } break; } $index = $i + 1; break; } } // If not, make an internal root request to trigger the extraction on // one of them and retrieve the temporary URL if ($index === false) { // Prevent redirect madness if target server doesn't think it's an // attachment server if ($localOnly) { throw new Exception("Internal attachments request hit a non-attachment server"); } $prefix = 'http://' . Z_CONFIG::$API_SUPER_USERNAME . ":" . Z_CONFIG::$API_SUPER_PASSWORD . "@"; $path = Zotero_API::getItemURI($item) . "/file/view?int=1"; $path = preg_replace('/^[^:]+:\\/\\/[^\\/]+/', '', $path); $context = stream_context_create(array('http' => array('follow_location' => 0))); $url = false; $hosts = Z_CONFIG::$ATTACHMENT_SERVER_HOSTS; // Try in random order shuffle($hosts); foreach ($hosts as $host) { // Don't try the local host again if we know it's not working if ($host == $skipHost) { continue; } $intURL = $prefix . $host . ":" . Z_CONFIG::$ATTACHMENT_SERVER_DYNAMIC_PORT . $path; Z_Core::debug("Making GET request to {$host}"); if (file_get_contents($intURL, false, $context) !== false) { foreach ($http_response_header as $header) { if (preg_match('/^Location:\\s*(.+)$/', $header, $matches)) { if (strpos($matches[1], $extURLPrefix) !== 0) { throw new Exception("Redirect location '" . $matches[1] . "'" . " does not begin with {$extURLPrefix}"); } return $matches[1]; } } } } return false; } // If this is an attachment host, do the extraction inline // and generate a random number with an embedded host id. // // The reverse proxy routes incoming file requests to the proper hosts // using the embedded id. // // A cron job deletes old attachment directories $randomStr = rand(1000000, 2147483647); // Seventh number is the host id $randomStr = substr($randomStr, 0, 6) . $index . substr($randomStr, 6); // Download and extract file $dir = $docroot . $randomStr . "/"; $tmpDir = $dir . "ztmp/"; if (!mkdir($tmpDir, 0777, true)) { throw new Exception("Unable to create directory '{$tmpDir}'"); } Z_Core::debug("Downloading attachment to {$dir}"); $response = Zotero_S3::downloadFile($info, $tmpDir); $success = self::extractZip($tmpDir . $info['filename'], $dir); unlink($tmpDir . $info['filename']); rmdir($tmpDir); if (!$success) { return false; } Z_Core::$MC->set($key, $randomStr, self::$cacheTime); return $extURLPrefix . "{$randomStr}/" . $realFilename; }