protected function doGetFileStatMulti(array $params) { $stats = array(); $auth = $this->getAuthentication(); $reqs = array(); foreach ($params['srcs'] as $path) { list($srcCont, $srcRel) = $this->resolveStoragePathReal($path); if ($srcRel === null) { $stats[$path] = false; continue; // invalid storage path } elseif (!$auth) { $stats[$path] = null; continue; } // (a) Check the container $cstat = $this->getContainerStat($srcCont, true); if ($cstat === false) { $stats[$path] = false; continue; // ok, nothing to do } elseif (!is_array($cstat)) { $stats[$path] = null; continue; } $reqs[$path] = array('method' => 'HEAD', 'url' => $this->storageUrl($auth, $srcCont, $srcRel), 'headers' => $this->authTokenHeaders($auth) + $this->headersFromParams($params)); } $opts = array('maxConnsPerHost' => $params['concurrency']); $reqs = $this->http->runMulti($reqs, $opts); foreach ($params['srcs'] as $path) { if (array_key_exists($path, $stats)) { continue; // some sort of failure above } // (b) Check the file list($rcode, $rdesc, $rhdrs, $rbody, $rerr) = $reqs[$path]['response']; if ($rcode === 200 || $rcode === 204) { // Update the object if it is missing some headers $rhdrs = $this->addMissingMetadata($rhdrs, $path); // Fetch all of the custom metadata headers $metadata = array(); foreach ($rhdrs as $name => $value) { if (strpos($name, 'x-object-meta-') === 0) { $metadata[substr($name, strlen('x-object-meta-'))] = $value; } } // Fetch all of the custom raw HTTP headers $headers = $this->sanitizeHdrs(array('headers' => $rhdrs)); $stat = array('mtime' => $this->convertSwiftDate($rhdrs['last-modified'], TS_MW), 'size' => isset($rhdrs['content-length']) ? (int) $rhdrs['content-length'] : 0, 'sha1' => $rhdrs['x-object-meta-sha1base36'], 'md5' => ctype_xdigit($rhdrs['etag']) ? $rhdrs['etag'] : null, 'xattr' => array('metadata' => $metadata, 'headers' => $headers)); } elseif ($rcode === 404) { $stat = false; } else { $stat = null; $this->onError(null, __METHOD__, $params, $rerr, $rcode, $rdesc); } $stats[$path] = $stat; } return $stats; }
/** * Runs all the queries. */ public function run() { $http = new MultiHttpClient(array('reqTimeout' => $this->timeout, 'connTimeout' => 3)); $responses = $http->runMulti($this->getMultiHttpQueries($this->queries)); foreach ($responses as $index => $response) { $this->responses[$index] = $response; } $this->hasRun = true; }
protected function doGetFileStatMulti(array $params) { $stats = array(); $auth = $this->getAuthentication(); $reqs = array(); foreach ($params['srcs'] as $path) { list($srcCont, $srcRel) = $this->resolveStoragePathReal($path); if ($srcRel === null) { $stats[$path] = false; continue; // invalid storage path } elseif (!$auth) { $stats[$path] = null; continue; } // (a) Check the container $cstat = $this->getContainerStat($srcCont); if ($cstat === false) { $stats[$path] = false; continue; // ok, nothing to do } elseif (!is_array($cstat)) { $stats[$path] = null; continue; } $reqs[$path] = array('method' => 'HEAD', 'url' => $this->storageUrl($auth, $srcCont, $srcRel), 'headers' => $this->authTokenHeaders($auth) + $this->headersFromParams($params)); } $opts = array('maxConnsPerHost' => $params['concurrency']); $reqs = $this->http->runMulti($reqs, $opts); foreach ($params['srcs'] as $path) { if (array_key_exists($path, $stats)) { continue; // some sort of failure above } // (b) Check the file list($rcode, $rdesc, $rhdrs, $rbody, $rerr) = $reqs[$path]['response']; if ($rcode === 200 || $rcode === 204) { // Update the object if it is missing some headers $rhdrs = $this->addMissingMetadata($rhdrs, $path); // Load the stat array from the headers $stat = $this->getStatFromHeaders($rhdrs); if ($this->isRGW) { $stat['latest'] = true; // strong consistency } } elseif ($rcode === 404) { $stat = false; } else { $stat = null; $this->onError(null, __METHOD__, $params, $rerr, $rcode, $rdesc); } $stats[$path] = $stat; } return $stats; }
/** * Execute a set of virtual HTTP(S) requests concurrently * * A map of requests keys to response maps is returned. Each response map has: * - code : HTTP response code or 0 if there was a serious cURL error * - reason : HTTP response reason (empty if there was a serious cURL error) * - headers : <header name/value associative array> * - body : HTTP response body or resource (if "stream" was set) * - error : Any cURL error string * The map also stores integer-indexed copies of these values. This lets callers do: * @code * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $responses[0]; * @endcode * * @param array $reqs Map of Virtual HTTP request maps * @return array $reqs Map of corresponding response values with the same keys/order * @throws Exception */ public function runMulti(array $reqs) { foreach ($reqs as $index => &$req) { if (isset($req[0])) { $req['method'] = $req[0]; // short-form unset($req[0]); } if (isset($req[1])) { $req['url'] = $req[1]; // short-form unset($req[1]); } $req['chain'] = []; // chain or list of replaced requests } unset($req); // don't assign over this by accident $curUniqueId = 0; $armoredIndexMap = []; // (original index => new index) $doneReqs = []; // (index => request) $executeReqs = []; // (index => request) $replaceReqsByService = []; // (prefix => index => request) $origPending = []; // (index => 1) for original requests foreach ($reqs as $origIndex => $req) { // Re-index keys to consecutive integers (they will be swapped back later) $index = $curUniqueId++; $armoredIndexMap[$origIndex] = $index; $origPending[$index] = 1; if (preg_match('#^(http|ftp)s?://#', $req['url'])) { // Absolute FTP/HTTP(S) URL, run it as normal $executeReqs[$index] = $req; } else { // Must be a virtual HTTP URL; resolve it list($prefix, $service) = $this->getMountAndService($req['url']); if (!$service) { throw new UnexpectedValueException("Path '{$req['url']}' has no service."); } // Set the URL to the mount-relative portion $req['url'] = substr($req['url'], strlen($prefix)); $replaceReqsByService[$prefix][$index] = $req; } } // Function to get IDs that won't collide with keys in $armoredIndexMap $idFunc = function () use(&$curUniqueId) { return $curUniqueId++; }; $rounds = 0; do { if (++$rounds > 5) { // sanity throw new Exception("Too many replacement rounds detected. Aborting."); } // Track requests executed this round that have a prefix/service. // Note that this also includes requests where 'response' was forced. $checkReqIndexesByPrefix = []; // Resolve the virtual URLs valid and qualified HTTP(S) URLs // and add any required authentication headers for the backend. // Services can also replace requests with new ones, either to // defer the original or to set a proxy response to the original. $newReplaceReqsByService = []; foreach ($replaceReqsByService as $prefix => $servReqs) { $service = $this->instances[$prefix]; foreach ($service->onRequests($servReqs, $idFunc) as $index => $req) { // Services use unique IDs for replacement requests if (isset($servReqs[$index]) || isset($origPending[$index])) { // A current or original request which was not modified } else { // Replacement request that will convert to original requests $newReplaceReqsByService[$prefix][$index] = $req; } if (isset($req['response'])) { // Replacement requests with pre-set responses should not execute unset($executeReqs[$index]); unset($origPending[$index]); $doneReqs[$index] = $req; } else { // Original or mangled request included $executeReqs[$index] = $req; } $checkReqIndexesByPrefix[$prefix][$index] = 1; } } // Update index of requests to inspect for replacement $replaceReqsByService = $newReplaceReqsByService; // Run the actual work HTTP requests foreach ($this->http->runMulti($executeReqs) as $index => $ranReq) { $doneReqs[$index] = $ranReq; unset($origPending[$index]); } $executeReqs = []; // Services can also replace requests with new ones, either to // defer the original or to set a proxy response to the original. // Any replacement requests executed above will need to be replaced // with new requests (eventually the original). The responses can be // forced by setting 'response' rather than actually be sent over the wire. $newReplaceReqsByService = []; foreach ($checkReqIndexesByPrefix as $prefix => $servReqIndexes) { $service = $this->instances[$prefix]; // $doneReqs actually has the requests (with 'response' set) $servReqs = array_intersect_key($doneReqs, $servReqIndexes); foreach ($service->onResponses($servReqs, $idFunc) as $index => $req) { // Services use unique IDs for replacement requests if (isset($servReqs[$index]) || isset($origPending[$index])) { // A current or original request which was not modified } else { // Replacement requests with pre-set responses should not execute $newReplaceReqsByService[$prefix][$index] = $req; } if (isset($req['response'])) { // Replacement requests with pre-set responses should not execute unset($origPending[$index]); $doneReqs[$index] = $req; } else { // Update the request in case it was mangled $executeReqs[$index] = $req; } } } // Update index of requests to inspect for replacement $replaceReqsByService = $newReplaceReqsByService; } while (count($origPending)); $responses = []; // Update $reqs to include 'response' and normalized request 'headers'. // This maintains the original order of $reqs. foreach ($reqs as $origIndex => $req) { $index = $armoredIndexMap[$origIndex]; if (!isset($doneReqs[$index])) { throw new UnexpectedValueException("Response for request '{$index}' is NULL."); } $responses[$origIndex] = $doneReqs[$index]['response']; } return $responses; }