protected function doGetFileStatMulti(array $params)
{
$stats = array();
$auth = $this->getAuthentication();
$reqs = array();
foreach ($params['srcs'] as $path) {
list($srcCont, $srcRel) = $this->resolveStoragePathReal($path);
if ($srcRel === null) {
$stats[$path] = false;
continue;
// invalid storage path
} elseif (!$auth) {
$stats[$path] = null;
continue;
}
// (a) Check the container
$cstat = $this->getContainerStat($srcCont, true);
if ($cstat === false) {
$stats[$path] = false;
continue;
// ok, nothing to do
} elseif (!is_array($cstat)) {
$stats[$path] = null;
continue;
}
$reqs[$path] = array('method' => 'HEAD', 'url' => $this->storageUrl($auth, $srcCont, $srcRel), 'headers' => $this->authTokenHeaders($auth) + $this->headersFromParams($params));
}
$opts = array('maxConnsPerHost' => $params['concurrency']);
$reqs = $this->http->runMulti($reqs, $opts);
foreach ($params['srcs'] as $path) {
if (array_key_exists($path, $stats)) {
continue;
// some sort of failure above
}
// (b) Check the file
list($rcode, $rdesc, $rhdrs, $rbody, $rerr) = $reqs[$path]['response'];
if ($rcode === 200 || $rcode === 204) {
// Update the object if it is missing some headers
$rhdrs = $this->addMissingMetadata($rhdrs, $path);
// Fetch all of the custom metadata headers
$metadata = array();
foreach ($rhdrs as $name => $value) {
if (strpos($name, 'x-object-meta-') === 0) {
$metadata[substr($name, strlen('x-object-meta-'))] = $value;
}
}
// Fetch all of the custom raw HTTP headers
$headers = $this->sanitizeHdrs(array('headers' => $rhdrs));
$stat = array('mtime' => $this->convertSwiftDate($rhdrs['last-modified'], TS_MW), 'size' => isset($rhdrs['content-length']) ? (int) $rhdrs['content-length'] : 0, 'sha1' => $rhdrs['x-object-meta-sha1base36'], 'md5' => ctype_xdigit($rhdrs['etag']) ? $rhdrs['etag'] : null, 'xattr' => array('metadata' => $metadata, 'headers' => $headers));
} elseif ($rcode === 404) {
$stat = false;
} else {
$stat = null;
$this->onError(null, __METHOD__, $params, $rerr, $rcode, $rdesc);
}
$stats[$path] = $stat;
}
return $stats;
}
/**
* Runs all the queries.
*/
public function run()
{
$http = new MultiHttpClient(array('reqTimeout' => $this->timeout, 'connTimeout' => 3));
$responses = $http->runMulti($this->getMultiHttpQueries($this->queries));
foreach ($responses as $index => $response) {
$this->responses[$index] = $response;
}
$this->hasRun = true;
}
protected function doGetFileStatMulti(array $params)
{
$stats = array();
$auth = $this->getAuthentication();
$reqs = array();
foreach ($params['srcs'] as $path) {
list($srcCont, $srcRel) = $this->resolveStoragePathReal($path);
if ($srcRel === null) {
$stats[$path] = false;
continue;
// invalid storage path
} elseif (!$auth) {
$stats[$path] = null;
continue;
}
// (a) Check the container
$cstat = $this->getContainerStat($srcCont);
if ($cstat === false) {
$stats[$path] = false;
continue;
// ok, nothing to do
} elseif (!is_array($cstat)) {
$stats[$path] = null;
continue;
}
$reqs[$path] = array('method' => 'HEAD', 'url' => $this->storageUrl($auth, $srcCont, $srcRel), 'headers' => $this->authTokenHeaders($auth) + $this->headersFromParams($params));
}
$opts = array('maxConnsPerHost' => $params['concurrency']);
$reqs = $this->http->runMulti($reqs, $opts);
foreach ($params['srcs'] as $path) {
if (array_key_exists($path, $stats)) {
continue;
// some sort of failure above
}
// (b) Check the file
list($rcode, $rdesc, $rhdrs, $rbody, $rerr) = $reqs[$path]['response'];
if ($rcode === 200 || $rcode === 204) {
// Update the object if it is missing some headers
$rhdrs = $this->addMissingMetadata($rhdrs, $path);
// Load the stat array from the headers
$stat = $this->getStatFromHeaders($rhdrs);
if ($this->isRGW) {
$stat['latest'] = true;
// strong consistency
}
} elseif ($rcode === 404) {
$stat = false;
} else {
$stat = null;
$this->onError(null, __METHOD__, $params, $rerr, $rcode, $rdesc);
}
$stats[$path] = $stat;
}
return $stats;
}
/**
* Execute a set of virtual HTTP(S) requests concurrently
*
* A map of requests keys to response maps is returned. Each response map has:
* - code : HTTP response code or 0 if there was a serious cURL error
* - reason : HTTP response reason (empty if there was a serious cURL error)
* - headers : <header name/value associative array>
* - body : HTTP response body or resource (if "stream" was set)
* - error : Any cURL error string
* The map also stores integer-indexed copies of these values. This lets callers do:
* @code
* list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $responses[0];
* @endcode
*
* @param array $reqs Map of Virtual HTTP request maps
* @return array $reqs Map of corresponding response values with the same keys/order
* @throws Exception
*/
public function runMulti(array $reqs)
{
foreach ($reqs as $index => &$req) {
if (isset($req[0])) {
$req['method'] = $req[0];
// short-form
unset($req[0]);
}
if (isset($req[1])) {
$req['url'] = $req[1];
// short-form
unset($req[1]);
}
$req['chain'] = [];
// chain or list of replaced requests
}
unset($req);
// don't assign over this by accident
$curUniqueId = 0;
$armoredIndexMap = [];
// (original index => new index)
$doneReqs = [];
// (index => request)
$executeReqs = [];
// (index => request)
$replaceReqsByService = [];
// (prefix => index => request)
$origPending = [];
// (index => 1) for original requests
foreach ($reqs as $origIndex => $req) {
// Re-index keys to consecutive integers (they will be swapped back later)
$index = $curUniqueId++;
$armoredIndexMap[$origIndex] = $index;
$origPending[$index] = 1;
if (preg_match('#^(http|ftp)s?://#', $req['url'])) {
// Absolute FTP/HTTP(S) URL, run it as normal
$executeReqs[$index] = $req;
} else {
// Must be a virtual HTTP URL; resolve it
list($prefix, $service) = $this->getMountAndService($req['url']);
if (!$service) {
throw new UnexpectedValueException("Path '{$req['url']}' has no service.");
}
// Set the URL to the mount-relative portion
$req['url'] = substr($req['url'], strlen($prefix));
$replaceReqsByService[$prefix][$index] = $req;
}
}
// Function to get IDs that won't collide with keys in $armoredIndexMap
$idFunc = function () use(&$curUniqueId) {
return $curUniqueId++;
};
$rounds = 0;
do {
if (++$rounds > 5) {
// sanity
throw new Exception("Too many replacement rounds detected. Aborting.");
}
// Track requests executed this round that have a prefix/service.
// Note that this also includes requests where 'response' was forced.
$checkReqIndexesByPrefix = [];
// Resolve the virtual URLs valid and qualified HTTP(S) URLs
// and add any required authentication headers for the backend.
// Services can also replace requests with new ones, either to
// defer the original or to set a proxy response to the original.
$newReplaceReqsByService = [];
foreach ($replaceReqsByService as $prefix => $servReqs) {
$service = $this->instances[$prefix];
foreach ($service->onRequests($servReqs, $idFunc) as $index => $req) {
// Services use unique IDs for replacement requests
if (isset($servReqs[$index]) || isset($origPending[$index])) {
// A current or original request which was not modified
} else {
// Replacement request that will convert to original requests
$newReplaceReqsByService[$prefix][$index] = $req;
}
if (isset($req['response'])) {
// Replacement requests with pre-set responses should not execute
unset($executeReqs[$index]);
unset($origPending[$index]);
$doneReqs[$index] = $req;
} else {
// Original or mangled request included
$executeReqs[$index] = $req;
}
$checkReqIndexesByPrefix[$prefix][$index] = 1;
}
}
// Update index of requests to inspect for replacement
$replaceReqsByService = $newReplaceReqsByService;
// Run the actual work HTTP requests
foreach ($this->http->runMulti($executeReqs) as $index => $ranReq) {
$doneReqs[$index] = $ranReq;
unset($origPending[$index]);
}
$executeReqs = [];
// Services can also replace requests with new ones, either to
// defer the original or to set a proxy response to the original.
// Any replacement requests executed above will need to be replaced
// with new requests (eventually the original). The responses can be
// forced by setting 'response' rather than actually be sent over the wire.
$newReplaceReqsByService = [];
foreach ($checkReqIndexesByPrefix as $prefix => $servReqIndexes) {
$service = $this->instances[$prefix];
// $doneReqs actually has the requests (with 'response' set)
$servReqs = array_intersect_key($doneReqs, $servReqIndexes);
foreach ($service->onResponses($servReqs, $idFunc) as $index => $req) {
// Services use unique IDs for replacement requests
if (isset($servReqs[$index]) || isset($origPending[$index])) {
// A current or original request which was not modified
} else {
// Replacement requests with pre-set responses should not execute
$newReplaceReqsByService[$prefix][$index] = $req;
}
if (isset($req['response'])) {
// Replacement requests with pre-set responses should not execute
unset($origPending[$index]);
$doneReqs[$index] = $req;
} else {
// Update the request in case it was mangled
$executeReqs[$index] = $req;
}
}
}
// Update index of requests to inspect for replacement
$replaceReqsByService = $newReplaceReqsByService;
} while (count($origPending));
$responses = [];
// Update $reqs to include 'response' and normalized request 'headers'.
// This maintains the original order of $reqs.
foreach ($reqs as $origIndex => $req) {
$index = $armoredIndexMap[$origIndex];
if (!isset($doneReqs[$index])) {
throw new UnexpectedValueException("Response for request '{$index}' is NULL.");
}
$responses[$origIndex] = $doneReqs[$index]['response'];
}
return $responses;
}
