protected function fetch($origin, $fileUrl, $progress, $options, $exec)
{
do {
$this->retry = false;
$request = new Aspects\HttpGetRequest($origin, $fileUrl, $this->io);
$request->setSpecial(array('github' => $this->config->get('github-domains') ?: array(), 'gitlab' => $this->config->get('gitlab-domains') ?: array()));
$this->onPreDownload = Factory::getPreEvent($request);
$this->onPostDownload = Factory::getPostEvent($request);
if ($this->degradedMode) {
$this->onPreDownload->attach(new Aspects\AspectDegradedMode());
}
$options += $this->options;
// override
if ('github' === $request->special && isset($options['github-token'])) {
$request->query['access_token'] = $options['github-token'];
}
if ('gitlab' === $request->special && isset($options['gitlab-token'])) {
$request->query['access_token'] = $options['gitlab-token'];
}
if ($this->io->isDebug()) {
$this->io->write('Downloading ' . $fileUrl);
}
if ($progress) {
$this->io->write(" Downloading: <comment>Connecting...</comment>", false);
$request->curlOpts[CURLOPT_NOPROGRESS] = false;
$request->curlOpts[CURLOPT_PROGRESSFUNCTION] = array($this, 'progress');
} else {
$request->curlOpts[CURLOPT_NOPROGRESS] = true;
$request->curlOpts[CURLOPT_PROGRESSFUNCTION] = null;
}
$this->onPreDownload->notify();
$opts = $request->getCurlOpts();
if (empty($opts[CURLOPT_USERPWD])) {
unset($opts[CURLOPT_USERPWD]);
}
$ch = Factory::getConnection($origin, isset($opts[CURLOPT_USERPWD]));
curl_setopt_array($ch, $opts);
list($execStatus, $response) = $exec($ch, $request);
} while ($this->retry);
if ($progress) {
$this->io->overwrite(" Downloading: <comment>100%</comment>");
}
return $execStatus;
}
/**
* @param Package\PackageInterface[] $packages
* @param array $pluginConfig
* @return void
*/
public function download(array $packages, array $pluginConfig)
{
$mh = curl_multi_init();
$unused = array();
$maxConns = $pluginConfig['maxConnections'];
for ($i = 0; $i < $maxConns; ++$i) {
$unused[] = curl_init();
}
/// @codeCoverageIgnoreStart
if (function_exists('curl_share_init')) {
$sh = curl_share_init();
curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
foreach ($unused as $ch) {
curl_setopt($ch, CURLOPT_SHARE, $sh);
}
}
if (function_exists('curl_multi_setopt')) {
if ($pluginConfig['pipeline']) {
curl_multi_setopt($mh, CURLMOPT_PIPELINING, true);
}
}
/// @codeCoverageIgnoreEnd
$cachedir = rtrim($this->config->get('cache-files-dir'), '\\/');
$chFpMap = array();
$running = 0;
//ref type
$remains = 0;
//ref type
$this->totalCnt = count($packages);
$this->successCnt = 0;
$this->failureCnt = 0;
$this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
do {
// prepare curl resources
while ($unused && $packages) {
$package = array_pop($packages);
$filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package);
if (file_exists($filepath)) {
++$this->successCnt;
continue;
}
$ch = array_pop($unused);
// make file resource
$fp = CurlRemoteFilesystem::createFile($filepath);
$chFpMap[(int) $ch] = compact('fp', 'filepath');
// make url
$url = $package->getDistUrl();
$request = new Aspects\HttpGetRequest(parse_url($url, PHP_URL_HOST), $url, $this->io);
$request->verbose = $pluginConfig['verbose'];
if (in_array($package->getName(), $pluginConfig['privatePackages'])) {
$request->maybePublic = false;
} else {
$request->maybePublic = preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl());
}
$onPreDownload = Factory::getPreEvent($request);
$onPreDownload->notify();
$opts = $request->getCurlOpts();
unset($opts[CURLOPT_ENCODING]);
unset($opts[CURLOPT_USERPWD]);
curl_setopt_array($ch, $opts);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_multi_add_handle($mh, $ch);
}
// start multi download
do {
$stat = curl_multi_exec($mh, $running);
} while ($stat === CURLM_CALL_MULTI_PERFORM);
// wait for any event
do {
switch (curl_multi_select($mh, 5)) {
case -1:
usleep(10);
do {
$stat = curl_multi_exec($mh, $running);
} while ($stat === CURLM_CALL_MULTI_PERFORM);
continue 2;
case 0:
continue 2;
default:
do {
$stat = curl_multi_exec($mh, $running);
} while ($stat === CURLM_CALL_MULTI_PERFORM);
do {
if ($raised = curl_multi_info_read($mh, $remains)) {
$ch = $raised['handle'];
$errno = curl_errno($ch);
$info = curl_getinfo($ch);
curl_setopt($ch, CURLOPT_FILE, STDOUT);
$index = (int) $ch;
$fileinfo = $chFpMap[$index];
unset($chFpMap[$index]);
$fp = $fileinfo['fp'];
$filepath = $fileinfo['filepath'];
fclose($fp);
if (CURLE_OK === $errno && 200 === $info['http_code']) {
++$this->successCnt;
} else {
++$this->failureCnt;
unlink($filepath);
}
$this->io->write($this->makeDownloadingText($info['url']));
curl_multi_remove_handle($mh, $ch);
$unused[] = $ch;
}
} while ($remains);
if ($packages) {
break 2;
}
}
} while ($running);
} while ($packages);
$this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
foreach ($unused as $ch) {
curl_close($ch);
}
curl_multi_close($mh);
}
/**
* @param string $origin
* @param string $fileUrl
* @param boolean $progress
* @param \Closure $exec
*/
protected function fetch($origin, $fileUrl, $progress, $options, $exec)
{
do {
$this->_retry = false;
$request = Factory::getHttpGetRequest($origin, $fileUrl, $this->io, $this->config, $this->pluginConfig);
$this->onPreDownload = Factory::getPreEvent($request);
$this->onPostDownload = Factory::getPostEvent($request);
$options += $this->options;
$request->processRFSOption($options);
if ($this->io->isDebug()) {
$this->io->write('Downloading ' . $fileUrl);
}
if ($progress) {
$this->io->write(" Downloading: <comment>Connecting...</comment>", false);
$request->curlOpts[CURLOPT_NOPROGRESS] = false;
$request->curlOpts[CURLOPT_PROGRESSFUNCTION] = array($this, 'progress');
} else {
$request->curlOpts[CURLOPT_NOPROGRESS] = true;
$request->curlOpts[CURLOPT_PROGRESSFUNCTION] = null;
}
$this->onPreDownload->notify();
$opts = $request->getCurlOpts();
$ch = Factory::getConnection($origin, isset($opts[CURLOPT_USERPWD]));
curl_setopt_array($ch, $opts);
list($execStatus, ) = $exec($ch, $request);
} while ($this->_retry);
if ($progress) {
$this->io->overwrite(" Downloading: <comment>100%</comment>");
}
return $execStatus;
}
/**
* @param Package\PackageInterface[] $packages
* @param array $pluginConfig
* @return void
*/
public function download(array $packages, array $pluginConfig)
{
$mh = curl_multi_init();
$unused = array();
$maxConns = $pluginConfig['maxConnections'];
for ($i = 0; $i < $maxConns; ++$i) {
$unused[] = curl_init();
}
// @codeCoverageIgnoreStart
if (function_exists('curl_share_init')) {
$sh = curl_share_init();
curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
foreach ($unused as $ch) {
curl_setopt($ch, CURLOPT_SHARE, $sh);
}
}
if (function_exists('curl_multi_setopt')) {
if ($pluginConfig['pipeline']) {
curl_multi_setopt($mh, CURLMOPT_PIPELINING, true);
}
}
// @codeCoverageIgnoreEnd
$cachedir = rtrim($this->config->get('cache-files-dir'), '\\/');
$chFpMap = array();
$running = 0;
//ref type
$remains = 0;
//ref type
$this->totalCnt = count($packages);
$this->successCnt = 0;
$this->failureCnt = 0;
$this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
EVENTLOOP:
// prepare curl resources
while (count($unused) > 0 && count($packages) > 0) {
$package = array_pop($packages);
$filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package);
if (file_exists($filepath)) {
++$this->successCnt;
continue;
}
$ch = array_pop($unused);
// make file resource
$chFpMap[(int) $ch] = $outputFile = new OutputFile($filepath);
// make url
$url = $package->getDistUrls();
if (count($url) > 0) {
$url = $url[0];
} else {
$url = $package->getDistUrl();
}
$host = parse_url($url, PHP_URL_HOST) ?: '';
$request = new Aspects\HttpGetRequest($host, $url, $this->io);
$request->verbose = $pluginConfig['verbose'];
if (in_array($package->getName(), $pluginConfig['privatePackages'])) {
$request->maybePublic = false;
} else {
$request->maybePublic = (bool) preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl());
}
$onPreDownload = Factory::getPreEvent($request);
$onPreDownload->notify();
$opts = $request->getCurlOpts();
if ($pluginConfig['insecure']) {
$opts[CURLOPT_SSL_VERIFYPEER] = false;
}
if (!empty($pluginConfig['userAgent'])) {
$opts[CURLOPT_USERAGENT] = $pluginConfig['userAgent'];
}
if (!empty($pluginConfig['capath'])) {
$opts[CURLOPT_CAPATH] = $pluginConfig['capath'];
}
unset($opts[CURLOPT_ENCODING]);
unset($opts[CURLOPT_USERPWD]);
// ParallelDownloader doesn't support private packages.
curl_setopt_array($ch, $opts);
curl_setopt($ch, CURLOPT_FILE, $outputFile->getPointer());
curl_multi_add_handle($mh, $ch);
}
// wait for any event
do {
$runningBefore = $running;
while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) {
}
SELECT:
$eventCount = curl_multi_select($mh, 5);
if ($eventCount === -1) {
usleep(200 * 1000);
continue;
}
if ($eventCount === 0) {
continue;
}
while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) {
}
if ($running > 0 && $running === $runningBefore) {
goto SELECT;
}
do {
if ($raised = curl_multi_info_read($mh, $remains)) {
$ch = $raised['handle'];
$errno = curl_errno($ch);
$info = curl_getinfo($ch);
curl_setopt($ch, CURLOPT_FILE, STDOUT);
$index = (int) $ch;
$outputFile = $chFpMap[$index];
unset($chFpMap[$index]);
if (CURLE_OK === $errno && 200 === $info['http_code']) {
++$this->successCnt;
} else {
++$this->failureCnt;
$outputFile->setFailure();
}
unset($outputFile);
$this->io->write($this->makeDownloadingText($info['url']));
curl_multi_remove_handle($mh, $ch);
$unused[] = $ch;
}
} while ($remains > 0);
if (count($packages) > 0) {
goto EVENTLOOP;
}
} while ($running > 0);
$this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
foreach ($unused as $ch) {
curl_close($ch);
}
curl_multi_close($mh);
}
