protected function fetch($origin, $fileUrl, $progress, $options, $exec) { do { $this->retry = false; $request = new Aspects\HttpGetRequest($origin, $fileUrl, $this->io); $request->setSpecial(array('github' => $this->config->get('github-domains') ?: array(), 'gitlab' => $this->config->get('gitlab-domains') ?: array())); $this->onPreDownload = Factory::getPreEvent($request); $this->onPostDownload = Factory::getPostEvent($request); if ($this->degradedMode) { $this->onPreDownload->attach(new Aspects\AspectDegradedMode()); } $options += $this->options; // override if ('github' === $request->special && isset($options['github-token'])) { $request->query['access_token'] = $options['github-token']; } if ('gitlab' === $request->special && isset($options['gitlab-token'])) { $request->query['access_token'] = $options['gitlab-token']; } if ($this->io->isDebug()) { $this->io->write('Downloading ' . $fileUrl); } if ($progress) { $this->io->write(" Downloading: <comment>Connecting...</comment>", false); $request->curlOpts[CURLOPT_NOPROGRESS] = false; $request->curlOpts[CURLOPT_PROGRESSFUNCTION] = array($this, 'progress'); } else { $request->curlOpts[CURLOPT_NOPROGRESS] = true; $request->curlOpts[CURLOPT_PROGRESSFUNCTION] = null; } $this->onPreDownload->notify(); $opts = $request->getCurlOpts(); if (empty($opts[CURLOPT_USERPWD])) { unset($opts[CURLOPT_USERPWD]); } $ch = Factory::getConnection($origin, isset($opts[CURLOPT_USERPWD])); curl_setopt_array($ch, $opts); list($execStatus, $response) = $exec($ch, $request); } while ($this->retry); if ($progress) { $this->io->overwrite(" Downloading: <comment>100%</comment>"); } return $execStatus; }
/** * @param Package\PackageInterface[] $packages * @param array $pluginConfig * @return void */ public function download(array $packages, array $pluginConfig) { $mh = curl_multi_init(); $unused = array(); $maxConns = $pluginConfig['maxConnections']; for ($i = 0; $i < $maxConns; ++$i) { $unused[] = curl_init(); } /// @codeCoverageIgnoreStart if (function_exists('curl_share_init')) { $sh = curl_share_init(); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); foreach ($unused as $ch) { curl_setopt($ch, CURLOPT_SHARE, $sh); } } if (function_exists('curl_multi_setopt')) { if ($pluginConfig['pipeline']) { curl_multi_setopt($mh, CURLMOPT_PIPELINING, true); } } /// @codeCoverageIgnoreEnd $cachedir = rtrim($this->config->get('cache-files-dir'), '\\/'); $chFpMap = array(); $running = 0; //ref type $remains = 0; //ref type $this->totalCnt = count($packages); $this->successCnt = 0; $this->failureCnt = 0; $this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); do { // prepare curl resources while ($unused && $packages) { $package = array_pop($packages); $filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package); if (file_exists($filepath)) { ++$this->successCnt; continue; } $ch = array_pop($unused); // make file resource $fp = CurlRemoteFilesystem::createFile($filepath); $chFpMap[(int) $ch] = compact('fp', 'filepath'); // make url $url = $package->getDistUrl(); $request = new Aspects\HttpGetRequest(parse_url($url, PHP_URL_HOST), $url, $this->io); $request->verbose = $pluginConfig['verbose']; if (in_array($package->getName(), $pluginConfig['privatePackages'])) { $request->maybePublic = false; } else { $request->maybePublic = preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl()); } $onPreDownload = Factory::getPreEvent($request); $onPreDownload->notify(); $opts = $request->getCurlOpts(); unset($opts[CURLOPT_ENCODING]); unset($opts[CURLOPT_USERPWD]); curl_setopt_array($ch, $opts); curl_setopt($ch, CURLOPT_FILE, $fp); curl_multi_add_handle($mh, $ch); } // start multi download do { $stat = curl_multi_exec($mh, $running); } while ($stat === CURLM_CALL_MULTI_PERFORM); // wait for any event do { switch (curl_multi_select($mh, 5)) { case -1: usleep(10); do { $stat = curl_multi_exec($mh, $running); } while ($stat === CURLM_CALL_MULTI_PERFORM); continue 2; case 0: continue 2; default: do { $stat = curl_multi_exec($mh, $running); } while ($stat === CURLM_CALL_MULTI_PERFORM); do { if ($raised = curl_multi_info_read($mh, $remains)) { $ch = $raised['handle']; $errno = curl_errno($ch); $info = curl_getinfo($ch); curl_setopt($ch, CURLOPT_FILE, STDOUT); $index = (int) $ch; $fileinfo = $chFpMap[$index]; unset($chFpMap[$index]); $fp = $fileinfo['fp']; $filepath = $fileinfo['filepath']; fclose($fp); if (CURLE_OK === $errno && 200 === $info['http_code']) { ++$this->successCnt; } else { ++$this->failureCnt; unlink($filepath); } $this->io->write($this->makeDownloadingText($info['url'])); curl_multi_remove_handle($mh, $ch); $unused[] = $ch; } } while ($remains); if ($packages) { break 2; } } } while ($running); } while ($packages); $this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); foreach ($unused as $ch) { curl_close($ch); } curl_multi_close($mh); }
/** * @param string $origin * @param string $fileUrl * @param boolean $progress * @param \Closure $exec */ protected function fetch($origin, $fileUrl, $progress, $options, $exec) { do { $this->_retry = false; $request = Factory::getHttpGetRequest($origin, $fileUrl, $this->io, $this->config, $this->pluginConfig); $this->onPreDownload = Factory::getPreEvent($request); $this->onPostDownload = Factory::getPostEvent($request); $options += $this->options; $request->processRFSOption($options); if ($this->io->isDebug()) { $this->io->write('Downloading ' . $fileUrl); } if ($progress) { $this->io->write(" Downloading: <comment>Connecting...</comment>", false); $request->curlOpts[CURLOPT_NOPROGRESS] = false; $request->curlOpts[CURLOPT_PROGRESSFUNCTION] = array($this, 'progress'); } else { $request->curlOpts[CURLOPT_NOPROGRESS] = true; $request->curlOpts[CURLOPT_PROGRESSFUNCTION] = null; } $this->onPreDownload->notify(); $opts = $request->getCurlOpts(); $ch = Factory::getConnection($origin, isset($opts[CURLOPT_USERPWD])); curl_setopt_array($ch, $opts); list($execStatus, ) = $exec($ch, $request); } while ($this->_retry); if ($progress) { $this->io->overwrite(" Downloading: <comment>100%</comment>"); } return $execStatus; }
/** * @param Package\PackageInterface[] $packages * @param array $pluginConfig * @return void */ public function download(array $packages, array $pluginConfig) { $mh = curl_multi_init(); $unused = array(); $maxConns = $pluginConfig['maxConnections']; for ($i = 0; $i < $maxConns; ++$i) { $unused[] = curl_init(); } // @codeCoverageIgnoreStart if (function_exists('curl_share_init')) { $sh = curl_share_init(); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); foreach ($unused as $ch) { curl_setopt($ch, CURLOPT_SHARE, $sh); } } if (function_exists('curl_multi_setopt')) { if ($pluginConfig['pipeline']) { curl_multi_setopt($mh, CURLMOPT_PIPELINING, true); } } // @codeCoverageIgnoreEnd $cachedir = rtrim($this->config->get('cache-files-dir'), '\\/'); $chFpMap = array(); $running = 0; //ref type $remains = 0; //ref type $this->totalCnt = count($packages); $this->successCnt = 0; $this->failureCnt = 0; $this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); EVENTLOOP: // prepare curl resources while (count($unused) > 0 && count($packages) > 0) { $package = array_pop($packages); $filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package); if (file_exists($filepath)) { ++$this->successCnt; continue; } $ch = array_pop($unused); // make file resource $chFpMap[(int) $ch] = $outputFile = new OutputFile($filepath); // make url $url = $package->getDistUrls(); if (count($url) > 0) { $url = $url[0]; } else { $url = $package->getDistUrl(); } $host = parse_url($url, PHP_URL_HOST) ?: ''; $request = new Aspects\HttpGetRequest($host, $url, $this->io); $request->verbose = $pluginConfig['verbose']; if (in_array($package->getName(), $pluginConfig['privatePackages'])) { $request->maybePublic = false; } else { $request->maybePublic = (bool) preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl()); } $onPreDownload = Factory::getPreEvent($request); $onPreDownload->notify(); $opts = $request->getCurlOpts(); if ($pluginConfig['insecure']) { $opts[CURLOPT_SSL_VERIFYPEER] = false; } if (!empty($pluginConfig['userAgent'])) { $opts[CURLOPT_USERAGENT] = $pluginConfig['userAgent']; } if (!empty($pluginConfig['capath'])) { $opts[CURLOPT_CAPATH] = $pluginConfig['capath']; } unset($opts[CURLOPT_ENCODING]); unset($opts[CURLOPT_USERPWD]); // ParallelDownloader doesn't support private packages. curl_setopt_array($ch, $opts); curl_setopt($ch, CURLOPT_FILE, $outputFile->getPointer()); curl_multi_add_handle($mh, $ch); } // wait for any event do { $runningBefore = $running; while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) { } SELECT: $eventCount = curl_multi_select($mh, 5); if ($eventCount === -1) { usleep(200 * 1000); continue; } if ($eventCount === 0) { continue; } while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) { } if ($running > 0 && $running === $runningBefore) { goto SELECT; } do { if ($raised = curl_multi_info_read($mh, $remains)) { $ch = $raised['handle']; $errno = curl_errno($ch); $info = curl_getinfo($ch); curl_setopt($ch, CURLOPT_FILE, STDOUT); $index = (int) $ch; $outputFile = $chFpMap[$index]; unset($chFpMap[$index]); if (CURLE_OK === $errno && 200 === $info['http_code']) { ++$this->successCnt; } else { ++$this->failureCnt; $outputFile->setFailure(); } unset($outputFile); $this->io->write($this->makeDownloadingText($info['url'])); curl_multi_remove_handle($mh, $ch); $unused[] = $ch; } } while ($remains > 0); if (count($packages) > 0) { goto EVENTLOOP; } } while ($running > 0); $this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); foreach ($unused as $ch) { curl_close($ch); } curl_multi_close($mh); }