/** * Validate repository URL * * @param $repositoryUrl * @return bool */ protected function isRepositoryValid($repositoryUrl) { $io = new NullIO(); $config = Factory::createConfig(); $io->loadConfiguration($config); $repository = new VcsRepository(array('url' => $repositoryUrl), $io, $config); if (!($driver = $repository->getDriver())) { return false; } $information = $driver->getComposerInformation($driver->getRootIdentifier()); return !empty($information['name']); }
/** * {@inheritdoc} */ protected function execute(InputInterface $input, OutputInterface $output) { $verbose = $input->getOption('verbose'); $force = $input->getOption('force'); $package = $input->getArgument('package'); $doctrine = $this->getContainer()->get('doctrine'); $router = $this->getContainer()->get('router'); $flags = 0; if ($package) { $packages = array(array('id' => $doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)->getId())); $flags = Updater::UPDATE_TAGS; } elseif ($force) { $packages = $doctrine->getEntityManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC'); $flags = Updater::UPDATE_TAGS; } else { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackages(); } $ids = array(); foreach ($packages as $package) { $ids[] = $package['id']; } if ($input->getOption('delete-before')) { $flags = Updater::DELETE_BEFORE; } $updater = $this->getContainer()->get('packagist.package_updater'); $start = new \DateTime(); $input->setInteractive(false); $io = $verbose ? new ConsoleIO($input, $output, $this->getApplication()->getHelperSet()) : new NullIO(); $config = Factory::createConfig(); $loader = new ValidatingArrayLoader(new ArrayLoader()); while ($ids) { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($ids, 0, 50)); foreach ($packages as $package) { if ($verbose) { $output->writeln('Importing ' . $package->getRepository()); } try { $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config); $repository->setLoader($loader); $updater->update($package, $repository, $flags, $start); } catch (\Exception $e) { $output->writeln('<error>Exception: ' . $e->getMessage() . ' at ' . $e->getFile() . ':' . $e->getLine() . ', skipping package ' . $router->generate('view_package', array('name' => $package->getName()), true) . '</error>'); } } $doctrine->getEntityManager()->clear(); unset($packages); } }
/** * @param string $url * @param string $token * @param bool $cacheDir * @param null $bufferIO * * @throws \Exception */ public function __construct($url, $token = '', $cacheDir = false, $bufferIO = null) { $this->url = $url; $this->io = new NullIO(); $this->log = $bufferIO ? $bufferIO : new BufferIO(); $config = Factory::createConfig(); $cfg = ['config' => []]; if ($cacheDir) { $cfg['config']['cache-dir'] = $cacheDir; } if ($token) { $cfg['config']['github-oauth'] = ['github.com' => $token]; } $config->merge($cfg); $this->cacheDir = $cacheDir; $this->io->loadConfiguration($config); $this->repository = new Repository\VcsRepository(['url' => $url, 'no-api' => false], $this->io, $config); $driver = $this->vcsDriver = $this->repository->getDriver(); if (!$driver) { throw new \Exception('No driver found for <' . $url . '>'); } $this->driver = $driver; $composerInfoMaster = $this->driver->getComposerInformation($this->driver->getRootIdentifier()); if (!$composerInfoMaster) { throw new \Exception('master must have a valid composer.json'); } $this->name = $composerInfoMaster['name']; list($this->vendorName, $this->packageName) = explode('/', $this->name); preg_match('#^(?:(?:https?|git)://([^/]+)/|git@([^:]+):)([^/]+)/(.+?)(?:\\.git|/)?$#', $this->url, $match); $this->gitHubVendorName = $match[3]; $this->gitHubRepositoryName = $match[4]; $client = new \Github\Client(); $client->authenticate($token, null, \GitHub\Client::AUTH_URL_TOKEN); $this->client = $client; }
/** * @return string[] */ private function fetch() { $this->io->write(sprintf(' - Updating <info>%s</info> tag list', $this->package)); if (isset($_ENV['HRDNS_PHANTOMJS_VERSION']) && !empty($_ENV['HRDNS_PHANTOMJS_VERSION'])) { $this->cache = [$_ENV['HRDNS_PHANTOMJS_VERSION']]; } if ($this->cache === null) { $this->cache = []; $this->io->writeError(" Downloading: <comment>Connecting...</comment>", false); $repo = new VcsRepository(['url' => $this->url, 'type' => $this->type], $this->io, $this->config); $this->cache = array_keys($repo->getDriver()->getTags()); $this->io->overwriteError('', false); $this->io->overwriteError(sprintf("\r Found <comment>%d</comment> versions\n", count($this->cache))); } return $this->cache; }
/** * {@inheritdoc} */ public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher = null, array $drivers = null) { $drivers = array('git' => 'Drupal\\ParseComposer\\GitDriver'); $repoConfig['type'] = 'git'; parent::__construct($repoConfig, $io, $config, $dispatcher, $drivers); $parts = preg_split('{[/:]}', $this->url); $last = end($parts); $this->repoConfig['drupalProjectName'] = current(explode('.', $last)); }
/** * Initialize repository * * @param array $repoConfig * @param IOInterface $io * @param Config $config * @param EventDispatcher $dispatcher * @param array $drivers */ public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher = null, array $drivers = null) { $drivers[self::TYPE_VCS] = 'Composer\\Repository\\Vcs\\VcsDriver'; $drivers[self::TYPE_GIT] = 'Composer\\Repository\\Vcs\\GitDriver'; $drivers[self::TYPE_GITLAB] = 'Composer\\Repository\\Vcs\\GitLabDriver'; $drivers[self::TYPE_GITHUB] = 'Composer\\Repository\\Vcs\\GitHubDriver'; $drivers[self::TYPE_GIT_BITBACKET] = 'Composer\\Repository\\Vcs\\GitBitbucketDriver'; parent::__construct($repoConfig, $io, $config, $dispatcher, $drivers); }
/** * Intercept exceptions to be able to backlist some repos */ protected function initialize() { try { parent::initialize(); } catch (InvalidRepositoryException $e) { self::$blacklist[] = $this->url; // Todo: handle this info somehow! throw $e; } }
protected function preProcess(VcsDriverInterface $driver, array $data, $identifier) { $data = parent::preProcess($driver, $data, $identifier); if (isset($data['source']['type']) && $data['source']['type'] === 'svn') { $data['source']['type'] = 'svn-export'; } if (isset($data['dist']['type']) && $data['dist']['type'] === 'svn') { $data['dist']['type'] = 'svn-export'; } return $data; }
/** * Constructor. * * @param array $repoConfig * @param IOInterface $io * @param Config $config * @param EventDispatcher $dispatcher * @param array $drivers */ public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher = null, array $drivers = null) { $drivers = $drivers ?: Assets::getVcsDrivers(); $assetType = substr($repoConfig['type'], 0, strpos($repoConfig['type'], '-')); $assetType = Assets::createType($assetType); $repoConfig['asset-type'] = $assetType->getName(); $repoConfig['filename'] = $assetType->getFilename(); $this->assetType = $assetType; $this->dispatcher = $dispatcher; $this->filter = isset($repoConfig['vcs-package-filter']) && $repoConfig['vcs-package-filter'] instanceof VcsPackageFilter ? $repoConfig['vcs-package-filter'] : null; parent::__construct($repoConfig, $io, $config, $dispatcher, $drivers); }
/** * @return string[] */ private function fetch() { $this->io->write(sprintf(' - Updating <info>%s</info> tag list', $this->package)); if (isset($_ENV['HRDNS_SELENIUM_VERSION']) && !empty($_ENV['HRDNS_SELENIUM_VERSION'])) { $this->cache = [$_ENV['HRDNS_SELENIUM_VERSION']]; } if ($this->cache === null) { $this->cache = []; $this->io->writeError(" Downloading: <comment>Connecting...</comment>", false); $repo = new VcsRepository(['url' => $this->url, 'type' => $this->type], $this->io, $this->config); $this->cache = array_keys($repo->getDriver()->getTags()); $this->io->overwriteError(" Downloading: <comment>100%</comment>", false); $this->cache = array_filter($this->cache, function ($version) { return strpos($version, 'selenium-') === 0; }); array_walk($this->cache, function (&$value) { $value = substr($value, 9); }); $this->io->overwriteError(sprintf("\r Found <comment>%d</comment> versions\n", count($this->cache))); } return $this->cache; }
/** * Perform the package update * * @param Request $request the current request * @param string $url the repository's URL (deducted from the request) * @param string $urlRegex the regex used to split the user packages into domain and path * @return Response */ protected function receivePost(Request $request, $url, $urlRegex) { // try to parse the URL first to avoid the DB lookup on malformed requests if (!preg_match($urlRegex, $url)) { return new Response(json_encode(array('status' => 'error', 'message' => 'Could not parse payload repository URL')), 406); } // find the user $user = $this->findUser($request); if (!$user) { return new Response(json_encode(array('status' => 'error', 'message' => 'Invalid credentials')), 403); } // try to find the user package $packages = $this->findPackagesByUrl($user, $url, $urlRegex); if (!$packages) { return new Response(json_encode(array('status' => 'error', 'message' => 'Could not find a package that matches this request (does user maintain the package?)')), 404); } // don't die if this takes a while set_time_limit(3600); // put both updating the database and scanning the repository in a transaction $em = $this->get('doctrine.orm.entity_manager'); $updater = $this->get('packagist.package_updater'); $config = Factory::createConfig(); $io = new BufferIO('', OutputInterface::VERBOSITY_VERBOSE, new HtmlOutputFormatter(Factory::createAdditionalStyles())); $io->loadConfiguration($config); try { foreach ($packages as $package) { $em->transactional(function ($em) use($package, $updater, $io, $config) { // prepare dependencies $loader = new ValidatingArrayLoader(new ArrayLoader()); // prepare repository $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config); $repository->setLoader($loader); // perform the actual update (fetch and re-scan the repository's source) $updater->update($io, $config, $package, $repository); // update the package entity $package->setAutoUpdated(true); $em->flush(); }); } } catch (\Exception $e) { if ($e instanceof InvalidRepositoryException) { $this->get('packagist.package_manager')->notifyUpdateFailure($package, $e, $io->getOutput()); } return new Response(json_encode(array('status' => 'error', 'message' => '[' . get_class($e) . '] ' . $e->getMessage(), 'details' => '<pre>' . $io->getOutput() . '</pre>')), 400); } return new JsonResponse(array('status' => 'success'), 202); }
private function getRepoConfig(VcsRepository $repository) { return $repository->getRepoConfig(); }
/** * Set repository * * @param string $repository */ public function setRepository($repoUrl) { $this->vcsDriver = null; // prevent local filesystem URLs if (preg_match('{^(\\.|[a-z]:|/)}i', $repoUrl)) { return; } $this->repository = $repoUrl; // avoid user@host URLs if (preg_match('{https?://.+@}', $repoUrl)) { return; } try { $io = new NullIO(); $config = Factory::createConfig(); $io->loadConfiguration($config); $repository = new VcsRepository(array('url' => $this->repository), $io, $config); $driver = $this->vcsDriver = $repository->getDriver(); if (!$driver) { return; } $information = $driver->getComposerInformation($driver->getRootIdentifier()); if (!isset($information['name'])) { return; } if (null === $this->getName()) { $this->setName($information['name']); } if ($driver instanceof GitHubDriver) { $this->repository = $driver->getRepositoryUrl(); } } catch (\Exception $e) { $this->vcsDriverError = '[' . get_class($e) . '] ' . $e->getMessage(); } }
protected function receivePost(Request $request, $urlRegex, $optionalRepositorySuffix) { $payload = json_decode($request->request->get('payload'), true); if (!$payload || !isset($payload['repository']['url'])) { return new Response(json_encode(array('status' => 'error', 'message' => 'Missing or invalid payload')), 406); } // try to parse the URL first to avoid the DB lookup on malformed requests if (!preg_match($urlRegex, $payload['repository']['url'], $match)) { return new Response(json_encode(array('status' => 'error', 'message' => 'Could not parse payload repository URL')), 406); } $payloadRepositoryChunk = $match['url']; $username = $request->request->has('username') ? $request->request->get('username') : $request->query->get('username'); $apiToken = $request->request->has('apiToken') ? $request->request->get('apiToken') : $request->query->get('apiToken'); $user = $this->get('packagist.user_repository')->findOneBy(array('username' => $username, 'apiToken' => $apiToken)); if (!$user) { return new Response(json_encode(array('status' => 'error', 'message' => 'Invalid credentials')), 403); } $updated = false; $config = Factory::createConfig(); $loader = new ValidatingArrayLoader(new ArrayLoader()); $updater = $this->get('packagist.package_updater'); $em = $this->get('doctrine.orm.entity_manager'); foreach ($user->getPackages() as $package) { if (preg_match('{' . preg_quote($payloadRepositoryChunk) . $optionalRepositorySuffix . '}', $package->getRepository())) { set_time_limit(3600); $updated = true; $repository = new VcsRepository(array('url' => $package->getRepository()), new NullIO(), $config); $repository->setLoader($loader); $package->setAutoUpdated(true); $em->flush(); $updater->update($package, $repository); } } if ($updated) { return new Response('{"status": "success"}', 202); } return new Response(json_encode(array('status' => 'error', 'message' => 'Could not find a package that matches this request (does user maintain the package?)')), 404); }
public function testLoadVersions() { $expected = array('0.6.0' => true, '1.0.0' => true, '1.0.x-dev' => true, '1.1.x-dev' => true, 'dev-feature-b' => true, 'dev-feature-a' => true, 'dev-master' => true); $repo = new VcsRepository(array('url' => self::$gitRepo, 'type' => 'vcs'), new NullIO()); $packages = $repo->getPackages(); $dumper = new ArrayDumper(); foreach ($packages as $package) { if (isset($expected[$package->getPrettyVersion()])) { unset($expected[$package->getPrettyVersion()]); } else { $this->fail('Unexpected version ' . $package->getPrettyVersion() . ' in ' . json_encode($dumper->dump($package))); } } $this->assertEmpty($expected, 'Missing versions: ' . implode(', ', array_keys($expected))); }
/** * @Template() * @Route("/packages/{name}", name="update_package", requirements={"name"="[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+"}, defaults={"_format" = "json"}) * @Method({"PUT"}) */ public function updatePackageAction(Request $req, $name) { $doctrine = $this->getDoctrine(); try { /** @var Package $package */ $package = $doctrine->getRepository('PackagistWebBundle:Package')->getPackageByName($name); } catch (NoResultException $e) { return new Response(json_encode(array('status' => 'error', 'message' => 'Package not found')), 404); } $username = $req->request->has('username') ? $req->request->get('username') : $req->query->get('username'); $apiToken = $req->request->has('apiToken') ? $req->request->get('apiToken') : $req->query->get('apiToken'); $update = $req->request->get('update', $req->query->get('update')); $autoUpdated = $req->request->get('autoUpdated', $req->query->get('autoUpdated')); $updateEqualRefs = $req->request->get('updateAll', $req->query->get('updateAll')); $user = $this->getUser() ?: $doctrine->getRepository('PackagistWebBundle:User')->findOneBy(array('username' => $username, 'apiToken' => $apiToken)); if (!$user) { return new Response(json_encode(array('status' => 'error', 'message' => 'Invalid credentials')), 403); } if ($package->getMaintainers()->contains($user) || $this->isGranted('ROLE_UPDATE_PACKAGES')) { $req->getSession()->save(); if (null !== $autoUpdated) { $package->setAutoUpdated((bool) $autoUpdated); $doctrine->getManager()->flush(); } if ($update) { set_time_limit(3600); $updater = $this->get('packagist.package_updater'); $io = new BufferIO('', OutputInterface::VERBOSITY_VERY_VERBOSE, new HtmlOutputFormatter(Factory::createAdditionalStyles())); $config = Factory::createConfig(); $io->loadConfiguration($config); $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config); $loader = new ValidatingArrayLoader(new ArrayLoader()); $repository->setLoader($loader); try { $updater->update($io, $config, $package, $repository, $updateEqualRefs ? Updater::UPDATE_EQUAL_REFS : 0); } catch (\Exception $e) { return new Response(json_encode(array('status' => 'error', 'message' => '[' . get_class($e) . '] ' . $e->getMessage(), 'details' => '<pre>' . $io->getOutput() . '</pre>')), 400); } } return new Response('{"status": "success"}', 202); } return new JsonResponse(array('status' => 'error', 'message' => 'Could not find a package that matches this request (does user maintain the package?)'), 404); }
/** * Set repository * * @param string $repository */ public function setRepository($repository) { $this->vcsDriver = null; // prevent local filesystem URLs if (preg_match('{^(\\.|[a-z]:|/)}i', $repository)) { return; } $this->repository = $repository; // avoid user@host URLs if (preg_match('{//.+@}', $repository)) { return; } try { $config = Factory::createConfig(); $repository = new VcsRepository(array('url' => $repository), new NullIO(), $config); $driver = $this->vcsDriver = $repository->getDriver(); if (!$driver) { return; } $information = $driver->getComposerInformation($driver->getRootIdentifier()); if (!isset($information['name'])) { throw new \RuntimeException('No name found in composer.json'); } if (null === $this->getName()) { $this->setName($information['name']); } } catch (\Exception $e) { } }
/** * Set repository * * @param string $repository */ public function setRepository($repository) { $this->repository = $repository; // avoid user@host URLs if (preg_match('{//.+@}', $repository)) { return; } try { $repository = new VcsRepository(array('url' => $repository), new NullIO()); $repo = $this->repositoryClass = $repository->getDriver(); if (!$repo) { return; } $information = $repo->getComposerInformation($repo->getRootIdentifier()); $this->setName($information['name']); } catch (\UnexpectedValueException $e) { } }
/** * {@inheritdoc} */ protected function execute(InputInterface $input, OutputInterface $output) { $verbose = $input->getOption('verbose'); $force = $input->getOption('force'); $package = $input->getArgument('package'); $doctrine = $this->getContainer()->get('doctrine'); $logger = $this->getContainer()->get('logger'); $this->versionParser = new VersionParser(); if ($package) { $packages = array($doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)); } elseif ($force) { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->findAll(); } else { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackages(); } $start = new \DateTime(); foreach ($packages as $package) { if ($verbose) { $output->writeln('Importing ' . $package->getRepository()); } try { $repository = new VcsRepository(array('url' => $package->getRepository()), new NullIO()); if ($verbose) { $repository->setDebug(true); } $versions = $repository->getPackages(); usort($versions, function ($a, $b) { return version_compare($a->getVersion(), $b->getVersion()); }); // clear existing versions to force a clean reloading if --force is enabled if ($force) { $versionRepo = $doctrine->getRepository('PackagistWebBundle:Version'); foreach ($package->getVersions() as $version) { $versionRepo->remove($version); } $doctrine->getEntityManager()->flush(); $doctrine->getEntityManager()->refresh($package); } foreach ($versions as $version) { if ($verbose) { $output->writeln('Storing ' . $version->getPrettyVersion() . ' (' . $version->getVersion() . ')'); } $this->updateInformation($output, $doctrine, $package, $version); $doctrine->getEntityManager()->flush(); } // remove outdated -dev versions foreach ($package->getVersions() as $version) { if ($version->getDevelopment() && $version->getUpdatedAt() < $start) { if ($verbose) { $output->writeln('Deleting stale version: ' . $version->getVersion()); } $doctrine->getRepository('PackagistWebBundle:Version')->remove($version); } } $package->setUpdatedAt(new \DateTime()); $package->setCrawledAt(new \DateTime()); $doctrine->getEntityManager()->flush(); } catch (\Exception $e) { $output->writeln('<error>Exception: ' . $e->getMessage() . ', skipping package ' . $package->getName() . '.</error>'); } } }
/** * @Template() * @Route("/packages/{name}", name="update_package", requirements={"name"="[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+"}, defaults={"_format" = "json"}) * @Method({"PUT"}) */ public function updatePackageAction($name) { $doctrine = $this->getDoctrine(); try { $package = $doctrine->getRepository('PackagistWebBundle:Package')->getFullPackageByName($name); } catch (NoResultException $e) { return new Response(json_encode(array('status' => 'error', 'message' => 'Package not found')), 404); } $req = $this->getRequest(); $username = $req->request->has('username') ? $req->request->get('username') : $req->query->get('username'); $apiToken = $req->request->has('apiToken') ? $req->request->get('apiToken') : $req->query->get('apiToken'); $update = $req->request->get('update', $req->query->get('update')); $autoUpdated = $req->request->get('autoUpdated', $req->query->get('autoUpdated')); $user = $this->getUser() ?: $doctrine->getRepository('PackagistWebBundle:User')->findOneBy(array('username' => $username, 'apiToken' => $apiToken)); if (!$user) { return new Response(json_encode(array('status' => 'error', 'message' => 'Invalid credentials')), 403); } if ($package->getMaintainers()->contains($user) || $this->get('security.context')->isGranted('ROLE_UPDATE_PACKAGES')) { if (null !== $autoUpdated) { $package->setAutoUpdated((bool) $autoUpdated); $doctrine->getManager()->flush(); } if ($update) { set_time_limit(3600); $updater = $this->get('packagist.package_updater'); $config = Factory::createConfig(); $repository = new VcsRepository(array('url' => $package->getRepository()), new NullIO(), $config); $loader = new ValidatingArrayLoader(new ArrayLoader()); $repository->setLoader($loader); $updater->update($package, $repository, Updater::UPDATE_TAGS); } return new Response('{"status": "success"}', 202); } return new Response(json_encode(array('status' => 'error', 'message' => 'Could not find a package that matches this request (does user maintain the package?)')), 404); }
/** * Initialize GITLab downloader * * @param array $repoConfig * @param IOInterface $io * @param Config $config * @param EventDispatcher $dispatcher * @param array $drivers */ public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $dispatcher = null, array $drivers = null) { $drivers[self::TYPE] = 'AndKirby\\Composer\\MultiRepo\\Repository\\Vcs\\GitLabDriver'; parent::__construct($repoConfig, $io, $config, $dispatcher, $drivers); }
/** * {@inheritdoc} */ protected function execute(InputInterface $input, OutputInterface $output) { $verbose = $input->getOption('verbose'); $force = $input->getOption('force'); $package = $input->getArgument('package'); $doctrine = $this->getContainer()->get('doctrine'); $router = $this->getContainer()->get('router'); $flags = 0; if ($package) { $packages = array(array('id' => $doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)->getId())); $flags = $force ? Updater::UPDATE_EQUAL_REFS : 0; } elseif ($force) { $packages = $doctrine->getManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC'); $flags = Updater::UPDATE_EQUAL_REFS; } else { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackages(); } $ids = array(); foreach ($packages as $package) { $ids[] = $package['id']; } if ($input->getOption('delete-before')) { $flags = Updater::DELETE_BEFORE; } elseif ($input->getOption('update-equal-refs')) { $flags = Updater::UPDATE_EQUAL_REFS; } $updater = $this->getContainer()->get('packagist.package_updater'); $start = new \DateTime(); if ($verbose && $input->getOption('notify-failures')) { throw new \LogicException('Failures can not be notified in verbose mode since the output is piped to the CLI'); } $input->setInteractive(false); $config = Factory::createConfig(); $io = $verbose ? new ConsoleIO($input, $output, $this->getApplication()->getHelperSet()) : new BufferIO(''); $io->loadConfiguration($config); $loader = new ValidatingArrayLoader(new ArrayLoader()); while ($ids) { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($ids, 0, 50)); foreach ($packages as $package) { if ($verbose) { $output->writeln('Importing ' . $package->getRepository()); } try { if (null === $io || $io instanceof BufferIO) { $io = new BufferIO(''); $io->loadConfiguration($config); } $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config); $repository->setLoader($loader); $updater->update($io, $config, $package, $repository, $flags, $start); } catch (InvalidRepositoryException $e) { $output->writeln('<error>Broken repository in ' . $router->generate('view_package', array('name' => $package->getName()), true) . ': ' . $e->getMessage() . '</error>'); if ($input->getOption('notify-failures')) { if (!$this->getContainer()->get('packagist.package_manager')->notifyUpdateFailure($package, $e, $io->getOutput())) { $output->writeln('<error>Failed to notify maintainers</error>'); } } } catch (\Exception $e) { $output->writeln('<error>Error updating ' . $router->generate('view_package', array('name' => $package->getName()), true) . ' [' . get_class($e) . ']: ' . $e->getMessage() . ' at ' . $e->getFile() . ':' . $e->getLine() . '</error>'); } } $doctrine->getManager()->clear(); unset($packages); } }
private function updateGitHubInfo(RemoteFilesystem $rfs, Package $package, $owner, $repo, VcsRepository $repository) { $baseApiUrl = 'https://api.github.com/repos/' . $owner . '/' . $repo; $driver = $repository->getDriver(); if (!$driver instanceof GitHubDriver) { return; } $repoData = $driver->getRepoData(); try { $opts = ['http' => ['header' => ['Accept: application/vnd.github.v3.html']]]; $readme = $rfs->getContents('github.com', $baseApiUrl . '/readme', false, $opts); } catch (\Exception $e) { if (!$e instanceof \Composer\Downloader\TransportException || $e->getCode() !== 404) { return; } // 404s just mean no readme present so we proceed with the rest } if (!empty($readme)) { $elements = array('p', 'br', 'small', 'strong', 'b', 'em', 'i', 'strike', 'sub', 'sup', 'ins', 'del', 'ol', 'ul', 'li', 'h1', 'h2', 'h3', 'dl', 'dd', 'dt', 'pre', 'code', 'samp', 'kbd', 'q', 'blockquote', 'abbr', 'cite', 'table', 'thead', 'tbody', 'th', 'tr', 'td', 'a[href|target|rel|id]', 'img[src|title|alt|width|height|style]'); $config = \HTMLPurifier_Config::createDefault(); $config->set('HTML.Allowed', implode(',', $elements)); $config->set('Attr.EnableID', true); $config->set('Attr.AllowedFrameTargets', ['_blank']); $purifier = new \HTMLPurifier($config); $readme = $purifier->purify($readme); $dom = new \DOMDocument(); $dom->loadHTML('<?xml encoding="UTF-8">' . $readme); // Links can not be trusted, mark them nofollow and convert relative to absolute links $links = $dom->getElementsByTagName('a'); foreach ($links as $link) { $link->setAttribute('rel', 'nofollow noopener external'); if ('#' === substr($link->getAttribute('href'), 0, 1)) { $link->setAttribute('href', '#user-content-' . substr($link->getAttribute('href'), 1)); } elseif ('mailto:' === substr($link->getAttribute('href'), 0, 7)) { // do nothing } elseif (false === strpos($link->getAttribute('href'), '//')) { $link->setAttribute('href', 'https://github.com/' . $owner . '/' . $repo . '/blob/HEAD/' . $link->getAttribute('href')); } } // convert relative to absolute images $images = $dom->getElementsByTagName('img'); foreach ($images as $img) { if (false === strpos($img->getAttribute('src'), '//')) { $img->setAttribute('src', 'https://raw.github.com/' . $owner . '/' . $repo . '/HEAD/' . $img->getAttribute('src')); } } // remove first title as it's usually the project name which we don't need if ($dom->getElementsByTagName('h1')->length) { $first = $dom->getElementsByTagName('h1')->item(0); $first->parentNode->removeChild($first); } elseif ($dom->getElementsByTagName('h2')->length) { $first = $dom->getElementsByTagName('h2')->item(0); $first->parentNode->removeChild($first); } $readme = $dom->saveHTML(); $readme = substr($readme, strpos($readme, '<body>') + 6); $readme = substr($readme, 0, strrpos($readme, '</body>')); $package->setReadme($readme); } if (!empty($repoData['language'])) { $package->setLanguage($repoData['language']); } if (isset($repoData['stargazers_count'])) { $package->setGitHubStars($repoData['stargazers_count']); } if (isset($repoData['subscribers_count'])) { $package->setGitHubWatches($repoData['subscribers_count']); } if (isset($repoData['network_count'])) { $package->setGitHubForks($repoData['network_count']); } if (isset($repoData['open_issues_count'])) { $package->setGitHubOpenIssues($repoData['open_issues_count']); } }