private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package) { $document->id = $package->getId(); $document->name = $package->getName(); $document->description = $package->getDescription(); $tags = array(); foreach ($package->getVersions() as $version) { foreach ($version->getTags() as $tag) { $tags[mb_strtolower($tag->getName(), 'UTF-8')] = true; } } $document->tags = array_keys($tags); }
/** * @Route( * "/packages/{name}/stats.{_format}", * name="view_package_stats", * requirements={"name"="[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?", "_format"="(json)"}, * defaults={"_format"="html"} * ) * @Template() */ public function statsAction(Request $req, Package $package) { $versions = $package->getVersions()->toArray(); usort($versions, Package::class . '::sortVersions'); $date = $this->guessStatsStartDate($package); $data = ['downloads' => $this->get('packagist.download_manager')->getDownloads($package), 'versions' => $versions, 'average' => $this->guessStatsAverage($date), 'date' => $date->format('Y-m-d')]; if ($req->getRequestFormat() === 'json') { $data['versions'] = array_map(function ($version) { /** @var Version $version */ return $version->getVersion(); }, $data['versions']); return new JsonResponse($data); } $data['package'] = $package; $expandedVersion = reset($versions); foreach ($versions as $v) { /** @var Version $v */ if (!$v->isDevelopment()) { $expandedVersion = $v; break; } } $data['expandedId'] = $expandedVersion ? $expandedVersion->getId() : false; return $data; }
/** * Update a project * * @param \Packagist\WebBundle\Entity\Package $package * @param RepositoryInterface $repository the repository instance used to update from * @param int $flags a few of the constants of this class * @param \DateTime $start */ public function update(IOInterface $io, Config $config, Package $package, RepositoryInterface $repository, $flags = 0, \DateTime $start = null) { $rfs = new RemoteFilesystem($io, $config); $blacklist = '{^symfony/symfony (2.0.[456]|dev-charset|dev-console)}i'; if (null === $start) { $start = new \DateTime(); } $pruneDate = clone $start; $pruneDate->modify('-1min'); $em = $this->doctrine->getManager(); $apc = extension_loaded('apcu'); if ($repository instanceof VcsRepository) { $cfg = $repository->getRepoConfig(); if (isset($cfg['url']) && preg_match('{\\bgithub\\.com\\b}', $cfg['url'])) { foreach ($package->getMaintainers() as $maintainer) { if (!($newGithubToken = $maintainer->getGithubToken())) { continue; } $valid = null; if ($apc) { $valid = apcu_fetch('is_token_valid_' . $maintainer->getUsernameCanonical()); } if (true !== $valid) { $context = stream_context_create(['http' => ['header' => 'User-agent: packagist-token-check']]); $rate = json_decode(@file_get_contents('https://api.github.com/rate_limit?access_token=' . $newGithubToken, false, $context), true); // invalid/outdated token, wipe it so we don't try it again if (!$rate && (strpos($http_response_header[0], '403') || strpos($http_response_header[0], '401'))) { $maintainer->setGithubToken(null); $em->flush($maintainer); continue; } } if ($apc) { apcu_store('is_token_valid_' . $maintainer->getUsernameCanonical(), true, 86400); } $io->setAuthentication('github.com', $newGithubToken, 'x-oauth-basic'); break; } } } $versions = $repository->getPackages(); usort($versions, function ($a, $b) { $aVersion = $a->getVersion(); $bVersion = $b->getVersion(); if ($aVersion === '9999999-dev' || 'dev-' === substr($aVersion, 0, 4)) { $aVersion = 'dev'; } if ($bVersion === '9999999-dev' || 'dev-' === substr($bVersion, 0, 4)) { $bVersion = 'dev'; } $aIsDev = $aVersion === 'dev' || substr($aVersion, -4) === '-dev'; $bIsDev = $bVersion === 'dev' || substr($bVersion, -4) === '-dev'; // push dev versions to the end if ($aIsDev !== $bIsDev) { return $aIsDev ? 1 : -1; } // equal versions are sorted by date if ($aVersion === $bVersion) { return $a->getReleaseDate() > $b->getReleaseDate() ? 1 : -1; } // the rest is sorted by version return version_compare($aVersion, $bVersion); }); $versionRepository = $this->doctrine->getRepository('PackagistWebBundle:Version'); if ($flags & self::DELETE_BEFORE) { foreach ($package->getVersions() as $version) { $versionRepository->remove($version); } $em->flush(); $em->refresh($package); } $lastUpdated = true; $lastProcessed = null; foreach ($versions as $version) { if ($version instanceof AliasPackage) { continue; } if (preg_match($blacklist, $version->getName() . ' ' . $version->getPrettyVersion())) { continue; } if ($lastProcessed && $lastProcessed->getVersion() === $version->getVersion()) { $io->write('Skipping version ' . $version->getPrettyVersion() . ' (duplicate of ' . $lastProcessed->getPrettyVersion() . ')', true, IOInterface::VERBOSE); continue; } $lastProcessed = $version; $lastUpdated = $this->updateInformation($package, $version, $flags); if ($lastUpdated) { $em->flush(); } } if (!$lastUpdated) { $em->flush(); } // remove outdated versions foreach ($package->getVersions() as $version) { if ($version->getUpdatedAt() < $pruneDate) { $versionRepository->remove($version); } } if (preg_match('{^(?:git://|git@|https?://)github.com[:/]([^/]+)/(.+?)(?:\\.git|/)?$}i', $package->getRepository(), $match) && $repository instanceof VcsRepository) { $this->updateGitHubInfo($rfs, $package, $match[1], $match[2], $repository); } $package->setUpdatedAt(new \DateTime()); $package->setCrawledAt(new \DateTime()); $em->flush(); if ($repository->hadInvalidBranches()) { throw new InvalidRepositoryException('Some branches contained invalid data and were discarded, it is advised to review the log and fix any issues present in branches'); } }
/** * Update a project * * @param \Packagist\WebBundle\Entity\Package $package * @param RepositoryInterface $repository the repository instance used to update from * @param int $flags a few of the constants of this class * @param \DateTime $start */ public function update(IOInterface $io, Config $config, Package $package, RepositoryInterface $repository, $flags = 0, \DateTime $start = null) { $rfs = new RemoteFilesystem($io, $config); $blacklist = '{^symfony/symfony (2.0.[456]|dev-charset|dev-console)}i'; if (null === $start) { $start = new \DateTime(); } $pruneDate = clone $start; $pruneDate->modify('-1min'); $versions = $repository->getPackages(); $em = $this->doctrine->getManager(); usort($versions, function ($a, $b) { $aVersion = $a->getVersion(); $bVersion = $b->getVersion(); if ($aVersion === '9999999-dev' || 'dev-' === substr($aVersion, 0, 4)) { $aVersion = 'dev'; } if ($bVersion === '9999999-dev' || 'dev-' === substr($bVersion, 0, 4)) { $bVersion = 'dev'; } $aIsDev = $aVersion === 'dev' || substr($aVersion, -4) === '-dev'; $bIsDev = $bVersion === 'dev' || substr($bVersion, -4) === '-dev'; // push dev versions to the end if ($aIsDev !== $bIsDev) { return $aIsDev ? 1 : -1; } // equal versions are sorted by date if ($aVersion === $bVersion) { return $a->getReleaseDate() > $b->getReleaseDate() ? 1 : -1; } // the rest is sorted by version return version_compare($aVersion, $bVersion); }); $versionRepository = $this->doctrine->getRepository('PackagistWebBundle:Version'); if ($flags & self::DELETE_BEFORE) { foreach ($package->getVersions() as $version) { $versionRepository->remove($version); } $em->flush(); $em->refresh($package); } $lastUpdated = true; foreach ($versions as $version) { if ($version instanceof AliasPackage) { continue; } if (preg_match($blacklist, $version->getName() . ' ' . $version->getPrettyVersion())) { continue; } $lastUpdated = $this->updateInformation($package, $version, $flags); if ($lastUpdated) { $em->flush(); } } if (!$lastUpdated) { $em->flush(); } // remove outdated versions foreach ($package->getVersions() as $version) { if ($version->getUpdatedAt() < $pruneDate) { $versionRepository->remove($version); } } if (preg_match('{^(?:git://|git@|https?://)github.com[:/]([^/]+)/(.+?)(?:\\.git|/)?$}i', $package->getRepository(), $match)) { $this->updateGitHubInfo($rfs, $package, $match[1], $match[2]); } $package->setUpdatedAt(new \DateTime()); $package->setCrawledAt(new \DateTime()); $em->flush(); if ($repository->hadInvalidBranches()) { throw new InvalidRepositoryException('Some branches contained invalid data and were discarded, it is advised to review the log and fix any issues present in branches'); } }
private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package, $redis) { $document->setField('id', $package->getId()); $document->setField('name', $package->getName()); $document->setField('description', $package->getDescription()); $document->setField('type', $package->getType()); $document->setField('trendiness', $redis->zscore('downloads:trending', $package->getId())); $document->setField('repository', $package->getRepository()); if ($package->isAbandoned()) { $document->setField('abandoned', 1); $document->setField('replacementPackage', $package->getReplacementPackage() ?: ''); } else { $document->setField('abandoned', 0); $document->setField('replacementPackage', ''); } $tags = array(); foreach ($package->getVersions() as $version) { foreach ($version->getTags() as $tag) { $tags[mb_strtolower($tag->getName(), 'UTF-8')] = true; } } $document->setField('tags', array_keys($tags)); }
private function updateInformation(Package $package, PackageInterface $data, $flags) { $em = $this->doctrine->getEntityManager(); $version = new Version(); $version->setNormalizedVersion($data->getVersion()); // check if we have that version yet foreach ($package->getVersions() as $existingVersion) { if ($existingVersion->getNormalizedVersion() === $version->getNormalizedVersion()) { if ($existingVersion->getDevelopment() || $flags & self::UPDATE_TAGS) { $version = $existingVersion; break; } // mark it updated to avoid it being pruned $existingVersion->setUpdatedAt(new \DateTime()); return; } } $version->setName($package->getName()); $version->setVersion($data->getPrettyVersion()); $version->setDevelopment($data->isDev()); $em->persist($version); $version->setDescription($data->getDescription()); $package->setDescription($data->getDescription()); $version->setHomepage($data->getHomepage()); $version->setLicense($data->getLicense() ?: array()); $version->setPackage($package); $version->setUpdatedAt(new \DateTime()); $version->setReleasedAt($data->getReleaseDate()); if ($data->getSourceType()) { $source['type'] = $data->getSourceType(); $source['url'] = $data->getSourceUrl(); $source['reference'] = $data->getSourceReference(); $version->setSource($source); } if ($data->getDistType()) { $dist['type'] = $data->getDistType(); $dist['url'] = $data->getDistUrl(); $dist['reference'] = $data->getDistReference(); $dist['shasum'] = $data->getDistSha1Checksum(); $version->setDist($dist); } if ($data->getType()) { $version->setType($data->getType()); if ($data->getType() && $data->getType() !== $package->getType()) { $package->setType($data->getType()); } } $version->setTargetDir($data->getTargetDir()); $version->setAutoload($data->getAutoload()); $version->setExtra($data->getExtra()); $version->setBinaries($data->getBinaries()); $version->setIncludePaths($data->getIncludePaths()); $version->setSupport($data->getSupport()); $version->getTags()->clear(); if ($data->getKeywords()) { foreach ($data->getKeywords() as $keyword) { $tag = Tag::getByName($em, $keyword, true); if (!$version->getTags()->contains($tag)) { $version->addTag($tag); } } } $authorRepository = $this->doctrine->getRepository('PackagistWebBundle:Author'); $version->getAuthors()->clear(); if ($data->getAuthors()) { foreach ($data->getAuthors() as $authorData) { $author = null; // skip authors with no information if (empty($authorData['email']) && empty($authorData['name'])) { continue; } if (!empty($authorData['email'])) { $author = $authorRepository->findOneByEmail($authorData['email']); } if (!$author && !empty($authorData['homepage'])) { $author = $authorRepository->findOneBy(array('name' => $authorData['name'], 'homepage' => $authorData['homepage'])); } if (!$author && !empty($authorData['name'])) { $author = $authorRepository->findOneByNameAndPackage($authorData['name'], $package); } if (!$author) { $author = new Author(); $em->persist($author); } foreach (array('email', 'name', 'homepage', 'role') as $field) { if (isset($authorData[$field])) { $author->{'set' . $field}($authorData[$field]); } } $author->setUpdatedAt(new \DateTime()); if (!$version->getAuthors()->contains($author)) { $version->addAuthor($author); } if (!$author->getVersions()->contains($version)) { $author->addVersion($version); } } } // handle links foreach ($this->supportedLinkTypes as $linkType => $opts) { $links = array(); foreach ($data->{$opts['method']}() as $link) { $constraint = $link->getPrettyConstraint(); if (false !== strpos($constraint, '~')) { $constraint = str_replace(array('[', ']'), '', $link->getConstraint()); $constraint = preg_replace('{(\\d\\.\\d)(\\.0)+(?=$|,|-)}', '$1', $constraint); $constraint = preg_replace('{([><=,]) }', '$1', $constraint); $constraint = preg_replace('{(<[0-9.]+)-dev}', '$1', $constraint); } if (false !== strpos($constraint, ',') && false !== strpos($constraint, '@')) { $constraint = preg_replace_callback('{([><]=?\\s*[^@]+?)@([a-z]+)}i', function ($matches) { if ($matches[2] === 'stable') { return $matches[1]; } return $matches[1] . '-' . $matches[2]; }, $constraint); } $links[$link->getTarget()] = $constraint; } foreach ($version->{'get' . $linkType}() as $link) { // clear links that have changed/disappeared (for updates) if (!isset($links[$link->getPackageName()]) || $links[$link->getPackageName()] !== $link->getPackageVersion()) { $version->{'get' . $linkType}()->removeElement($link); $em->remove($link); } else { // clear those that are already set unset($links[$link->getPackageName()]); } } foreach ($links as $linkPackageName => $linkPackageVersion) { $class = 'Packagist\\WebBundle\\Entity\\' . $opts['entity']; $link = new $class(); $link->setPackageName($linkPackageName); $link->setPackageVersion($linkPackageVersion); $version->{'add' . $linkType . 'Link'}($link); $link->setVersion($version); $em->persist($link); } } // handle suggests if ($suggests = $data->getSuggests()) { foreach ($version->getSuggest() as $link) { // clear links that have changed/disappeared (for updates) if (!isset($suggests[$link->getPackageName()]) || $suggests[$link->getPackageName()] !== $link->getPackageVersion()) { $version->getSuggest()->removeElement($link); $em->remove($link); } else { // clear those that are already set unset($suggests[$link->getPackageName()]); } } foreach ($suggests as $linkPackageName => $linkPackageVersion) { $link = new SuggestLink(); $link->setPackageName($linkPackageName); $link->setPackageVersion($linkPackageVersion); $version->addSuggestLink($link); $link->setVersion($version); $em->persist($link); } } if (!$package->getVersions()->contains($version)) { $package->addVersions($version); } }
private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package) { $document->setField('id', strtolower($package->getName())); $document->setField('name', $package->getName()); $document->setField('description', $package->getDescription()); $document->setField('type', $package->getType()); $tags = array(); foreach ($package->getVersions() as $version) { foreach ($version->getTags() as $tag) { $tags[mb_strtolower($tag->getName(), 'UTF-8')] = true; } } $document->setField('tags', array_keys($tags)); }