private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package)
{
$document->id = $package->getId();
$document->name = $package->getName();
$document->description = $package->getDescription();
$tags = array();
foreach ($package->getVersions() as $version) {
foreach ($version->getTags() as $tag) {
$tags[mb_strtolower($tag->getName(), 'UTF-8')] = true;
}
}
$document->tags = array_keys($tags);
}
/**
* @Route(
* "/packages/{name}/stats.{_format}",
* name="view_package_stats",
* requirements={"name"="[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?", "_format"="(json)"},
* defaults={"_format"="html"}
* )
* @Template()
*/
public function statsAction(Request $req, Package $package)
{
$versions = $package->getVersions()->toArray();
usort($versions, Package::class . '::sortVersions');
$date = $this->guessStatsStartDate($package);
$data = ['downloads' => $this->get('packagist.download_manager')->getDownloads($package), 'versions' => $versions, 'average' => $this->guessStatsAverage($date), 'date' => $date->format('Y-m-d')];
if ($req->getRequestFormat() === 'json') {
$data['versions'] = array_map(function ($version) {
/** @var Version $version */
return $version->getVersion();
}, $data['versions']);
return new JsonResponse($data);
}
$data['package'] = $package;
$expandedVersion = reset($versions);
foreach ($versions as $v) {
/** @var Version $v */
if (!$v->isDevelopment()) {
$expandedVersion = $v;
break;
}
}
$data['expandedId'] = $expandedVersion ? $expandedVersion->getId() : false;
return $data;
}
/**
* Update a project
*
* @param \Packagist\WebBundle\Entity\Package $package
* @param RepositoryInterface $repository the repository instance used to update from
* @param int $flags a few of the constants of this class
* @param \DateTime $start
*/
public function update(IOInterface $io, Config $config, Package $package, RepositoryInterface $repository, $flags = 0, \DateTime $start = null)
{
$rfs = new RemoteFilesystem($io, $config);
$blacklist = '{^symfony/symfony (2.0.[456]|dev-charset|dev-console)}i';
if (null === $start) {
$start = new \DateTime();
}
$pruneDate = clone $start;
$pruneDate->modify('-1min');
$em = $this->doctrine->getManager();
$apc = extension_loaded('apcu');
if ($repository instanceof VcsRepository) {
$cfg = $repository->getRepoConfig();
if (isset($cfg['url']) && preg_match('{\\bgithub\\.com\\b}', $cfg['url'])) {
foreach ($package->getMaintainers() as $maintainer) {
if (!($newGithubToken = $maintainer->getGithubToken())) {
continue;
}
$valid = null;
if ($apc) {
$valid = apcu_fetch('is_token_valid_' . $maintainer->getUsernameCanonical());
}
if (true !== $valid) {
$context = stream_context_create(['http' => ['header' => 'User-agent: packagist-token-check']]);
$rate = json_decode(@file_get_contents('https://api.github.com/rate_limit?access_token=' . $newGithubToken, false, $context), true);
// invalid/outdated token, wipe it so we don't try it again
if (!$rate && (strpos($http_response_header[0], '403') || strpos($http_response_header[0], '401'))) {
$maintainer->setGithubToken(null);
$em->flush($maintainer);
continue;
}
}
if ($apc) {
apcu_store('is_token_valid_' . $maintainer->getUsernameCanonical(), true, 86400);
}
$io->setAuthentication('github.com', $newGithubToken, 'x-oauth-basic');
break;
}
}
}
$versions = $repository->getPackages();
usort($versions, function ($a, $b) {
$aVersion = $a->getVersion();
$bVersion = $b->getVersion();
if ($aVersion === '9999999-dev' || 'dev-' === substr($aVersion, 0, 4)) {
$aVersion = 'dev';
}
if ($bVersion === '9999999-dev' || 'dev-' === substr($bVersion, 0, 4)) {
$bVersion = 'dev';
}
$aIsDev = $aVersion === 'dev' || substr($aVersion, -4) === '-dev';
$bIsDev = $bVersion === 'dev' || substr($bVersion, -4) === '-dev';
// push dev versions to the end
if ($aIsDev !== $bIsDev) {
return $aIsDev ? 1 : -1;
}
// equal versions are sorted by date
if ($aVersion === $bVersion) {
return $a->getReleaseDate() > $b->getReleaseDate() ? 1 : -1;
}
// the rest is sorted by version
return version_compare($aVersion, $bVersion);
});
$versionRepository = $this->doctrine->getRepository('PackagistWebBundle:Version');
if ($flags & self::DELETE_BEFORE) {
foreach ($package->getVersions() as $version) {
$versionRepository->remove($version);
}
$em->flush();
$em->refresh($package);
}
$lastUpdated = true;
$lastProcessed = null;
foreach ($versions as $version) {
if ($version instanceof AliasPackage) {
continue;
}
if (preg_match($blacklist, $version->getName() . ' ' . $version->getPrettyVersion())) {
continue;
}
if ($lastProcessed && $lastProcessed->getVersion() === $version->getVersion()) {
$io->write('Skipping version ' . $version->getPrettyVersion() . ' (duplicate of ' . $lastProcessed->getPrettyVersion() . ')', true, IOInterface::VERBOSE);
continue;
}
$lastProcessed = $version;
$lastUpdated = $this->updateInformation($package, $version, $flags);
if ($lastUpdated) {
$em->flush();
}
}
if (!$lastUpdated) {
$em->flush();
}
// remove outdated versions
foreach ($package->getVersions() as $version) {
if ($version->getUpdatedAt() < $pruneDate) {
$versionRepository->remove($version);
}
}
if (preg_match('{^(?:git://|git@|https?://)github.com[:/]([^/]+)/(.+?)(?:\\.git|/)?$}i', $package->getRepository(), $match) && $repository instanceof VcsRepository) {
$this->updateGitHubInfo($rfs, $package, $match[1], $match[2], $repository);
}
$package->setUpdatedAt(new \DateTime());
$package->setCrawledAt(new \DateTime());
$em->flush();
if ($repository->hadInvalidBranches()) {
throw new InvalidRepositoryException('Some branches contained invalid data and were discarded, it is advised to review the log and fix any issues present in branches');
}
}
/**
* Update a project
*
* @param \Packagist\WebBundle\Entity\Package $package
* @param RepositoryInterface $repository the repository instance used to update from
* @param int $flags a few of the constants of this class
* @param \DateTime $start
*/
public function update(IOInterface $io, Config $config, Package $package, RepositoryInterface $repository, $flags = 0, \DateTime $start = null)
{
$rfs = new RemoteFilesystem($io, $config);
$blacklist = '{^symfony/symfony (2.0.[456]|dev-charset|dev-console)}i';
if (null === $start) {
$start = new \DateTime();
}
$pruneDate = clone $start;
$pruneDate->modify('-1min');
$versions = $repository->getPackages();
$em = $this->doctrine->getManager();
usort($versions, function ($a, $b) {
$aVersion = $a->getVersion();
$bVersion = $b->getVersion();
if ($aVersion === '9999999-dev' || 'dev-' === substr($aVersion, 0, 4)) {
$aVersion = 'dev';
}
if ($bVersion === '9999999-dev' || 'dev-' === substr($bVersion, 0, 4)) {
$bVersion = 'dev';
}
$aIsDev = $aVersion === 'dev' || substr($aVersion, -4) === '-dev';
$bIsDev = $bVersion === 'dev' || substr($bVersion, -4) === '-dev';
// push dev versions to the end
if ($aIsDev !== $bIsDev) {
return $aIsDev ? 1 : -1;
}
// equal versions are sorted by date
if ($aVersion === $bVersion) {
return $a->getReleaseDate() > $b->getReleaseDate() ? 1 : -1;
}
// the rest is sorted by version
return version_compare($aVersion, $bVersion);
});
$versionRepository = $this->doctrine->getRepository('PackagistWebBundle:Version');
if ($flags & self::DELETE_BEFORE) {
foreach ($package->getVersions() as $version) {
$versionRepository->remove($version);
}
$em->flush();
$em->refresh($package);
}
$lastUpdated = true;
foreach ($versions as $version) {
if ($version instanceof AliasPackage) {
continue;
}
if (preg_match($blacklist, $version->getName() . ' ' . $version->getPrettyVersion())) {
continue;
}
$lastUpdated = $this->updateInformation($package, $version, $flags);
if ($lastUpdated) {
$em->flush();
}
}
if (!$lastUpdated) {
$em->flush();
}
// remove outdated versions
foreach ($package->getVersions() as $version) {
if ($version->getUpdatedAt() < $pruneDate) {
$versionRepository->remove($version);
}
}
if (preg_match('{^(?:git://|git@|https?://)github.com[:/]([^/]+)/(.+?)(?:\\.git|/)?$}i', $package->getRepository(), $match)) {
$this->updateGitHubInfo($rfs, $package, $match[1], $match[2]);
}
$package->setUpdatedAt(new \DateTime());
$package->setCrawledAt(new \DateTime());
$em->flush();
if ($repository->hadInvalidBranches()) {
throw new InvalidRepositoryException('Some branches contained invalid data and were discarded, it is advised to review the log and fix any issues present in branches');
}
}
private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package, $redis)
{
$document->setField('id', $package->getId());
$document->setField('name', $package->getName());
$document->setField('description', $package->getDescription());
$document->setField('type', $package->getType());
$document->setField('trendiness', $redis->zscore('downloads:trending', $package->getId()));
$document->setField('repository', $package->getRepository());
if ($package->isAbandoned()) {
$document->setField('abandoned', 1);
$document->setField('replacementPackage', $package->getReplacementPackage() ?: '');
} else {
$document->setField('abandoned', 0);
$document->setField('replacementPackage', '');
}
$tags = array();
foreach ($package->getVersions() as $version) {
foreach ($version->getTags() as $tag) {
$tags[mb_strtolower($tag->getName(), 'UTF-8')] = true;
}
}
$document->setField('tags', array_keys($tags));
}
private function updateInformation(Package $package, PackageInterface $data, $flags)
{
$em = $this->doctrine->getEntityManager();
$version = new Version();
$version->setNormalizedVersion($data->getVersion());
// check if we have that version yet
foreach ($package->getVersions() as $existingVersion) {
if ($existingVersion->getNormalizedVersion() === $version->getNormalizedVersion()) {
if ($existingVersion->getDevelopment() || $flags & self::UPDATE_TAGS) {
$version = $existingVersion;
break;
}
// mark it updated to avoid it being pruned
$existingVersion->setUpdatedAt(new \DateTime());
return;
}
}
$version->setName($package->getName());
$version->setVersion($data->getPrettyVersion());
$version->setDevelopment($data->isDev());
$em->persist($version);
$version->setDescription($data->getDescription());
$package->setDescription($data->getDescription());
$version->setHomepage($data->getHomepage());
$version->setLicense($data->getLicense() ?: array());
$version->setPackage($package);
$version->setUpdatedAt(new \DateTime());
$version->setReleasedAt($data->getReleaseDate());
if ($data->getSourceType()) {
$source['type'] = $data->getSourceType();
$source['url'] = $data->getSourceUrl();
$source['reference'] = $data->getSourceReference();
$version->setSource($source);
}
if ($data->getDistType()) {
$dist['type'] = $data->getDistType();
$dist['url'] = $data->getDistUrl();
$dist['reference'] = $data->getDistReference();
$dist['shasum'] = $data->getDistSha1Checksum();
$version->setDist($dist);
}
if ($data->getType()) {
$version->setType($data->getType());
if ($data->getType() && $data->getType() !== $package->getType()) {
$package->setType($data->getType());
}
}
$version->setTargetDir($data->getTargetDir());
$version->setAutoload($data->getAutoload());
$version->setExtra($data->getExtra());
$version->setBinaries($data->getBinaries());
$version->setIncludePaths($data->getIncludePaths());
$version->setSupport($data->getSupport());
$version->getTags()->clear();
if ($data->getKeywords()) {
foreach ($data->getKeywords() as $keyword) {
$tag = Tag::getByName($em, $keyword, true);
if (!$version->getTags()->contains($tag)) {
$version->addTag($tag);
}
}
}
$authorRepository = $this->doctrine->getRepository('PackagistWebBundle:Author');
$version->getAuthors()->clear();
if ($data->getAuthors()) {
foreach ($data->getAuthors() as $authorData) {
$author = null;
// skip authors with no information
if (empty($authorData['email']) && empty($authorData['name'])) {
continue;
}
if (!empty($authorData['email'])) {
$author = $authorRepository->findOneByEmail($authorData['email']);
}
if (!$author && !empty($authorData['homepage'])) {
$author = $authorRepository->findOneBy(array('name' => $authorData['name'], 'homepage' => $authorData['homepage']));
}
if (!$author && !empty($authorData['name'])) {
$author = $authorRepository->findOneByNameAndPackage($authorData['name'], $package);
}
if (!$author) {
$author = new Author();
$em->persist($author);
}
foreach (array('email', 'name', 'homepage', 'role') as $field) {
if (isset($authorData[$field])) {
$author->{'set' . $field}($authorData[$field]);
}
}
$author->setUpdatedAt(new \DateTime());
if (!$version->getAuthors()->contains($author)) {
$version->addAuthor($author);
}
if (!$author->getVersions()->contains($version)) {
$author->addVersion($version);
}
}
}
// handle links
foreach ($this->supportedLinkTypes as $linkType => $opts) {
$links = array();
foreach ($data->{$opts['method']}() as $link) {
$constraint = $link->getPrettyConstraint();
if (false !== strpos($constraint, '~')) {
$constraint = str_replace(array('[', ']'), '', $link->getConstraint());
$constraint = preg_replace('{(\\d\\.\\d)(\\.0)+(?=$|,|-)}', '$1', $constraint);
$constraint = preg_replace('{([><=,]) }', '$1', $constraint);
$constraint = preg_replace('{(<[0-9.]+)-dev}', '$1', $constraint);
}
if (false !== strpos($constraint, ',') && false !== strpos($constraint, '@')) {
$constraint = preg_replace_callback('{([><]=?\\s*[^@]+?)@([a-z]+)}i', function ($matches) {
if ($matches[2] === 'stable') {
return $matches[1];
}
return $matches[1] . '-' . $matches[2];
}, $constraint);
}
$links[$link->getTarget()] = $constraint;
}
foreach ($version->{'get' . $linkType}() as $link) {
// clear links that have changed/disappeared (for updates)
if (!isset($links[$link->getPackageName()]) || $links[$link->getPackageName()] !== $link->getPackageVersion()) {
$version->{'get' . $linkType}()->removeElement($link);
$em->remove($link);
} else {
// clear those that are already set
unset($links[$link->getPackageName()]);
}
}
foreach ($links as $linkPackageName => $linkPackageVersion) {
$class = 'Packagist\\WebBundle\\Entity\\' . $opts['entity'];
$link = new $class();
$link->setPackageName($linkPackageName);
$link->setPackageVersion($linkPackageVersion);
$version->{'add' . $linkType . 'Link'}($link);
$link->setVersion($version);
$em->persist($link);
}
}
// handle suggests
if ($suggests = $data->getSuggests()) {
foreach ($version->getSuggest() as $link) {
// clear links that have changed/disappeared (for updates)
if (!isset($suggests[$link->getPackageName()]) || $suggests[$link->getPackageName()] !== $link->getPackageVersion()) {
$version->getSuggest()->removeElement($link);
$em->remove($link);
} else {
// clear those that are already set
unset($suggests[$link->getPackageName()]);
}
}
foreach ($suggests as $linkPackageName => $linkPackageVersion) {
$link = new SuggestLink();
$link->setPackageName($linkPackageName);
$link->setPackageVersion($linkPackageVersion);
$version->addSuggestLink($link);
$link->setVersion($version);
$em->persist($link);
}
}
if (!$package->getVersions()->contains($version)) {
$package->addVersions($version);
}
}
private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package)
{
$document->setField('id', strtolower($package->getName()));
$document->setField('name', $package->getName());
$document->setField('description', $package->getDescription());
$document->setField('type', $package->getType());
$tags = array();
foreach ($package->getVersions() as $version) {
foreach ($version->getTags() as $tag) {
$tags[mb_strtolower($tag->getName(), 'UTF-8')] = true;
}
}
$document->setField('tags', array_keys($tags));
}
