/**
* @param string $order
* @param \App\Model\Build $build
*/
private function send($order, Build $build, $vars = [])
{
$this->logger->info('sending scheduler order', ['order' => $order, 'build_id' => $build->getId(), 'routingKey' => $build->getRoutingKey()]);
$vars = array_merge(['build_id' => $build->getId()], $vars);
$routingKey = $build->getRoutingKey();
$this->producers[$order]->publish(json_encode($vars), $routingKey);
}
public function __construct(Build $build, $cmd = null)
{
$config = ['Memory' => 256 * 1024 * 1204, 'Image' => $build->getImageName(), 'Env' => ['SYMFONY_ENV=prod']];
if (null !== $cmd) {
$config['Cmd'] = $cmd;
}
parent::__construct($config);
}
public function execute(AMQPMessage $message)
{
$logger = $this->logger;
$redis = $this->redis;
$body = json_decode($message->body, true);
if (!array_key_exists('BUILD_ID', $body['env'])) {
$logger->warn('no build information', ['container' => $body['container'], 'message' => $body]);
return true;
}
$buildId = $body['env']['BUILD_ID'];
$logger->debug('processing log fragment', ['build' => $buildId, 'container' => isset($body['container']) ? $body['container'] : null, 'keys' => array_keys($body)]);
$build = new Build();
$build->setId($buildId);
$redis->rpush($build->getLogsList(), json_encode(['type' => Build::LOG_OUTPUT, 'message' => $body['content'], 'stream' => $this->getStreamType($body['type']), 'microtime' => $body['timestamp'], 'fragment_id' => $body['fragment_id'], 'build_id' => $buildId]));
}
private function getBuildEnv(Build $build)
{
$env = ['BUILD_ID=' . $build->getId(), 'PROJECT_ID=' . $build->getProject()->getId(), 'CHANNEL=' . $build->getChannel(), 'SSH_URL=' . $build->getProject()->getGitUrl(), 'REF=' . $build->getRef(), 'HASH=' . $build->getHash(), 'IS_PULL_REQUEST=' . ($build->isPullRequest() ? 1 : 0), 'SYMFONY_ENV=prod'];
$user = $build->getProject()->getUsers()->first();
if ($user->hasProviderAccessToken('github')) {
/**
* @todo there must be a way to avoid requiring a valid access token
* I think the token is only used to avoid hitting github's
* API limit through composer, so maybe there's a way to use a
* stage1 specific token instead
*/
$env[] = 'GITHUB_ACCESS_TOKEN=' . $user->getProviderAccessToken('github');
}
return $env;
}
/**
* Schedules a build
*
* @see App\CoreBundle\EventListener\BuildBranchRelationSubscriber for automatic creation of non-existing branches
*/
public function schedule(Project $project, $ref, $hash, PayloadInterface $payload = null, $options = [])
{
$logger = $this->logger;
$logger->info('scheduling build', ['project' => $project->getId(), 'ref' => $ref, 'hash' => $hash]);
$em = $this->doctrine->getManager();
// @todo I guess this should be in a build.scheduled event listener
$alreadyRunningBuilds = $em->getRepository('Model:Build')->findPendingByRef($project, $ref);
foreach ($alreadyRunningBuilds as $build) {
// @todo instead of retrieving then updating builds to be canceled, directly issue an UPDATE
// it should avoid most race conditions
if ($build->isScheduled()) {
$logger->info('canceling same ref build', ['ref' => $ref, 'canceled_build' => $build->getId()]);
$build->setStatus(Build::STATUS_CANCELED);
$em->persist($build);
$em->flush();
} else {
$logger->info('killing same ref build', ['ref' => $ref, 'canceled_build' => $build->getId()]);
$scheduler->kill($build);
}
}
$build = new Build();
$build->setProject($project);
$build->setStatus(Build::STATUS_SCHEDULED);
$build->setRef($ref);
$build->setHash($hash);
$build->setCommitUrl(sprintf('https://github.com/%s/commit/%s', $project->getFullName(), $hash));
if (null !== $payload) {
$build->setIsPullRequest($payload->isPullRequest());
$build->setRawPayload($payload->getRawContent());
} else {
$build->setIsPullRequest(false);
}
if (isset($options['force_local_build_yml']) && $options['force_local_build_yml']) {
$build->setForceLocalBuildYml(true);
}
$builderHost = null;
$logger->info('electing builder', ['builder_host_allow' => $this->getOption('builder_host_allow')]);
if (count($builderHostAllow = $this->getOption('builder_host_allow')) > 0) {
$builderHost = $builderHostAllow[array_rand($builderHostAllow)];
}
$build->setBuilderHost($builderHost);
/**
* @todo move this outside, it belongs in a controller
* this will allow to remove the $options argument
*/
$em->persist($build);
$em->flush();
$this->logger->info('sending build order', ['build' => $build->getId(), 'builder_host' => $builderHost]);
$this->buildProducer->publish(json_encode(['build_id' => $build->getId()]), $build->getRoutingKey());
$message = $this->messageFactory->createBuildScheduled($build);
$this->websocketProducer->publish($message);
return $build;
}
/**
* @param Build $build
*
* @return Docker\Container
*/
public function run(Build $build, $timeout = null)
{
$producer = $this->websocketProducer;
$logger = $this->logger;
$docker = $this->docker;
$em = $this->doctrine->getManager();
$publish = function ($content) use($build, $producer) {
$message = new BuildMessage($build, $content);
$producer->publish((string) $message);
};
// $publish(' build started ('.date('r').')');
if ($this->getOption('dummy')) {
$logger->info('dummy build, sleeping', ['duration' => $this->getOption('dummy_duration')]);
sleep($this->getOption('dummy_duration'));
$build->setPort(42);
return true;
}
$project = $build->getProject();
$logger->info('starting build', ['build' => $build->getId(), 'project' => $project->getFullName(), 'project_id' => $project->getId(), 'ref' => $build->getRef(), 'hash' => $build->getHash(), 'timeout' => $timeout, 'force_local_build_yml' => $build->getForceLocalBuildYml()]);
/** Generate build script using Yuhao */
$logger->info('generating build script');
// $publish(' generating build script'.PHP_EOL);
$builder = $project->getDockerContextBuilder();
$builder->from('stage1/yuhao');
$docker->build($builder->getContext(), $build->getImageName('yuhao'), false, true, true);
$prepareContainer = new PrepareContainer($build);
// @todo move inside PrepareContainer
if ($build->getForceLocalBuildYml()) {
$prepareContainer->addEnv(['FORCE_LOCAL_STAGE1_YML=1']);
}
$manager = $docker->getContainerManager();
$manager->create($prepareContainer)->start($prepareContainer)->wait($prepareContainer, $timeout);
if ($prepareContainer->getExitCode() != 0) {
$exitCode = $prepareContainer->getExitCode();
if (isset(Process::$exitCodes[$exitCode])) {
$exitCodeLabel = Process::$exitCodes[$exitCode];
} else {
$exitCodeLabel = '';
}
$message = sprintf('failed to generate build scripts (exit code %d (%s))', $exitCode, $exitCodeLabel);
$publish($message . PHP_EOL);
$logger->error($message, ['build' => $build->getId(), 'container' => $prepareContainer->getId(), 'container_name' => $prepareContainer->getName(), 'exit_code' => $exitCode, 'exit_code_label' => $exitCodeLabel]);
$docker->commit($prepareContainer, ['repo' => $build->getImageName('yuhao'), 'tag' => 'failed']);
throw new Exception($message, $prepareContainer->getExitCode());
}
// @todo remove yuhao container
// $manager->remove($prepareContainer); => 406 ?!
$logger->info('yuhao finished executing, retrieving build script', ['build' => $build->getId()]);
$output = '';
$manager->attach($prepareContainer, true, false, false, true, true)->readAttach(function ($type, $chunk) use(&$output) {
$output .= $chunk;
});
$logger->info('got response from yuhao', ['build' => $build->getId(), 'response' => $output, 'parsed_response' => json_decode($output, true)]);
$script = BuildScript::fromJson($output);
$script->setBuild($build);
$options = $project->getDefaultBuildOptions();
$options = $options->resolve($script->getConfig());
$build->setOptions($options);
$logger->info('resolved options', ['options' => $options]);
$strategy = array_key_exists('path', $options['dockerfile']) ? new DockerfileStrategy($logger, $docker, $em, $this->websocketProducer, $this->redis, $this->options) : new DefaultStrategy($logger, $docker, $em, $this->websocketProducer, $this->options);
$logger->info('elected strategy', ['strategy' => get_class($strategy)]);
$em->persist($build);
$em->persist($script);
$em->flush();
$logger->info('starting build');
$strategy->build($build, $script, $timeout);
$logger->info('finished build');
$em->persist($build);
$em->persist($script);
$em->flush();
/**
* Launch App container
*/
$logger->info('starting app container', ['project' => $build->getProject()->getFullName(), 'image' => $build->getImageName()]);
$ports = new PortCollection(80);
# @todo DefaultStrategy containers should have an entrypoint
# so we don't need to provide an actual command
$appContainer = new AppContainer($build, $strategy->getCmd());
$appContainer->addEnv($build->getProject()->getContainerEnv());
$appContainer->setExposedPorts($ports);
if ($build->getForceLocalBuildYml()) {
$appContainer->addEnv(['FORCE_LOCAL_STAGE1_YML=1']);
}
try {
$manager->create($appContainer)->start($appContainer, ['PortBindings' => $ports->toSpec()]);
$logger->info('running app container', ['build' => $build->getId(), 'container' => $appContainer->getId()]);
} catch (\Docker\Exception\UnexpectedStatusCodeException $e) {
if ($e->getCode() === 404) {
throw new \RuntimeException('Could not start app container (' . $e->getMessage() . ')', 404, $e);
}
throw $e;
}
// $publish(' build finished ('.date('r').')'.PHP_EOL);
return $appContainer;
}
public function build(Build $build, BuildScript $script, $timeout)
{
$logger = $this->logger;
$docker = $this->docker;
$objectManager = $this->objectManager;
$websocketProducer = $this->websocketProducer;
$publish = function ($content) use($build, $websocketProducer) {
$message = new BuildMessage($build, $content);
$websocketProducer->publish((string) $message);
};
$options = $build->getOptions();
$project = $build->getProject();
/**
* Launch actual build
*/
$logger->info('building base build container', ['build' => $build->getId(), 'image_name' => $build->getImageName()]);
// $publish(' building base container'.PHP_EOL);
$baseImage = strpos($options['image'], 'stage1/') !== 0 ? 'stage1/' . $options['image'] : $options['image'];
$builder = $project->getDockerContextBuilder();
$builder->add('/usr/local/bin/yuhao_build', $script->getBuildScript());
$builder->add('/usr/local/bin/yuhao_run', $script->getRunScript());
$builder->run('chmod -R +x /usr/local/bin/');
$builder->from($baseImage);
$response = $docker->build($builder->getContext(), $build->getImageName(), false, true, true);
$buildContainer = new BuildContainer($build);
$buildContainer->addEnv($options['env']);
$script->setRuntimeEnv($buildContainer->getEnv());
if ($build->getForceLocalBuildYml()) {
$buildContainer->addEnv(['FORCE_LOCAL_STAGE1_YML=1']);
}
$manager = $docker->getContainerManager();
$logger->info('starting actual build', ['build' => $build->getId(), 'timeout' => $timeout]);
// $publish(' starting actual build'.PHP_EOL);
$hostConfig = [];
if ($this->getOption('composer_enable_global_cache')) {
$logger->info('enabling composer global cache', ['build' => $build->getId()]);
$hostConfig['Binds'] = [$this->getOption('composer_cache_path') . '/global:/.composer/cache'];
} elseif ($this->getOption('composer_enable_project_cache')) {
$cachePath = $this->getOption('composer_cache_path') . '/' . $project->getFullName();
$logger->info('enabling composer project cache', ['build' => $build->getId(), 'project' => $project->getFullName(), 'cache_path' => $cachePath]);
if (!is_dir($cachePath)) {
mkdir($cachePath, 0777, true);
}
$hostConfig['Binds'] = [realpath($cachePath) . ':/.composer/cache'];
}
$manager->create($buildContainer);
$build->setContainer($buildContainer);
$manager->start($buildContainer, $hostConfig);
$manager->wait($buildContainer, $timeout);
if ($buildContainer->getExitCode() !== 0) {
$exitCode = $buildContainer->getExitCode();
$exitCodeLabel = isset(Process::$exitCodes[$exitCode]) ? Process::$exitCodes[$exitCode] : '';
$message = sprintf('build container stopped with exit code %d (%s)', $exitCode, $exitCodeLabel);
$logger->error($message, ['build' => $build->getId(), 'container' => $buildContainer->getId(), 'container_name' => $buildContainer->getName(), 'exit_code' => $exitCode, 'exit_code_label' => $exitCodeLabel]);
$docker->commit($buildContainer, ['repo' => $build->getImageName(), 'tag' => 'failed']);
throw new Exception($message, $buildContainer->getExitCode());
}
/**
* Build successful!
*
* @todo the commit can timeout for no obvious reason, while actually committing
* catch the timeout and check if the image has been committed
* if yes, proceed
* if not, retry (3 times ?)
*/
$logger->info('build successful, committing', ['build' => $build->getId(), 'container' => $buildContainer->getId()]);
// $publish(' committing app container'.PHP_EOL);
$docker->commit($buildContainer, ['repo' => $build->getImageName()]);
$logger->info('removing build container', ['build' => $build->getId(), 'container' => $buildContainer->getId()]);
// $publish(' removing build container'.PHP_EOL);
$manager->remove($buildContainer);
}
/**
* @param Build $build
*
* @return null|App\Model\Build
*/
public function findPreviousDemoBuild(Build $build)
{
$query = $this->createQueryBuilder('b')->select()->where('b.host = ?1')->andWhere('b.status IN(?2)')->setParameters([1 => $build->getHost(), 2 => [Build::STATUS_RUNNING, Build::STATUS_OBSOLETE]])->setMaxResults(1)->orderBy('b.createdAt', 'DESC')->getQuery();
try {
return $query->getSingleResult();
} catch (NoResultException $e) {
return null;
}
}
/**
* @param Project $project
* @param Build $build
* @param string $status
*/
public function setCommitStatus(Project $project, Build $build, $status)
{
$client = $this->configureClientForProject($project);
$client->setDefaultOption('headers/Accept', 'application/vnd.github.she-hulk-preview+json');
$request = $client->post(['/repos/' . $project->getFullName() . '/statuses/{sha}', ['sha' => $build->getHash()]]);
$request->setBody(json_encode(['state' => 'success', 'target_url' => $build->getUrl(), 'description' => 'Stage1 instance ready', 'context' => 'stage1']));
$this->logger->info('sending commit status', ['build' => $build->getId(), 'project' => $project->getGithubFullNAme(), 'sha' => $build->getHash()]);
try {
$request->send();
} catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
$this->logger->error('error sending commit status', ['exception_class' => get_class($e), 'exception_message' => $e->getMessage(), 'url' => $e->getRequest()->getUrl(), 'response' => (string) $e->getResponse()->getBody()]);
}
}
protected function setCurrentBuild(Build $build)
{
$this->get('request')->attributes->set('current_build_id', (int) $build->getId());
$this->SetCurrentProject($build->getProject());
}
public function __construct(Build $build)
{
parent::__construct(['Image' => $build->getImageName('yuhao'), 'Cmd' => ['yuhao.sh'], 'Env' => ['SSH_URL=' . $build->getProject()->getGitUrl(), 'REF=' . $build->getRef(), 'IS_PULL_REQUEST=' . ($build->isPullRequest() ? 1 : 0)]]);
}
public function build(Build $build, BuildScript $script, $timeout)
{
$logger = $this->logger;
$docker = $this->docker;
$websocketProducer = $this->websocketProducer;
$redis = $this->redis;
$publish = function ($content) use($build, $websocketProducer, $redis) {
static $fragment = 0;
$message = new BuildMessage($build, $content);
$websocketProducer->publish((string) $message);
$redis->rpush($build->getLogsList(), json_encode(['type' => Build::LOG_OUTPUT, 'message' => $content, 'stream' => 'stdout', 'microtime' => microtime(true), 'fragment_id' => $fragment++, 'build_id' => $build->getId()]));
};
$project = $build->getProject();
$options = $build->getOptions();
$workdir = sys_get_temp_dir() . '/stage1/workdir/' . $build->getId();
if (is_dir($workdir)) {
$fs = new Filesystem();
$fs->remove($workdir);
}
mkdir($workdir, 0777, true);
$logger->info('using workdir', ['workdir' => $workdir]);
mkdir($workdir . '/ssh', 0755, true);
$project->dumpSshKeys($workdir . '/ssh', 'root');
file_put_contents($workdir . '/ssh/config', $project->getSshConfig($workdir . '/ssh'));
file_put_contents($workdir . '/git_ssh', "#!/bin/bash\nexec /usr/bin/ssh -F {$workdir}/ssh/config \"\$@\"");
chmod($workdir . '/git_ssh', 0777);
$GIT_SSH = $workdir . '/git_ssh';
if ($build->isPullRequest()) {
$clone = ProcessBuilder::create(['git', 'clone', '--quiet', '--depth', '1', $project->getGitUrl(), $workdir . '/source'])->setEnv('GIT_SSH', $GIT_SSH)->getProcess();
$commandLine = $clone->getCommandLine();
$logger->info('cloning repository', ['command_line' => $commandLine]);
$publish('$ ' . substr($commandLine, 0, strrpos($commandLine, ' ')) . PHP_EOL);
$clone->run();
$fetch = ProcessBuilder::create(['git', 'fetch', '--quiet', 'origin', 'refs/' . $build->getRef()])->setEnv('GIT_SSH', $GIT_SSH)->setWorkingDirectory($workdir . '/source')->getProcess();
$logger->info('fecthing pull request', ['command_line' => $fetch->getCommandLine()]);
$publish('$ ' . $fetch->getCommandLine() . PHP_EOL);
$fetch->run();
$checkout = ProcessBuilder::create(['git', 'checkout', '--quiet', '-b', 'pull_request', 'FETCH_HEAD'])->setEnv('GIT_SSH', $GIT_SSH)->setWorkingDirectory($workdir . '/source')->getProcess();
$logger->info('checkouting pull request', ['command_line' => $checkout->getCommandLine()]);
$publish('$ ' . $checkout->getCommandLine() . PHP_EOL);
$checkout->run();
} else {
$clone = ProcessBuilder::create(['git', 'clone', '--quiet', '--depth', '1', '--branch', $build->getRef(), $project->getGitUrl(), $workdir . '/source'])->setEnv('GIT_SSH', $GIT_SSH)->getProcess();
$commandLine = $clone->getCommandLine();
$logger->info('cloning repository', ['command_line' => $commandLine]);
$publish('$ ' . substr($commandLine, 0, strrpos($commandLine, ' ')) . PHP_EOL);
$clone->run();
}
$contextPath = $workdir . '/source/' . $options['dockerfile']['path'];
$logger->info('creating docker build context from path', ['context_path' => $contextPath]);
$context = new Context($contextPath);
$logger->info('starting actual build', ['build' => $build->getId(), 'timeout' => $timeout]);
$response = $docker->build($context, $build->getImageName(), false, false, true, false);
$error = false;
$response->read(function ($output) use($logger, $response, $publish, &$error) {
if ($response->headers->get('content-type') === 'application/json') {
$output = json_decode($output, true);
$logger->info('got data chunk', ['output' => $output]);
if (isset($output['stream'])) {
$publish($output['stream']);
}
} else {
$message = $output;
}
});
}
