It can be one of two directions:
- Backup: Package up data on an environment and store it in a local file
- Restore: Transfer data from a local file into an environment, extract assets and/or restore a database
The choice of database and/or assets is represented in the "Mode".
There's always one file archive involved (stored as the has_one "ArchiveFile") on the local Deploynaut environment.
Each transfer is executed by a Resque job, so the model also contains
a reference to a Resque token (which might still be in progress).
The "Environment" points to the source or target involved.
/** * * @param string $status * @global array $databaseConfig */ protected function updateStatus($status) { global $databaseConfig; DB::connect($databaseConfig); $env = DNDataTransfer::get()->byID($this->args['dataTransferID']); $env->Status = $status; $env->write(); }
public function testGenerateFileName() { $project1 = $this->objFromFixture('DNProject', 'project1'); $project1uatEnv = $this->objFromFixture('DNEnvironment', 'project1-uat'); $dataTransfer = new DNDataTransfer(); $dataTransfer->Direction = 'get'; $dataTransfer->Mode = 'all'; $dataTransfer->write(); $archive = new DNDataArchive(); $archive->OriginalEnvironmentID = $project1uatEnv->ID; $archive->write(); $filename = $archive->generateFilename($dataTransfer); $this->assertNotNull($filename); $this->assertContains('project_1', $filename); $this->assertContains('uat', $filename); $this->assertContains('all', $filename); }
public function perform() { echo "[-] DeployJob starting" . PHP_EOL; $log = new DeploynautLogFile($this->args['logfile']); $deployment = DNDeployment::get()->byID($this->args['deploymentID']); $environment = $deployment->Environment(); $currentBuild = $environment->CurrentBuild(); $project = $environment->Project(); $backupDataTransfer = null; $backupMode = !empty($this->args['backup_mode']) ? $this->args['backup_mode'] : 'db'; // Perform pre-deploy backup here if required. Note that the backup is done here within // the deploy job, so that the order of backup is done before deployment, and so it // doesn't tie up another worker. It also puts the backup output into // the same log as the deployment so there is visibility on what is going on. // Note that the code has to be present for a backup to be performed, so the first // deploy onto a clean environment will not be performing any backup regardless of // whether the predeploy_backup option was passed or not. // Sometimes predeploy_backup comes through as string false from the frontend. if (!empty($this->args['predeploy_backup']) && $this->args['predeploy_backup'] !== 'false' && !empty($currentBuild)) { $backupDataTransfer = DNDataTransfer::create(); $backupDataTransfer->EnvironmentID = $environment->ID; $backupDataTransfer->Direction = 'get'; $backupDataTransfer->Mode = $backupMode; $backupDataTransfer->ResqueToken = $deployment->ResqueToken; $backupDataTransfer->AuthorID = $deployment->DeployerID; $backupDataTransfer->write(); $deployment->BackupDataTransferID = $backupDataTransfer->ID; $deployment->write(); } try { // Disallow concurrent deployments (don't rely on queuing implementation to restrict this) // Only consider deployments started in the last 30 minutes (older jobs probably got stuck) $runningDeployments = $environment->runningDeployments()->exclude('ID', $this->args['deploymentID']); if ($runningDeployments->count()) { $runningDeployment = $runningDeployments->first(); $message = sprintf('Error: another deployment is in progress (started at %s by %s)', $runningDeployment->dbObject('Created')->Nice(), $runningDeployment->Deployer()->Title); $log->write($message); throw new \RuntimeException($message); } $this->performBackup($backupDataTransfer, $log); $environment->Backend()->deploy($environment, $log, $project, $this->args); } catch (Exception $e) { // DeploynautJob will automatically trigger onFailure. echo "[-] DeployJob failed" . PHP_EOL; throw $e; } $this->updateStatus(DNDeployment::TR_COMPLETE); echo "[-] DeployJob finished" . PHP_EOL; }
/** * Makes the dummy deployment step * * @return Pipeline */ public function getDummyPipeline($restoreDB = true) { // Get default backups $previous = DNDeployment::create(); $previous->write(); $current = DNDeployment::create(); $current->write(); $snapshot = DNDataTransfer::create(); $snapshot->write(); // Setup default pipeline $pipeline = $this->objFromFixture('Pipeline', 'testpipesmoketest'); $pipeline->Config = serialize(array('RollbackStep1' => array('Class' => 'RollbackStep', 'RestoreDB' => $restoreDB, 'MaxDuration' => '3600'))); $pipeline->PreviousDeploymentID = $previous->ID; $pipeline->CurrentDeploymentID = $current->ID; $pipeline->PreviousSnapshotID = $snapshot->ID; $pipeline->write(); return $pipeline; }
/** * @return PaginatedList */ public function DataTransferLogs() { $project = $this->getCurrentProject(); $transfers = DNDataTransfer::get()->filterByCallback(function ($record) use($project) { return $record->Environment()->Project()->ID == $project->ID && ($record->Environment()->canRestore() || $record->Environment()->canBackup() || $record->Environment()->canUploadArchive() || $record->Environment()->canDownloadArchive()); }); return new PaginatedList($transfers->sort("Created", "DESC"), $this->request); }
/** * Create a snapshot of the db and store the ID on the Pipline * * @return bool True if success */ protected function createSnapshot() { // Mark self as creating a snapshot $this->Status = 'Started'; $this->Doing = 'Snapshot'; $this->log("{$this->Title} creating snapshot of database"); $this->write(); // Skip deployment for dry run if ($this->Pipeline()->DryRun) { $this->log("[Skipped] Create DNDataTransfer backup"); return true; } // create a snapshot $pipeline = $this->Pipeline(); $job = DNDataTransfer::create(); $job->EnvironmentID = $pipeline->EnvironmentID; $job->Direction = 'get'; $job->Mode = 'db'; $job->DataArchiveID = null; $job->AuthorID = $pipeline->AuthorID; $job->write(); $job->start(); $pipeline->PreviousSnapshotID = $job->ID; $pipeline->write(); return true; }
/** * Attach an sspak file path to this archive and associate the transfer. * Does the job of creating a {@link File} record, and setting correct paths into the assets directory. * * @param string $sspakFilepath * @param DNDataTransfer $dataTransfer * @return bool */ public function attachFile($sspakFilepath, DNDataTransfer $dataTransfer) { $sspakFilepath = ltrim(str_replace(array(ASSETS_PATH, realpath(ASSETS_PATH)), '', $sspakFilepath), DIRECTORY_SEPARATOR); $folder = Folder::find_or_make(dirname($sspakFilepath)); $file = new File(); $file->Name = basename($sspakFilepath); $file->Filename = $sspakFilepath; $file->ParentID = $folder->ID; $file->write(); // "Status" will be updated by the job execution $dataTransfer->write(); // Get file hash to ensure consistency. // Only do this when first associating the file since hashing large files is expensive. // Note that with CapistranoDeploymentBackend the file won't be available yet, as it // gets put in place immediately after this method gets called. In which case, it will // be hashed in setArchiveFromFiles() if (file_exists($file->FullPath)) { $this->ArchiveFileHash = md5_file($file->FullPath); } $this->ArchiveFileID = $file->ID; $this->DataTransfers()->add($dataTransfer); $this->write(); return true; }
/** * Extracts a *.sspak file referenced through the passed in $dataTransfer * and pushes it to the environment referenced in $dataTransfer. * * @param string $workingDir Directory for the unpacked files. * @param DNDataTransfer $dataTransfer * @param DeploynautLogFile $log */ protected function dataTransferRestore($workingDir, DNDataTransfer $dataTransfer, DeploynautLogFile $log) { $environment = $dataTransfer->Environment(); $name = $environment->getFullName(); // Rollback cleanup. $self = $this; $cleanupFn = function () use($self, $workingDir, $environment, $log) { // Rebuild makes sense even if failed - maybe we can at least partly recover. $self->rebuild($environment, $log); $process = new Process(sprintf('rm -rf %s', escapeshellarg($workingDir))); $process->run(); }; // Restore database into target environment if (in_array($dataTransfer->Mode, array('all', 'db'))) { $log->write(sprintf('Restore of database to "%s" started', $name)); $args = array('data_path' => $workingDir . DIRECTORY_SEPARATOR . 'database.sql'); $command = $this->getCommand('data:pushdb', 'db', $environment, $args, $log); $command->run(function ($type, $buffer) use($log) { $log->write($buffer); }); if (!$command->isSuccessful()) { $cleanupFn(); $log->write(sprintf('Restore of database to "%s" failed: %s', $name, $command->getErrorOutput())); $this->extend('dataTransferFailure', $environment, $log); throw new RuntimeException($command->getErrorOutput()); } $log->write(sprintf('Restore of database to "%s" done', $name)); } // Restore assets into target environment if (in_array($dataTransfer->Mode, array('all', 'assets'))) { $log->write(sprintf('Restore of assets to "%s" started', $name)); $args = array('data_path' => $workingDir . DIRECTORY_SEPARATOR . 'assets'); $command = $this->getCommand('data:pushassets', 'web', $environment, $args, $log); $command->run(function ($type, $buffer) use($log) { $log->write($buffer); }); if (!$command->isSuccessful()) { $cleanupFn(); $log->write(sprintf('Restore of assets to "%s" failed: %s', $name, $command->getErrorOutput())); $this->extend('dataTransferFailure', $environment, $log); throw new RuntimeException($command->getErrorOutput()); } $log->write(sprintf('Restore of assets to "%s" done', $name)); } $log->write('Rebuilding and cleaning up'); $cleanupFn(); }
/** * Extracts a *.sspak file referenced through the passed in $dataTransfer * and pushes it to the environment referenced in $dataTransfer. * * @param DNDataTransfer $dataTransfer * @param DeploynautLogFile $log */ protected function dataTransferRestore(DNDataTransfer $dataTransfer, DeploynautLogFile $log) { $environmentObj = $dataTransfer->Environment(); $project = $environmentObj->Project(); $projectName = $project->Name; $environmentName = $environmentObj->Name; $env = $project->getProcessEnv(); $project = DNProject::get()->filter('Name', $projectName)->first(); $name = $projectName . ':' . $environmentName; $tempPath = TEMP_FOLDER . DIRECTORY_SEPARATOR . 'deploynaut-transfer-' . $dataTransfer->ID; mkdir($tempPath, 0700, true); $self = $this; $cleanupFn = function () use($self, $tempPath, $name, $env, $log) { // Rebuild even if failed - maybe we can at least partly recover. $self->rebuild($name, $env, $log); $process = new Process('rm -rf ' . escapeshellarg($tempPath)); $process->run(); }; // Extract *.sspak to a temporary location $log->write('Extracting *.sspak file'); $sspakFilename = $dataTransfer->DataArchive()->ArchiveFile()->FullPath; $sspakCmd = sprintf('sspak extract %s %s', escapeshellarg($sspakFilename), escapeshellarg($tempPath)); $log->write($sspakCmd); $process = new Process($sspakCmd); $process->setTimeout(3600); $process->run(); if (!$process->isSuccessful()) { $log->write('Could not extract the *.sspak file: ' . $process->getErrorOutput()); $cleanupFn(); throw new RuntimeException($process->getErrorOutput()); } // TODO Validate that file actually contains the desired modes // Restore database if (in_array($dataTransfer->Mode, array('all', 'db'))) { // Upload into target environment $log->write('Restore of database to "' . $name . '" started'); $args = array('data_path' => $tempPath . DIRECTORY_SEPARATOR . 'database.sql.gz'); $command = $this->getCommand('data:pushdb', 'db', $name, $args, $env, $log); $command->run(function ($type, $buffer) use($log) { $log->write($buffer); }); if (!$command->isSuccessful()) { $cleanupFn(); $log->write('Restore of database to "' . $name . '" failed: ' . $command->getErrorOutput()); throw new RuntimeException($command->getErrorOutput()); } $log->write('Restore of database to "' . $name . '" done'); } // Restore assets if (in_array($dataTransfer->Mode, array('all', 'assets'))) { // Upload into target environment $log->write('Restore of assets to "' . $name . '" started'); // Extract assets.tar.gz into assets/ $extractCmd = sprintf('cd %s && tar xzf %s', escapeshellarg($tempPath), escapeshellarg($tempPath . DIRECTORY_SEPARATOR . 'assets.tar.gz')); $log->write($extractCmd); $process = new Process($extractCmd); $process->setTimeout(3600); $process->run(); if (!$process->isSuccessful()) { $log->write('Could not extract the assets archive'); $cleanupFn(); throw new RuntimeException($process->getErrorOutput()); } $args = array('data_path' => $tempPath . DIRECTORY_SEPARATOR . 'assets'); $command = $this->getCommand('data:pushassets', 'web', $name, $args, $env, $log); $command->run(function ($type, $buffer) use($log) { $log->write($buffer); }); if (!$command->isSuccessful()) { $cleanupFn(); $log->write('Restore of assets to "' . $name . '" failed: ' . $command->getErrorOutput()); throw new RuntimeException($command->getErrorOutput()); } $log->write('Restore of assets to "' . $name . '" done'); } $log->write('Rebuilding and cleaning up'); $cleanupFn(); }
public function run($request) { $args = $request->getVar('args'); $dryRun = $args && in_array('--dry-run', $args); $log = function ($message) { $message = sprintf('[%s] ', date('Y-m-d H:i:s')) . $message; echo $message . PHP_EOL; }; if (!Director::is_cli()) { $log('This task must be run under the command line'); return; } if ($dryRun) { $log('Running in dry-run mode. No data will be deleted'); } $count = 0; foreach (DNEnvironment::get() as $environment) { $project = $environment->Project(); if (!$project || !$project->exists()) { $log(sprintf('Environment (ID %s, Name: %s, Created: %s) is linked to a non-existent project. Deleting', $environment->ID, $environment->Name, $environment->Created)); if (!$dryRun) { $environment->delete(); $environment->destroy(); } $count++; } } foreach (DNDeployment::get() as $deployment) { $environment = $deployment->Environment(); if (!$environment || !$environment->exists()) { $log(sprintf('Deployment (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $deployment->ID, $deployment->Created)); if (!$dryRun) { $deployment->delete(); $deployment->destroy(); } $count++; } } foreach (DNDataTransfer::get() as $transfer) { $environment = $transfer->Environment(); if (!$environment || !$environment->exists()) { $log(sprintf('Data transfer (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $transfer->ID, $transfer->Created)); if (!$dryRun) { $transfer->delete(); $transfer->destroy(); } $count++; } } foreach (DNDataArchive::get() as $archive) { $environment = $archive->Environment(); if (!$environment || !$environment->exists()) { $log(sprintf('Archive (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $archive->ID, $archive->Created)); if (!$dryRun) { $archive->delete(); $archive->destroy(); } $count++; } } foreach (DNGitFetch::get() as $fetch) { $project = $fetch->Project(); if (!$project || !$project->exists()) { $log(sprintf('Git fetch (ID %s, Created: %s) is linked to a non-existent project. Deleting', $fetch->ID, $fetch->Created)); if (!$dryRun) { $fetch->delete(); $fetch->destroy(); } $count++; } } foreach (DNPing::get() as $ping) { $environment = $ping->Environment(); if (!$environment || !$environment->exists()) { $log(sprintf('Ping (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $ping->ID, $ping->Created)); if (!$dryRun) { $ping->delete(); $ping->destroy(); } $count++; } } $log(sprintf('Finished. Processed %s records', $count)); }
/** * Attach an sspak file path to this archive and associate the transfer. * Does the job of creating a {@link File} record, and setting correct paths into the assets directory. * * @param string $sspakFilepath * @param DNDataTransfer $dataTransfer * @return bool */ public function attachFile($sspakFilepath, DNDataTransfer $dataTransfer) { $sspakFilepath = ltrim(str_replace(array(ASSETS_PATH, realpath(ASSETS_PATH)), '', $sspakFilepath), DIRECTORY_SEPARATOR); $folder = Folder::find_or_make(dirname($sspakFilepath)); $file = new File(); $file->Name = basename($sspakFilepath); $file->Filename = $sspakFilepath; $file->ParentID = $folder->ID; $file->write(); // "Status" will be updated by the job execution $dataTransfer->write(); $this->ArchiveFileID = $file->ID; $this->DataTransfers()->add($dataTransfer); $this->write(); return true; }
/** * Create a snapshot of the db and store the ID on the Pipline * * @return bool True if success */ protected function startRevertDatabase() { // Mark self as creating a snapshot $this->Status = 'Started'; $this->Doing = 'Snapshot'; $this->log("{$this->Title} reverting database from snapshot"); // Skip deployment for dry run if ($this->Pipeline()->DryRun) { $this->write(); $this->log("[Skipped] Create DNDataTransfer restore"); return true; } // Get snapshot $pipeline = $this->Pipeline(); $backup = $pipeline->PreviousSnapshot(); if (empty($backup) || !$backup->exists()) { $this->log("No database to revert for {$this->Title}"); $this->markFailed(); return false; } // Create restore job $job = DNDataTransfer::create(); $job->EnvironmentID = $pipeline->EnvironmentID; $job->Direction = 'push'; $job->Mode = 'db'; $job->DataArchiveID = $backup->DataArchiveID; $job->AuthorID = $pipeline->AuthorID; $job->EnvironmentID = $pipeline->EnvironmentID; $job->write(); $job->start(); // Save rollback $this->RollbackDatabaseID = $job->ID; $this->write(); return true; }
public function __construct($record = null, $isSingleton = false, $model = null) { // Set the fields data. if (!$record) { $record = array('ID' => 0, 'ClassName' => 'DNDataTransfer', 'RecordClassName' => 'DNDataTransfer'); } parent::__construct($record, $isSingleton, $model); $this->class = 'DNDataTransfer'; }
/** * @return PaginatedList */ public function DataTransferLogs() { $environments = $this->getCurrentProject()->Environments()->column('ID'); $transfers = DNDataTransfer::get()->filter('EnvironmentID', $environments)->filterByCallback(function ($record) { return $record->Environment()->canRestore() || $record->Environment()->canBackup() || $record->Environment()->canUploadArchive() || $record->Environment()->canDownloadArchive(); }); return new PaginatedList($transfers->sort("Created", "DESC"), $this->request); }
/** * Provide rollback-able pipeline on the verge of failing. */ public function getFailingPipeline() { // Get default backups $previous = DNDeployment::create(); $previous->SHA = '9f0a012e97715b1871n41gk30f34268u12a0029q'; $previous->write(); $current = DNDeployment::create(); $current->write(); $snapshot = DNDataTransfer::create(); $snapshot->write(); $pipeline = $this->objFromFixture('Pipeline', 'FailingPipe'); $pipeline->Config = serialize(array('RollbackStep1' => array('Class' => 'RollbackStep', 'RestoreDB' => false, 'MaxDuration' => '3600'), 'RollbackStep2' => array('Class' => 'SmokeTestPipelineStep', 'MaxDuration' => '3600'))); $pipeline->PreviousDeploymentID = $previous->ID; $pipeline->CurrentDeploymentID = $current->ID; $pipeline->PreviousSnapshotID = $snapshot->ID; $pipeline->write(); return $pipeline; }