It can be one of two directions: - Backup: Package up data on an environment and store it in a local file - Restore: Transfer data from a local file into an environment, extract assets and/or restore a database The choice of database and/or assets is represented in the "Mode". There's always one file archive involved (stored as the has_one "ArchiveFile") on the local Deploynaut environment. Each transfer is executed by a Resque job, so the model also contains a reference to a Resque token (which might still be in progress). The "Environment" points to the source or target involved.
Inheritance: extends DataObject
Ejemplo n.º 1
0
 /**
  *
  * @param string $status
  * @global array $databaseConfig
  */
 protected function updateStatus($status)
 {
     global $databaseConfig;
     DB::connect($databaseConfig);
     $env = DNDataTransfer::get()->byID($this->args['dataTransferID']);
     $env->Status = $status;
     $env->write();
 }
Ejemplo n.º 2
0
 public function testGenerateFileName()
 {
     $project1 = $this->objFromFixture('DNProject', 'project1');
     $project1uatEnv = $this->objFromFixture('DNEnvironment', 'project1-uat');
     $dataTransfer = new DNDataTransfer();
     $dataTransfer->Direction = 'get';
     $dataTransfer->Mode = 'all';
     $dataTransfer->write();
     $archive = new DNDataArchive();
     $archive->OriginalEnvironmentID = $project1uatEnv->ID;
     $archive->write();
     $filename = $archive->generateFilename($dataTransfer);
     $this->assertNotNull($filename);
     $this->assertContains('project_1', $filename);
     $this->assertContains('uat', $filename);
     $this->assertContains('all', $filename);
 }
Ejemplo n.º 3
0
 public function perform()
 {
     echo "[-] DeployJob starting" . PHP_EOL;
     $log = new DeploynautLogFile($this->args['logfile']);
     $deployment = DNDeployment::get()->byID($this->args['deploymentID']);
     $environment = $deployment->Environment();
     $currentBuild = $environment->CurrentBuild();
     $project = $environment->Project();
     $backupDataTransfer = null;
     $backupMode = !empty($this->args['backup_mode']) ? $this->args['backup_mode'] : 'db';
     // Perform pre-deploy backup here if required. Note that the backup is done here within
     // the deploy job, so that the order of backup is done before deployment, and so it
     // doesn't tie up another worker. It also puts the backup output into
     // the same log as the deployment so there is visibility on what is going on.
     // Note that the code has to be present for a backup to be performed, so the first
     // deploy onto a clean environment will not be performing any backup regardless of
     // whether the predeploy_backup option was passed or not.
     // Sometimes predeploy_backup comes through as string false from the frontend.
     if (!empty($this->args['predeploy_backup']) && $this->args['predeploy_backup'] !== 'false' && !empty($currentBuild)) {
         $backupDataTransfer = DNDataTransfer::create();
         $backupDataTransfer->EnvironmentID = $environment->ID;
         $backupDataTransfer->Direction = 'get';
         $backupDataTransfer->Mode = $backupMode;
         $backupDataTransfer->ResqueToken = $deployment->ResqueToken;
         $backupDataTransfer->AuthorID = $deployment->DeployerID;
         $backupDataTransfer->write();
         $deployment->BackupDataTransferID = $backupDataTransfer->ID;
         $deployment->write();
     }
     try {
         // Disallow concurrent deployments (don't rely on queuing implementation to restrict this)
         // Only consider deployments started in the last 30 minutes (older jobs probably got stuck)
         $runningDeployments = $environment->runningDeployments()->exclude('ID', $this->args['deploymentID']);
         if ($runningDeployments->count()) {
             $runningDeployment = $runningDeployments->first();
             $message = sprintf('Error: another deployment is in progress (started at %s by %s)', $runningDeployment->dbObject('Created')->Nice(), $runningDeployment->Deployer()->Title);
             $log->write($message);
             throw new \RuntimeException($message);
         }
         $this->performBackup($backupDataTransfer, $log);
         $environment->Backend()->deploy($environment, $log, $project, $this->args);
     } catch (Exception $e) {
         // DeploynautJob will automatically trigger onFailure.
         echo "[-] DeployJob failed" . PHP_EOL;
         throw $e;
     }
     $this->updateStatus(DNDeployment::TR_COMPLETE);
     echo "[-] DeployJob finished" . PHP_EOL;
 }
 /**
  * Makes the dummy deployment step
  *
  * @return Pipeline
  */
 public function getDummyPipeline($restoreDB = true)
 {
     // Get default backups
     $previous = DNDeployment::create();
     $previous->write();
     $current = DNDeployment::create();
     $current->write();
     $snapshot = DNDataTransfer::create();
     $snapshot->write();
     // Setup default pipeline
     $pipeline = $this->objFromFixture('Pipeline', 'testpipesmoketest');
     $pipeline->Config = serialize(array('RollbackStep1' => array('Class' => 'RollbackStep', 'RestoreDB' => $restoreDB, 'MaxDuration' => '3600')));
     $pipeline->PreviousDeploymentID = $previous->ID;
     $pipeline->CurrentDeploymentID = $current->ID;
     $pipeline->PreviousSnapshotID = $snapshot->ID;
     $pipeline->write();
     return $pipeline;
 }
Ejemplo n.º 5
0
 /**
  * @return PaginatedList
  */
 public function DataTransferLogs()
 {
     $project = $this->getCurrentProject();
     $transfers = DNDataTransfer::get()->filterByCallback(function ($record) use($project) {
         return $record->Environment()->Project()->ID == $project->ID && ($record->Environment()->canRestore() || $record->Environment()->canBackup() || $record->Environment()->canUploadArchive() || $record->Environment()->canDownloadArchive());
     });
     return new PaginatedList($transfers->sort("Created", "DESC"), $this->request);
 }
Ejemplo n.º 6
0
 /**
  * Create a snapshot of the db and store the ID on the Pipline
  * 
  * @return bool True if success
  */
 protected function createSnapshot()
 {
     // Mark self as creating a snapshot
     $this->Status = 'Started';
     $this->Doing = 'Snapshot';
     $this->log("{$this->Title} creating snapshot of database");
     $this->write();
     // Skip deployment for dry run
     if ($this->Pipeline()->DryRun) {
         $this->log("[Skipped] Create DNDataTransfer backup");
         return true;
     }
     // create a snapshot
     $pipeline = $this->Pipeline();
     $job = DNDataTransfer::create();
     $job->EnvironmentID = $pipeline->EnvironmentID;
     $job->Direction = 'get';
     $job->Mode = 'db';
     $job->DataArchiveID = null;
     $job->AuthorID = $pipeline->AuthorID;
     $job->write();
     $job->start();
     $pipeline->PreviousSnapshotID = $job->ID;
     $pipeline->write();
     return true;
 }
Ejemplo n.º 7
0
 /**
  * Attach an sspak file path to this archive and associate the transfer.
  * Does the job of creating a {@link File} record, and setting correct paths into the assets directory.
  *
  * @param string $sspakFilepath
  * @param DNDataTransfer $dataTransfer
  * @return bool
  */
 public function attachFile($sspakFilepath, DNDataTransfer $dataTransfer)
 {
     $sspakFilepath = ltrim(str_replace(array(ASSETS_PATH, realpath(ASSETS_PATH)), '', $sspakFilepath), DIRECTORY_SEPARATOR);
     $folder = Folder::find_or_make(dirname($sspakFilepath));
     $file = new File();
     $file->Name = basename($sspakFilepath);
     $file->Filename = $sspakFilepath;
     $file->ParentID = $folder->ID;
     $file->write();
     // "Status" will be updated by the job execution
     $dataTransfer->write();
     // Get file hash to ensure consistency.
     // Only do this when first associating the file since hashing large files is expensive.
     // Note that with CapistranoDeploymentBackend the file won't be available yet, as it
     // gets put in place immediately after this method gets called. In which case, it will
     // be hashed in setArchiveFromFiles()
     if (file_exists($file->FullPath)) {
         $this->ArchiveFileHash = md5_file($file->FullPath);
     }
     $this->ArchiveFileID = $file->ID;
     $this->DataTransfers()->add($dataTransfer);
     $this->write();
     return true;
 }
 /**
  * Extracts a *.sspak file referenced through the passed in $dataTransfer
  * and pushes it to the environment referenced in $dataTransfer.
  *
  * @param string $workingDir Directory for the unpacked files.
  * @param DNDataTransfer $dataTransfer
  * @param DeploynautLogFile $log
  */
 protected function dataTransferRestore($workingDir, DNDataTransfer $dataTransfer, DeploynautLogFile $log)
 {
     $environment = $dataTransfer->Environment();
     $name = $environment->getFullName();
     // Rollback cleanup.
     $self = $this;
     $cleanupFn = function () use($self, $workingDir, $environment, $log) {
         // Rebuild makes sense even if failed - maybe we can at least partly recover.
         $self->rebuild($environment, $log);
         $process = new Process(sprintf('rm -rf %s', escapeshellarg($workingDir)));
         $process->run();
     };
     // Restore database into target environment
     if (in_array($dataTransfer->Mode, array('all', 'db'))) {
         $log->write(sprintf('Restore of database to "%s" started', $name));
         $args = array('data_path' => $workingDir . DIRECTORY_SEPARATOR . 'database.sql');
         $command = $this->getCommand('data:pushdb', 'db', $environment, $args, $log);
         $command->run(function ($type, $buffer) use($log) {
             $log->write($buffer);
         });
         if (!$command->isSuccessful()) {
             $cleanupFn();
             $log->write(sprintf('Restore of database to "%s" failed: %s', $name, $command->getErrorOutput()));
             $this->extend('dataTransferFailure', $environment, $log);
             throw new RuntimeException($command->getErrorOutput());
         }
         $log->write(sprintf('Restore of database to "%s" done', $name));
     }
     // Restore assets into target environment
     if (in_array($dataTransfer->Mode, array('all', 'assets'))) {
         $log->write(sprintf('Restore of assets to "%s" started', $name));
         $args = array('data_path' => $workingDir . DIRECTORY_SEPARATOR . 'assets');
         $command = $this->getCommand('data:pushassets', 'web', $environment, $args, $log);
         $command->run(function ($type, $buffer) use($log) {
             $log->write($buffer);
         });
         if (!$command->isSuccessful()) {
             $cleanupFn();
             $log->write(sprintf('Restore of assets to "%s" failed: %s', $name, $command->getErrorOutput()));
             $this->extend('dataTransferFailure', $environment, $log);
             throw new RuntimeException($command->getErrorOutput());
         }
         $log->write(sprintf('Restore of assets to "%s" done', $name));
     }
     $log->write('Rebuilding and cleaning up');
     $cleanupFn();
 }
 /**
  * Extracts a *.sspak file referenced through the passed in $dataTransfer
  * and pushes it to the environment referenced in $dataTransfer.
  *
  * @param  DNDataTransfer    $dataTransfer
  * @param  DeploynautLogFile $log
  */
 protected function dataTransferRestore(DNDataTransfer $dataTransfer, DeploynautLogFile $log)
 {
     $environmentObj = $dataTransfer->Environment();
     $project = $environmentObj->Project();
     $projectName = $project->Name;
     $environmentName = $environmentObj->Name;
     $env = $project->getProcessEnv();
     $project = DNProject::get()->filter('Name', $projectName)->first();
     $name = $projectName . ':' . $environmentName;
     $tempPath = TEMP_FOLDER . DIRECTORY_SEPARATOR . 'deploynaut-transfer-' . $dataTransfer->ID;
     mkdir($tempPath, 0700, true);
     $self = $this;
     $cleanupFn = function () use($self, $tempPath, $name, $env, $log) {
         // Rebuild even if failed - maybe we can at least partly recover.
         $self->rebuild($name, $env, $log);
         $process = new Process('rm -rf ' . escapeshellarg($tempPath));
         $process->run();
     };
     // Extract *.sspak to a temporary location
     $log->write('Extracting *.sspak file');
     $sspakFilename = $dataTransfer->DataArchive()->ArchiveFile()->FullPath;
     $sspakCmd = sprintf('sspak extract %s %s', escapeshellarg($sspakFilename), escapeshellarg($tempPath));
     $log->write($sspakCmd);
     $process = new Process($sspakCmd);
     $process->setTimeout(3600);
     $process->run();
     if (!$process->isSuccessful()) {
         $log->write('Could not extract the *.sspak file: ' . $process->getErrorOutput());
         $cleanupFn();
         throw new RuntimeException($process->getErrorOutput());
     }
     // TODO Validate that file actually contains the desired modes
     // Restore database
     if (in_array($dataTransfer->Mode, array('all', 'db'))) {
         // Upload into target environment
         $log->write('Restore of database to "' . $name . '" started');
         $args = array('data_path' => $tempPath . DIRECTORY_SEPARATOR . 'database.sql.gz');
         $command = $this->getCommand('data:pushdb', 'db', $name, $args, $env, $log);
         $command->run(function ($type, $buffer) use($log) {
             $log->write($buffer);
         });
         if (!$command->isSuccessful()) {
             $cleanupFn();
             $log->write('Restore of database to "' . $name . '" failed: ' . $command->getErrorOutput());
             throw new RuntimeException($command->getErrorOutput());
         }
         $log->write('Restore of database to "' . $name . '" done');
     }
     // Restore assets
     if (in_array($dataTransfer->Mode, array('all', 'assets'))) {
         // Upload into target environment
         $log->write('Restore of assets to "' . $name . '" started');
         // Extract assets.tar.gz into assets/
         $extractCmd = sprintf('cd %s && tar xzf %s', escapeshellarg($tempPath), escapeshellarg($tempPath . DIRECTORY_SEPARATOR . 'assets.tar.gz'));
         $log->write($extractCmd);
         $process = new Process($extractCmd);
         $process->setTimeout(3600);
         $process->run();
         if (!$process->isSuccessful()) {
             $log->write('Could not extract the assets archive');
             $cleanupFn();
             throw new RuntimeException($process->getErrorOutput());
         }
         $args = array('data_path' => $tempPath . DIRECTORY_SEPARATOR . 'assets');
         $command = $this->getCommand('data:pushassets', 'web', $name, $args, $env, $log);
         $command->run(function ($type, $buffer) use($log) {
             $log->write($buffer);
         });
         if (!$command->isSuccessful()) {
             $cleanupFn();
             $log->write('Restore of assets to "' . $name . '" failed: ' . $command->getErrorOutput());
             throw new RuntimeException($command->getErrorOutput());
         }
         $log->write('Restore of assets to "' . $name . '" done');
     }
     $log->write('Rebuilding and cleaning up');
     $cleanupFn();
 }
 public function run($request)
 {
     $args = $request->getVar('args');
     $dryRun = $args && in_array('--dry-run', $args);
     $log = function ($message) {
         $message = sprintf('[%s] ', date('Y-m-d H:i:s')) . $message;
         echo $message . PHP_EOL;
     };
     if (!Director::is_cli()) {
         $log('This task must be run under the command line');
         return;
     }
     if ($dryRun) {
         $log('Running in dry-run mode. No data will be deleted');
     }
     $count = 0;
     foreach (DNEnvironment::get() as $environment) {
         $project = $environment->Project();
         if (!$project || !$project->exists()) {
             $log(sprintf('Environment (ID %s, Name: %s, Created: %s) is linked to a non-existent project. Deleting', $environment->ID, $environment->Name, $environment->Created));
             if (!$dryRun) {
                 $environment->delete();
                 $environment->destroy();
             }
             $count++;
         }
     }
     foreach (DNDeployment::get() as $deployment) {
         $environment = $deployment->Environment();
         if (!$environment || !$environment->exists()) {
             $log(sprintf('Deployment (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $deployment->ID, $deployment->Created));
             if (!$dryRun) {
                 $deployment->delete();
                 $deployment->destroy();
             }
             $count++;
         }
     }
     foreach (DNDataTransfer::get() as $transfer) {
         $environment = $transfer->Environment();
         if (!$environment || !$environment->exists()) {
             $log(sprintf('Data transfer (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $transfer->ID, $transfer->Created));
             if (!$dryRun) {
                 $transfer->delete();
                 $transfer->destroy();
             }
             $count++;
         }
     }
     foreach (DNDataArchive::get() as $archive) {
         $environment = $archive->Environment();
         if (!$environment || !$environment->exists()) {
             $log(sprintf('Archive (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $archive->ID, $archive->Created));
             if (!$dryRun) {
                 $archive->delete();
                 $archive->destroy();
             }
             $count++;
         }
     }
     foreach (DNGitFetch::get() as $fetch) {
         $project = $fetch->Project();
         if (!$project || !$project->exists()) {
             $log(sprintf('Git fetch (ID %s, Created: %s) is linked to a non-existent project. Deleting', $fetch->ID, $fetch->Created));
             if (!$dryRun) {
                 $fetch->delete();
                 $fetch->destroy();
             }
             $count++;
         }
     }
     foreach (DNPing::get() as $ping) {
         $environment = $ping->Environment();
         if (!$environment || !$environment->exists()) {
             $log(sprintf('Ping (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $ping->ID, $ping->Created));
             if (!$dryRun) {
                 $ping->delete();
                 $ping->destroy();
             }
             $count++;
         }
     }
     $log(sprintf('Finished. Processed %s records', $count));
 }
Ejemplo n.º 11
0
 /**
  * Attach an sspak file path to this archive and associate the transfer.
  * Does the job of creating a {@link File} record, and setting correct paths into the assets directory.
  *
  * @param string $sspakFilepath
  * @param DNDataTransfer $dataTransfer
  * @return bool
  */
 public function attachFile($sspakFilepath, DNDataTransfer $dataTransfer)
 {
     $sspakFilepath = ltrim(str_replace(array(ASSETS_PATH, realpath(ASSETS_PATH)), '', $sspakFilepath), DIRECTORY_SEPARATOR);
     $folder = Folder::find_or_make(dirname($sspakFilepath));
     $file = new File();
     $file->Name = basename($sspakFilepath);
     $file->Filename = $sspakFilepath;
     $file->ParentID = $folder->ID;
     $file->write();
     // "Status" will be updated by the job execution
     $dataTransfer->write();
     $this->ArchiveFileID = $file->ID;
     $this->DataTransfers()->add($dataTransfer);
     $this->write();
     return true;
 }
 /**
  * Create a snapshot of the db and store the ID on the Pipline
  *
  * @return bool True if success
  */
 protected function startRevertDatabase()
 {
     // Mark self as creating a snapshot
     $this->Status = 'Started';
     $this->Doing = 'Snapshot';
     $this->log("{$this->Title} reverting database from snapshot");
     // Skip deployment for dry run
     if ($this->Pipeline()->DryRun) {
         $this->write();
         $this->log("[Skipped] Create DNDataTransfer restore");
         return true;
     }
     // Get snapshot
     $pipeline = $this->Pipeline();
     $backup = $pipeline->PreviousSnapshot();
     if (empty($backup) || !$backup->exists()) {
         $this->log("No database to revert for {$this->Title}");
         $this->markFailed();
         return false;
     }
     // Create restore job
     $job = DNDataTransfer::create();
     $job->EnvironmentID = $pipeline->EnvironmentID;
     $job->Direction = 'push';
     $job->Mode = 'db';
     $job->DataArchiveID = $backup->DataArchiveID;
     $job->AuthorID = $pipeline->AuthorID;
     $job->EnvironmentID = $pipeline->EnvironmentID;
     $job->write();
     $job->start();
     // Save rollback
     $this->RollbackDatabaseID = $job->ID;
     $this->write();
     return true;
 }
 public function __construct($record = null, $isSingleton = false, $model = null)
 {
     // Set the fields data.
     if (!$record) {
         $record = array('ID' => 0, 'ClassName' => 'DNDataTransfer', 'RecordClassName' => 'DNDataTransfer');
     }
     parent::__construct($record, $isSingleton, $model);
     $this->class = 'DNDataTransfer';
 }
Ejemplo n.º 14
0
 /**
  * @return PaginatedList
  */
 public function DataTransferLogs()
 {
     $environments = $this->getCurrentProject()->Environments()->column('ID');
     $transfers = DNDataTransfer::get()->filter('EnvironmentID', $environments)->filterByCallback(function ($record) {
         return $record->Environment()->canRestore() || $record->Environment()->canBackup() || $record->Environment()->canUploadArchive() || $record->Environment()->canDownloadArchive();
     });
     return new PaginatedList($transfers->sort("Created", "DESC"), $this->request);
 }
 /**
  * Provide rollback-able pipeline on the verge of failing.
  */
 public function getFailingPipeline()
 {
     // Get default backups
     $previous = DNDeployment::create();
     $previous->SHA = '9f0a012e97715b1871n41gk30f34268u12a0029q';
     $previous->write();
     $current = DNDeployment::create();
     $current->write();
     $snapshot = DNDataTransfer::create();
     $snapshot->write();
     $pipeline = $this->objFromFixture('Pipeline', 'FailingPipe');
     $pipeline->Config = serialize(array('RollbackStep1' => array('Class' => 'RollbackStep', 'RestoreDB' => false, 'MaxDuration' => '3600'), 'RollbackStep2' => array('Class' => 'SmokeTestPipelineStep', 'MaxDuration' => '3600')));
     $pipeline->PreviousDeploymentID = $previous->ID;
     $pipeline->CurrentDeploymentID = $current->ID;
     $pipeline->PreviousSnapshotID = $snapshot->ID;
     $pipeline->write();
     return $pipeline;
 }