The model can also represent a request to upload a file later,
through offline processes like mailing a DVD. In order to associate
and authenticate those requests easily, an upload token is generated for every archive.
The "OriginalEnvironment" points to original source of this snapshot
(the one it was backed up from). It will be empty if the snapshot has been created with offline process.
The "Environment" denotes the ownership of the snapshot. It will be initially set to match the
"OriginalEnvironment", but can be changed later. During the offline process the ownership can be set up
arbitrarily.
When moving snapshots, the file always remains in its initial location.
The archive can have associations to {@link DNDataTransfer}:
- Zero transfers if a manual upload was requested, but not fulfilled yet
- One transfer with Direction=get for a backup from an environment
- One or more transfers with Direction=push for a restore to an environment
The "Author" is either the person creating the archive through a "backup" operation,
the person uploading through a web form, or the person requesting a manual upload.
The "Mode" is what the "Author" said the file includes (either 'only assets', 'only
database', or both). This is used in the ArchiveList.ss template.
/**
* If this file is attached to a {@link DNDataArchive}, then we need to check security permissions, to ensure the
* currently logged in {@link Member} can download the file.
*
* This uses the secureassets module to provide the security of these assets.
*/
public function canDownload()
{
$member = Member::currentUser();
$file = $this->owner;
$archive = DNDataArchive::get()->filter('ArchiveFileID', $file->ID)->First();
if ($archive) {
return $archive->canDownload($member);
}
return true;
// By default, files can be downloaded from assets/ normally
}
/**
* @param array $data
* @param Form $form
*
* @return bool|SS_HTTPResponse
* @throws SS_HTTPResponse_Exception
* @throws ValidationException
* @throws null
*/
public function doMove($data, Form $form)
{
$this->setCurrentActionType(self::ACTION_SNAPSHOT);
// Performs canView permission check by limiting visible projects
$project = $this->getCurrentProject();
if (!$project) {
return $this->project404Response();
}
/** @var DNDataArchive $dataArchive */
$dataArchive = DNDataArchive::get()->byId($data['DataArchiveID']);
if (!$dataArchive) {
throw new LogicException('Invalid data archive');
}
// We check for canDownload because that implies access to the data.
if (!$dataArchive->canDownload()) {
throw new SS_HTTPResponse_Exception('Not allowed to access archive', 403);
}
// Validate $data['EnvironmentID'] by checking against $validEnvs.
$validEnvs = $dataArchive->validTargetEnvironments();
$environment = $validEnvs->find('ID', $data['EnvironmentID']);
if (!$environment) {
throw new LogicException('Invalid environment');
}
$dataArchive->EnvironmentID = $environment->ID;
$dataArchive->write();
return $this->redirectBack();
}
public function DataArchives()
{
$envIds = $this->Environments()->column('ID');
return DNDataArchive::get()->filter('EnvironmentID', $envIds);
}
/**
* Backs up database and/or assets to a designated folder,
* and packs up the files into a single sspak.
*
* @param DNDataTransfer $dataTransfer
* @param DeploynautLogFile $log
*/
protected function dataTransferBackup(DNDataTransfer $dataTransfer, DeploynautLogFile $log)
{
$environment = $dataTransfer->Environment();
$name = $environment->getFullName();
// Associate a new archive with the transfer.
// Doesn't retrieve a filepath just yet, need to generate the files first.
$dataArchive = DNDataArchive::create();
$dataArchive->Mode = $dataTransfer->Mode;
$dataArchive->AuthorID = $dataTransfer->AuthorID;
$dataArchive->OriginalEnvironmentID = $environment->ID;
$dataArchive->EnvironmentID = $environment->ID;
$dataArchive->IsBackup = $dataTransfer->IsBackupDataTransfer();
// Generate directory structure with strict permissions (contains very sensitive data)
$filepathBase = $dataArchive->generateFilepath($dataTransfer);
mkdir($filepathBase, 0700, true);
$databasePath = $filepathBase . DIRECTORY_SEPARATOR . 'database.sql';
// Backup database
if (in_array($dataTransfer->Mode, array('all', 'db'))) {
$log->write(sprintf('Backup of database from "%s" started', $name));
$command = $this->getCommand('data:getdb', 'db', $environment, array('data_path' => $databasePath), $log);
$command->run(function ($type, $buffer) use($log) {
$log->write($buffer);
});
if (!$command->isSuccessful()) {
$this->extend('dataTransferFailure', $environment, $log);
throw new RuntimeException($command->getErrorOutput());
}
$log->write(sprintf('Backup of database from "%s" done', $name));
}
// Backup assets
if (in_array($dataTransfer->Mode, array('all', 'assets'))) {
$log->write(sprintf('Backup of assets from "%s" started', $name));
$command = $this->getCommand('data:getassets', 'web', $environment, array('data_path' => $filepathBase), $log);
$command->run(function ($type, $buffer) use($log) {
$log->write($buffer);
});
if (!$command->isSuccessful()) {
$this->extend('dataTransferFailure', $environment, $log);
throw new RuntimeException($command->getErrorOutput());
}
$log->write(sprintf('Backup of assets from "%s" done', $name));
}
$sspakFilename = sprintf('%s.sspak', $dataArchive->generateFilename($dataTransfer));
$sspakFilepath = $filepathBase . DIRECTORY_SEPARATOR . $sspakFilename;
try {
$dataArchive->attachFile($sspakFilepath, $dataTransfer);
$dataArchive->setArchiveFromFiles($filepathBase);
} catch (Exception $e) {
$log->write($e->getMessage());
throw new RuntimeException($e->getMessage());
}
// Remove any assets and db files lying around, they're not longer needed as they're now part
// of the sspak file we just generated. Use --force to avoid errors when files don't exist,
// e.g. when just an assets backup has been requested and no database.sql exists.
$process = new Process(sprintf('rm -rf %s/assets && rm -f %s', $filepathBase, $databasePath));
$process->run();
if (!$process->isSuccessful()) {
$log->write('Could not delete temporary files');
throw new RuntimeException($process->getErrorOutput());
}
$log->write(sprintf('Creating sspak file done: %s', $dataArchive->ArchiveFile()->getAbsoluteURL()));
}
/**
* Backs up database and/or assets to a designated folder,
* and packs up the files into a single sspak.
*
* @param DNDataTransfer $dataTransfer
* @param DeploynautLogFile $log
*/
protected function dataTransferBackup(DNDataTransfer $dataTransfer, DeploynautLogFile $log)
{
$environmentObj = $dataTransfer->Environment();
$project = $environmentObj->Project();
$projectName = $project->Name;
$environmentName = $environmentObj->Name;
$env = $project->getProcessEnv();
$project = DNProject::get()->filter('Name', $projectName)->first();
$name = $projectName . ':' . $environmentName;
// Associate a new archive with the transfer.
// Doesn't retrieve a filepath just yet, need to generate the files first.
$dataArchive = DNDataArchive::create();
$dataArchive->Mode = $dataTransfer->Mode;
$dataArchive->AuthorID = $dataTransfer->AuthorID;
$dataArchive->OriginalEnvironmentID = $dataTransfer->Environment()->ID;
$dataArchive->EnvironmentID = $dataTransfer->Environment()->ID;
$dataArchive->IsBackup = $dataTransfer->IsBackupDataTransfer();
// Generate directory structure with strict permissions (contains very sensitive data)
$filepathBase = $dataArchive->generateFilepath($dataTransfer);
mkdir($filepathBase, 0700, true);
$databasePath = $filepathBase . DIRECTORY_SEPARATOR . 'database.sql';
// Backup database
if (in_array($dataTransfer->Mode, array('all', 'db'))) {
$log->write('Backup of database from "' . $name . '" started');
$args = array('data_path' => $databasePath);
$command = $this->getCommand("data:getdb", 'db', $name, $args, $env, $log);
$command->run(function ($type, $buffer) use($log) {
$log->write($buffer);
});
if (!$command->isSuccessful()) {
throw new RuntimeException($command->getErrorOutput());
}
$log->write('Backup of database from "' . $name . '" done');
}
// Backup assets
if (in_array($dataTransfer->Mode, array('all', 'assets'))) {
$log->write('Backup of assets from "' . $name . '" started');
$args = array('data_path' => $filepathBase);
$command = $this->getCommand("data:getassets", 'web', $name, $args, $env, $log);
$command->run(function ($type, $buffer) use($log) {
$log->write($buffer);
});
if (!$command->isSuccessful()) {
throw new RuntimeException($command->getErrorOutput());
}
$log->write('Backup of assets from "' . $name . '" done');
}
$log->write('Creating *.sspak file');
$sspakFilename = sprintf('%s.sspak', $dataArchive->generateFilename($dataTransfer));
$sspakCmd = sprintf('cd %s && sspak saveexisting %s 2>&1', $filepathBase, $sspakFilename);
if ($dataTransfer->Mode == 'db') {
$sspakCmd .= sprintf(' --db=%s', $databasePath);
} elseif ($dataTransfer->Mode == 'assets') {
$sspakCmd .= sprintf(' --assets=%s/assets', $filepathBase);
} else {
$sspakCmd .= sprintf(' --db=%s --assets=%s/assets', $databasePath, $filepathBase);
}
$process = new Process($sspakCmd);
$process->setTimeout(3600);
$process->run();
if (!$process->isSuccessful()) {
$log->write('Could not package the backup via sspak');
throw new RuntimeException($process->getErrorOutput());
}
// HACK: find_or_make() expects path relative to assets/
$sspakFilepath = ltrim(str_replace(array(ASSETS_PATH, realpath(ASSETS_PATH)), '', $filepathBase . DIRECTORY_SEPARATOR . $sspakFilename), DIRECTORY_SEPARATOR);
try {
$folder = Folder::find_or_make(dirname($sspakFilepath));
$file = new File();
$file->Name = $sspakFilename;
$file->Filename = $sspakFilepath;
$file->ParentID = $folder->ID;
$file->write();
// "Status" will be updated by the job execution
$dataTransfer->write();
// Get file hash to ensure consistency.
// Only do this when first associating the file since hashing large files is expensive.
$dataArchive->ArchiveFileHash = md5_file($file->FullPath);
$dataArchive->ArchiveFileID = $file->ID;
$dataArchive->DataTransfers()->add($dataTransfer);
$dataArchive->write();
} catch (Exception $e) {
$log->write('Failed to add sspak file: ' . $e->getMessage());
throw new RuntimeException($e->getMessage());
}
// Remove any assets and db files lying around, they're not longer needed as they're now part
// of the sspak file we just generated. Use --force to avoid errors when files don't exist,
// e.g. when just an assets backup has been requested and no database.sql exists.
$process = new Process(sprintf('rm -rf %s/assets && rm -f %s', $filepathBase, $databasePath));
$process->run();
if (!$process->isSuccessful()) {
$log->write('Could not delete temporary files');
throw new RuntimeException($process->getErrorOutput());
}
$log->write(sprintf('Creating *.sspak file done: %s', $file->getAbsoluteURL()));
}
public function testCanMoveTo()
{
$samantha = $this->objFromFixture('Member', 'project1-samantha');
$sarah = $this->objFromFixture('Member', 'project1-sarah');
$eva = $this->objFromFixture('Member', 'eva');
$uat1 = $this->objFromFixture('DNEnvironment', 'project1-uat');
$live1 = $this->objFromFixture('DNEnvironment', 'project1-live');
$uat2 = $this->objFromFixture('DNEnvironment', 'project2-uat');
$live2 = $this->objFromFixture('DNEnvironment', 'project2-live');
$archive = new DNDataArchive();
$archive->EnvironmentID = $uat1->ID;
$archive->write();
// Samantha doesn't have upload permission to live1.
$this->assertFalse($archive->canMoveTo($live1, $samantha));
// Cross-project moves are forbidden.
$this->assertFalse($archive->canMoveTo($uat2, $samantha));
// Eva has upload permission to live1.
$this->assertTrue($archive->canMoveTo($live1, $eva));
// Cross-project moves are forbidden.
$this->assertFalse($archive->canMoveTo($uat2, $eva));
// Sarah has upload permission to live1, but not download to uat1.
$this->assertFalse($archive->canMoveTo($live1, $sarah));
}
public function run($request)
{
$args = $request->getVar('args');
$dryRun = $args && in_array('--dry-run', $args);
$log = function ($message) {
$message = sprintf('[%s] ', date('Y-m-d H:i:s')) . $message;
echo $message . PHP_EOL;
};
if (!Director::is_cli()) {
$log('This task must be run under the command line');
return;
}
if ($dryRun) {
$log('Running in dry-run mode. No data will be deleted');
}
$count = 0;
foreach (DNEnvironment::get() as $environment) {
$project = $environment->Project();
if (!$project || !$project->exists()) {
$log(sprintf('Environment (ID %s, Name: %s, Created: %s) is linked to a non-existent project. Deleting', $environment->ID, $environment->Name, $environment->Created));
if (!$dryRun) {
$environment->delete();
$environment->destroy();
}
$count++;
}
}
foreach (DNDeployment::get() as $deployment) {
$environment = $deployment->Environment();
if (!$environment || !$environment->exists()) {
$log(sprintf('Deployment (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $deployment->ID, $deployment->Created));
if (!$dryRun) {
$deployment->delete();
$deployment->destroy();
}
$count++;
}
}
foreach (DNDataTransfer::get() as $transfer) {
$environment = $transfer->Environment();
if (!$environment || !$environment->exists()) {
$log(sprintf('Data transfer (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $transfer->ID, $transfer->Created));
if (!$dryRun) {
$transfer->delete();
$transfer->destroy();
}
$count++;
}
}
foreach (DNDataArchive::get() as $archive) {
$environment = $archive->Environment();
if (!$environment || !$environment->exists()) {
$log(sprintf('Archive (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $archive->ID, $archive->Created));
if (!$dryRun) {
$archive->delete();
$archive->destroy();
}
$count++;
}
}
foreach (DNGitFetch::get() as $fetch) {
$project = $fetch->Project();
if (!$project || !$project->exists()) {
$log(sprintf('Git fetch (ID %s, Created: %s) is linked to a non-existent project. Deleting', $fetch->ID, $fetch->Created));
if (!$dryRun) {
$fetch->delete();
$fetch->destroy();
}
$count++;
}
}
foreach (DNPing::get() as $ping) {
$environment = $ping->Environment();
if (!$environment || !$environment->exists()) {
$log(sprintf('Ping (ID %s, Created: %s) is linked to a non-existent environment. Deleting', $ping->ID, $ping->Created));
if (!$dryRun) {
$ping->delete();
$ping->destroy();
}
$count++;
}
}
$log(sprintf('Finished. Processed %s records', $count));
}
public function doMove($data, $form)
{
// Performs canView permission check by limiting visible projects
$project = $this->getCurrentProject();
if (!$project) {
return new SS_HTTPResponse("Project '" . Convert::raw2xml($this->getRequest()->latestParam('Project')) . "' not found.", 404);
}
$dataArchive = null;
$dataArchive = DNDataArchive::get()->byId($data['DataArchiveID']);
if (!$dataArchive) {
throw new LogicException('Invalid data archive');
}
// We check for canDownload because that implies access to the data.
if (!$dataArchive->canDownload()) {
throw new SS_HTTPResponse_Exception('Not allowed to access archive', 403);
}
// Validate $data['EnvironmentID'] by checking against $validEnvs.
$validEnvs = $dataArchive->validTargetEnvironments();
$environment = $validEnvs->find('ID', $data['EnvironmentID']);
if (!$environment) {
throw new LogicException('Invalid environment');
}
$dataArchive->EnvironmentID = $environment->ID;
$dataArchive->write();
$this->redirectBack();
}
public function testValidateArchiveContentsFileMissingFails()
{
$archive = DNDataArchive::create();
$filename = __DIR__ . '/sspaks/not.found.sspak';
$archive->ArchiveFile()->Filename = $filename;
$result = $archive->validateArchiveContents('all');
$this->assertFalse($result->valid());
$this->assertEquals('SSPak file "' . $filename . '" cannot be read.', current($result->messageList()));
}
