function takeIncrementalBackupSnapshot(backupJob $job, $snapshotGroup, $seedSnap)
{
global $config;
$scheduledBackup = $job->getScheduledBackup();
/****************************
TAKING INCREMENTAL BACKUP
****************************/
// Create object for the lock...
$runningBackup = new runningBackup();
$runningBackup->setInfoLogStream($this->infolog);
// Get scheduledBackup info
$sbInfo = $scheduledBackup->getInfo();
// Get scheduledBackup host
$sbHost = $scheduledBackup->getHost();
// get host info
$hostInfo = $sbHost->getInfo();
$mostRecentSnap = $snapshotGroup->getMostRecentCompletedBackupSnapshot();
// Initialize a backup for sbHost for scheduledBackup
$runningBackup->init($sbHost, $scheduledBackup);
// Release our queue tickets
$queueManager = new queueManager();
$queueManager->setLogStream($this->log);
foreach ($this->ticketsToReleaseOnStart as $ticket) {
$queueManager->releaseTicket($ticket);
}
try {
$lsn = $seedSnap->getLsn();
// Create new object shell
$snapshot = new backupSnapshot();
// Init the snapshot here and use the ID for the tempdirname
$snapshot->init($scheduledBackup, 'DELTA', 'INCREMENTAL', $snapshotGroup, $mostRecentSnap->id);
// Get runningBackup info
$rbInfo = $runningBackup->getInfo();
// Start another TRY block so we can catch any exceptions and clean up our snapshot status.
try {
$snapshotInfo = $snapshot->getInfo();
$tempDir = $runningBackup->getStagingTmpdir();
// Build the command to run the snapshot into the staging dir
$xbBinary = $scheduledBackup->getXtraBackupBinary();
// Command should look like this:
$xbCommand = "ssh -o StrictHostKeyChecking=no -p " . $hostInfo['ssh_port'] . " " . $sbInfo['backup_user'] . "@" . $hostInfo['hostname'] . " 'cd {$tempDir} ; innobackupex --ibbackup=" . $xbBinary . " --slave-info --incremental-lsn=" . $lsn . " " . $tempDir . "/deltas" . " --user="******" --safe-slave-backup " . " --password="******" --no-timestamp --incremental --throttle=" . $scheduledBackup->getXtraBackupThrottleValue() . " 1>&2 '";
// Set up how we'll interact with the IO file handlers of the process
$xbDescriptors = array(0 => array('file', '/dev/null', 'r'), 1 => array('pipe', 'w'), 2 => array('pipe', 'w'));
// Connect to the remote host and create the backup into a staging directory then copy it back via netcat and tar
////////
////////
// KICK OFF XTRABACKUP INCREMENTAL //
////////
////////
// Set the state of the snapshot to RUNNING
$snapshot->setStatus('RUNNING');
// Info output
$this->infolog->write("Staging an INCREMENTAL xtrabackup snapshot of " . $sbInfo['datadir_path'] . " via ssh: " . $sbInfo['backup_user'] . "@" . $hostInfo['hostname'] . " to " . $tempDir . "/deltas...", XBM_LOG_INFO);
// DEBUG
$this->infolog->write("Attempting to run the incremental backup with command:\n" . preg_replace('/--password=(.+?)\\s+/', '--password=XXXXXXX ', $xbCommand) . " \n", XBM_LOG_DEBUG);
// Start the xtrabackup process
$xbProc = proc_open($xbCommand, $xbDescriptors, $xbPipes);
// Check that we launched OK.
if (!is_resource($xbProc)) {
throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to use ssh to start xtrabackup with: {$xbCommand} .");
}
// Check the status of the backup every second...
$streamContents = '';
stream_set_blocking($xbPipes[2], 0);
do {
$streamContents .= stream_get_contents($xbPipes[2]);
if (!($xbStatus = proc_get_status($xbProc))) {
throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to retrieve status on backup process.");
}
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
} while ($xbStatus['running']);
if ($xbStatus['exitcode'] != 0) {
throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: There was an error backing up - The process returned code " . $xbStatus['exitcode'] . "." . " The output from the backup is as follows:\n" . $streamContents);
}
/*print_r($xbStatus);
echo "\nWe got STDOUT: \n";
echo stream_get_contents($xbPipes[1]);
echo "\nWe got STDERR: \n";
echo stream_get_contents($xbPipes[2]); */
$this->infolog->write("XtraBackup completed staging the backup with the following output:\n" . $streamContents, XBM_LOG_INFO);
// Copy it to local machine
$path = $snapshot->getPath();
// Fire up a netcat listener
// Set the command we plan to run
$ncBuilder = new netcatCommandBuilder();
$ncServer = $ncBuilder->getServerCommand($rbInfo['port']);
$ncLogfile = $config['LOGS']['logdir'] . '/hosts/' . $hostInfo['hostname'] . '.netcat.log';
// Change to the dir where we want to put the files
$ncCommand = ' cd ' . $path . ' ; ';
// Check to see if there us a GNU tar - used to make XBM work on Solaris 11, since their tar doesn't work correctly
$ncCommand .= ' if [ -f /usr/gnu/bin/tar ]; then TAR="/usr/gnu/bin/tar"; else TAR=tar; fi; ';
// Now start the netcat listener and pipe through the auto-detected tar command
$ncCommand .= $ncServer . ' | $TAR xvf - >> ' . $ncLogfile . ' 2>&1';
// Open the process with a stream to read from it
// Set up how we'll interact with the IO file handlers of the process
$ncDescriptors = array(0 => array('pipe', 'r'), 1 => array('pipe', 'w'), 2 => array('pipe', 'w'));
// Open the netcat listener proc
$this->infolog->write("Attempting to run netcat with command: " . $ncCommand, XBM_LOG_DEBUG);
$ncProc = proc_open($ncCommand, $ncDescriptors, $ncPipes);
if (!is_resource($ncProc)) {
throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to start netcat with command: {$ncCommand} .");
}
// Info output
$this->infolog->write("Started Netcat (nc) listener on port " . $rbInfo['port'] . " to receive backup tar stream into directory {$path} ...", XBM_LOG_INFO);
$ncClient = $ncBuilder->getClientCommand($config['SYSTEM']['xbm_hostname'], $rbInfo['port']);
// Copy the backup back via the netcat listener
$copyCommand = "ssh -o StrictHostKeyChecking=no -p " . $hostInfo['ssh_port'] . " " . $sbInfo['backup_user'] . "@" . $hostInfo['hostname'] . " 'cd " . $tempDir . "/deltas; tar cvf - . | " . $ncClient . " '";
// Set the state of the snapshot to COPYING
$snapshot->setStatus('COPYING');
// Set up how we'll interact with the IO file handlers of the process
$copyDescriptors = array(0 => array('file', '/dev/null', 'r'), 1 => array('pipe', 'w'), 2 => array('pipe', 'w'));
// Start the xtrabackup process
$this->infolog->write("Attempting to run copy with command: " . $copyCommand, XBM_LOG_DEBUG);
$copyProc = proc_open($copyCommand, $copyDescriptors, $copyPipes);
// Check that we launched OK.
if (!is_resource($copyProc)) {
throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to use ssh to start copy with: {$copyCommand} .");
}
// Check the status of the backup every second...
$streamContents = '';
stream_set_blocking($copyPipes[2], 0);
do {
$streamContents .= stream_get_contents($copyPipes[2]);
if (!($copyStatus = proc_get_status($copyProc))) {
throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to retrieve status on copy process.");
}
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
} while ($copyStatus['running']);
if ($copyStatus['exitcode'] != 0) {
throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: There was an error copying files - The process returned code " . $copyStatus['exitcode'] . "." . " The output from the backup is as follows:\n" . $streamContents);
}
/* For debugging...
print_r($copyStatus);
echo "\nWe got STDOUT: \n";
echo stream_get_contents($copyPipes[1]);
echo "\nWe got STDERR: \n";
echo stream_get_contents($copyPipes[2]);
*/
// Close out the copy and netcat processes
proc_close($copyProc);
proc_close($ncProc);
// Set the snapshot time to be now
$snapshot->setSnapshotTime(date('Y-m-d H:i:s'));
// Set the state of the snapshot to COMPLETED
$snapshot->setStatus('COMPLETED');
$this->infolog->write("Completed copying the backup via netcat with the following output:\n" . $streamContents, XBM_LOG_INFO);
} catch (Exception $e) {
if (get_class($e) == 'KillException') {
$this->infolog->write("The backup job was killed by an administrator. Aborting...", XBM_LOG_ERROR);
}
// If we had a netcat process, check to see if we need to kill it/clean up
if (isset($ncProc) && is_resource($ncProc)) {
proc_terminate($ncProc);
}
// Remove the snapshot files and mark it as failed.
if ($config['SYSTEM']['cleanup_on_failure'] == true) {
$this->infolog->write("Cleaning up files...", XBM_LOG_INFO);
$snapshot->deleteFiles();
} else {
$this->infolog->write("Skipping cleanup as cleanup_on_failure is turned off...", XBM_LOG_INFO);
}
$snapshot->setStatus('FAILED');
throw $e;
}
} catch (Exception $e) {
// Clean up the running backup entry and rethrow error..
$runningBackup->finish();
throw $e;
}
// Clean up after ourselves..
// release our lock on the netcat port, scheduled backup, etc.
$runningBackup->finish();
return $snapshot;
}
function destroy()
{
// Validate this...
if (!is_numeric($this->id)) {
throw new Exception('scheduledBackup->destroy: ' . "Error: The ID for this object is not an integer.");
}
$queueManager = new queueManager();
$queueManager->setLogStream($this->log);
// We need to take over all queues for this backup and make sure nothing is running.
$queues = array('scheduledBackup:' . $this->id, 'retentionApply:' . $this->id, 'postProcess:' . $this->id);
foreach ($queues as $queue) {
$ticket = $queueManager->getTicketNumber($queue);
if (!$queueManager->checkFrontOfQueue($queue, $ticket)) {
throw new ProcessingException("Error: Cannot remove the Scheduled Backup Task as it is currently running.");
}
}
// Check to see if anything is running for this scheduledBackup
$runningBackupGetter = new runningBackupGetter();
$runningBackupGetter->setLogStream($this->log);
$runningBackups = $runningBackupGetter->getByScheduledBackup($this);
if (sizeOf($runningBackups) > 0) {
throw new ProcessingException("Error: Cannot remove the Scheduled Backup Task as it is currently running.");
}
// Get all snapshots and destroy them..
$groups = $this->getSnapshotGroupsNewestToOldest();
foreach ($groups as $group) {
$snapshots = $group->getAllSnapshotsNewestToOldest();
foreach ($snapshots as $snapshot) {
$snapshot->destroy();
}
}
// If we have a materialized snapshot - destroy that too
if ($latestMaterialized = $this->getMostRecentCompletedMaterializedSnapshot()) {
$latestMaterialized->destroy();
}
$conn = dbConnection::getInstance($this->log);
// Remove DB the entries for this scheduledBackup
$sql = "DELETE sb.*, sbp.* FROM scheduled_backups sb JOIN scheduled_backup_params sbp USING (scheduled_backup_id) WHERE scheduled_backup_id = " . $this->id;
if (!$conn->query($sql)) {
throw new DBException('scheduledBackup->setParam: ' . "Error: Query {$sql} \nFailed with MySQL Error: {$conn->error}");
}
return;
}
function takeScheduledBackupSnapshot(scheduledBackup $scheduledBackup)
{
global $config;
$this->launchTime = time();
// Get the backup strategy for the scheduledBackup
$sbInfo = $scheduledBackup->getInfo();
// Get the host of the backup
$sbHost = $scheduledBackup->getHost();
// Get host info
$hostInfo = $sbHost->getInfo();
// create a backupTakerFactory
$takerFactory = new backupTakerFactory();
// use it to get the right object type for the backup strategy
$backupTaker = $takerFactory->getBackupTakerByStrategy($sbInfo['strategy_code']);
// Setup to write to host log
$infolog = new logStream($config['LOGS']['logdir'] . '/hosts/' . $hostInfo['hostname'] . '.log', $this->infologVerbose, $config['LOGS']['level']);
$this->setInfoLogStream($infolog);
try {
$msg = 'Initializing Scheduled Backup "' . $sbInfo['name'] . '" (ID #' . $sbInfo['scheduled_backup_id'] . ') for host: ' . $hostInfo['hostname'] . ' ... ';
$this->infolog->write($msg, XBM_LOG_INFO);
$msg = 'Using Backup Strategy: ' . $sbInfo['strategy_name'];
$this->infolog->write($msg, XBM_LOG_INFO);
// Create an entry for this backup job
$jobGetter = new backupJobGetter();
$jobGetter->setLogStream($this->log);
$job = $jobGetter->getNew($scheduledBackup);
// Check to see if we can even start this party
// First, take a number in the queues for the scheduledBackup and the host itself..
$queueManager = new queueManager();
$queueManager->setLogStream($this->log);
// Set the global queueName for use later.
$globalQueueName = 'scheduledBackup:GLOBAL';
// Take a number in the scheduledBackup queue for THIS backup...
$schedQueueName = 'scheduledBackup:' . $sbInfo['scheduled_backup_id'];
$schedQueueTicket = $queueManager->getTicketNumber($schedQueueName);
// Take a number in the host queue...
$hostQueueName = 'hostBackup:' . $sbInfo['host_id'];
$hostQueueTicket = $queueManager->getTicketNumber($hostQueueName);
// If we are not at the front of the scheduledBackup queue when we start up, then just exit
// assume another job is running already for this scheduledBackup.. we dont queue up dupe backup jobs
// we skip them
if ($queueManager->checkFrontOfQueue($schedQueueName, $schedQueueTicket) == false) {
// Release our tickets in the queues, then throw exception...
$queueManager->releaseTicket($hostQueueTicket);
$queueManager->releaseTicket($schedQueueTicket);
$this->infolog->write("Detected this scheduled backup job is already running, exiting...", XBM_LOG_ERROR);
throw new Exception('backupSnapshotTaker->takeScheduledBackupSnapshot: ' . "Error: Detected this scheduled backup job is already running.");
}
// Create this object now, so we don't recreate it a tonne of times in the loop
$runningBackupGetter = new runningBackupGetter();
// Mark us as "QUEUED" while we figure out if we can launch...
$job->setStatus('Queued');
$readyToRun = false;
while ($readyToRun == false) {
// If we are not at the front of the queue for this host, then sleep/wait until we are.
if (!$queueManager->checkFrontOfQueue($hostQueueName, $hostQueueTicket)) {
$this->infolog->write("There are jobs before this one in the queue for this host. Sleeping " . XBM_SLEEP_SECS . " before checking again...", XBM_LOG_INFO);
for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) {
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
}
continue;
}
// We are at the front of the queue for this host...
// Check to see how many backups are running for the host already...
$runningBackups = $sbHost->getRunningBackups();
// If we are at or greater than max num of backups for the host, then sleep before we try again.
if (sizeOf($runningBackups) >= $config['SYSTEM']['max_host_concurrent_backups']) {
// Output to info log - this currently spits out every 30 secs (define is 30 at time of writing)
// maybe it is too much
$this->infolog->write("Found " . sizeOf($runningBackups) . " backup(s) running for this host out of a maximum of " . $config['SYSTEM']['max_host_concurrent_backups'] . " per host. Sleeping " . XBM_SLEEP_SECS . " before retry...", XBM_LOG_INFO);
for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) {
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
}
continue;
}
// Only take a ticket in the global queue if we dont already have one
// Wait until this point to prevent blocking up the GLOBAL queue when the host itself is blocked..
if (!isset($globalQueueTicket)) {
$globalQueueTicket = $queueManager->getTicketNumber($globalQueueName);
}
// If we are not at the front of the queue for global backups, then sleep/wait until we are
if (!$queueManager->checkFrontOfQueue($globalQueueName, $globalQueueTicket)) {
$this->infolog->write("There are jobs before this one in the global backup queue. Sleeping " . XBM_SLEEP_SECS . " before checking again...", XBM_LOG_INFO);
for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) {
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
}
continue;
}
// Now check to see the how many backups are running globally and if we should be allowed to run...
$globalRunningBackups = $runningBackupGetter->getAll();
if (sizeOf($globalRunningBackups) >= $config['SYSTEM']['max_global_concurrent_backups']) {
//output to info log -- currentl every 30 secs based on define at time of writing
// maybe too much?
$this->infolog->write("Found " . sizeOf($globalRunningBackups) . " backup(s) running out of a global maximum of " . $config['SYSTEM']['max_global_concurrent_backups'] . ". Sleeping " . XBM_SLEEP_SECS . " before retry...", XBM_LOG_INFO);
for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) {
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
}
continue;
}
// If we made it to here - we are ready to run!
$readyToRun = true;
}
// Populate the backupTaker with the relevant settings like log/infolog/Verbose, etc.
$backupTaker->setLogStream($this->log);
$backupTaker->setInfoLogStream($this->infolog);
$backupTaker->setInfoLogVerbose($this->infologVerbose);
$backupTaker->setLaunchTime($this->launchTime);
$backupTaker->setTicketsToReleaseOnStart(array($hostQueueTicket, $globalQueueTicket));
// Kick off takeScheduledBackupSnapshot method of the actual backup taker
$job->setStatus('Performing Backup');
$backupTaker->takeScheduledBackupSnapshot($job);
// Release the ticket for running the backup..
$queueManager->releaseTicket($schedQueueTicket);
$retentionQueueName = 'retentionApply:' . $sbInfo['scheduled_backup_id'];
$retentionQueueTicket = $queueManager->getTicketNumber($retentionQueueName);
// Proceed once we're at the start of the retention policy queue for this scheduled backup
while (!$queueManager->checkFrontOfQueue($retentionQueueName, $retentionQueueTicket)) {
$this->infolog->write('There is already a task applying retention policy for this scheduled backup. Sleeping ' . XBM_SLEEP_SECS . ' before retry...', XBM_LOG_INFO);
for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) {
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
}
}
// Apply the retention policy
$this->infolog->write("Applying snapshot retention policy ...", XBM_LOG_INFO);
$job->setStatus('Deleting Old Backups');
$backupTaker->applyRetentionPolicy($job);
$this->infolog->write("Application of retention policy complete.", XBM_LOG_INFO);
$queueManager->releaseTicket($retentionQueueTicket);
// Perform any post processingA
// Get ticket/queue
$postProcessQueueName = 'postProcess:' . $sbInfo['scheduled_backup_id'];
$postProcessQueueTicket = $queueManager->getTicketNumber($postProcessQueueName);
while (!$queueManager->checkFrontOfQueue($postProcessQueueName, $postProcessQueueTicket)) {
$this->infolog->write('There is already a task performing post processing for this scheduled backup. Sleeping ' . XBM_SLEEP_SECS . ' before retry...', XBM_LOG_INFO);
for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) {
if ($job->isKilled()) {
throw new KillException('The backup was killed by an administrator.');
}
sleep(1);
}
}
$this->infolog->write("Performing any post-processing necessary ...", XBM_LOG_INFO);
$job->setStatus('Performing Post-Processing');
$backupTaker->postProcess($job);
$this->infolog->write("Post-processing completed.", XBM_LOG_INFO);
$queueManager->releaseTicket($postProcessQueueTicket);
$this->infolog->write("Scheduled Backup Task Complete!", XBM_LOG_INFO);
$job->setStatus('Completed');
} catch (KillException $e) {
if (isset($job)) {
$job->setStatus('Killed');
}
$this->infolog->write('Exiting after the backup job was killed...', XBM_LOG_ERROR);
throw $e;
} catch (Exception $e) {
if (isset($job)) {
$job->setStatus('Failed');
}
$this->infolog->write('An error occurred while trying to perform the backup. Proceeding to log some details to help debug...', XBM_LOG_ERROR);
$this->infolog->write('Error Caught: ' . $e->getMessage(), XBM_LOG_ERROR);
$this->infolog->write('Trace: ' . $e->getTraceAsString(), XBM_LOG_ERROR);
throw $e;
}
return true;
}
