function takeIncrementalBackupSnapshot(backupJob $job, $snapshotGroup, $seedSnap) { global $config; $scheduledBackup = $job->getScheduledBackup(); /**************************** TAKING INCREMENTAL BACKUP ****************************/ // Create object for the lock... $runningBackup = new runningBackup(); $runningBackup->setInfoLogStream($this->infolog); // Get scheduledBackup info $sbInfo = $scheduledBackup->getInfo(); // Get scheduledBackup host $sbHost = $scheduledBackup->getHost(); // get host info $hostInfo = $sbHost->getInfo(); $mostRecentSnap = $snapshotGroup->getMostRecentCompletedBackupSnapshot(); // Initialize a backup for sbHost for scheduledBackup $runningBackup->init($sbHost, $scheduledBackup); // Release our queue tickets $queueManager = new queueManager(); $queueManager->setLogStream($this->log); foreach ($this->ticketsToReleaseOnStart as $ticket) { $queueManager->releaseTicket($ticket); } try { $lsn = $seedSnap->getLsn(); // Create new object shell $snapshot = new backupSnapshot(); // Init the snapshot here and use the ID for the tempdirname $snapshot->init($scheduledBackup, 'DELTA', 'INCREMENTAL', $snapshotGroup, $mostRecentSnap->id); // Get runningBackup info $rbInfo = $runningBackup->getInfo(); // Start another TRY block so we can catch any exceptions and clean up our snapshot status. try { $snapshotInfo = $snapshot->getInfo(); $tempDir = $runningBackup->getStagingTmpdir(); // Build the command to run the snapshot into the staging dir $xbBinary = $scheduledBackup->getXtraBackupBinary(); // Command should look like this: $xbCommand = "ssh -o StrictHostKeyChecking=no -p " . $hostInfo['ssh_port'] . " " . $sbInfo['backup_user'] . "@" . $hostInfo['hostname'] . " 'cd {$tempDir} ; innobackupex --ibbackup=" . $xbBinary . " --slave-info --incremental-lsn=" . $lsn . " " . $tempDir . "/deltas" . " --user="******" --safe-slave-backup " . " --password="******" --no-timestamp --incremental --throttle=" . $scheduledBackup->getXtraBackupThrottleValue() . " 1>&2 '"; // Set up how we'll interact with the IO file handlers of the process $xbDescriptors = array(0 => array('file', '/dev/null', 'r'), 1 => array('pipe', 'w'), 2 => array('pipe', 'w')); // Connect to the remote host and create the backup into a staging directory then copy it back via netcat and tar //////// //////// // KICK OFF XTRABACKUP INCREMENTAL // //////// //////// // Set the state of the snapshot to RUNNING $snapshot->setStatus('RUNNING'); // Info output $this->infolog->write("Staging an INCREMENTAL xtrabackup snapshot of " . $sbInfo['datadir_path'] . " via ssh: " . $sbInfo['backup_user'] . "@" . $hostInfo['hostname'] . " to " . $tempDir . "/deltas...", XBM_LOG_INFO); // DEBUG $this->infolog->write("Attempting to run the incremental backup with command:\n" . preg_replace('/--password=(.+?)\\s+/', '--password=XXXXXXX ', $xbCommand) . " \n", XBM_LOG_DEBUG); // Start the xtrabackup process $xbProc = proc_open($xbCommand, $xbDescriptors, $xbPipes); // Check that we launched OK. if (!is_resource($xbProc)) { throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to use ssh to start xtrabackup with: {$xbCommand} ."); } // Check the status of the backup every second... $streamContents = ''; stream_set_blocking($xbPipes[2], 0); do { $streamContents .= stream_get_contents($xbPipes[2]); if (!($xbStatus = proc_get_status($xbProc))) { throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to retrieve status on backup process."); } if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } while ($xbStatus['running']); if ($xbStatus['exitcode'] != 0) { throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: There was an error backing up - The process returned code " . $xbStatus['exitcode'] . "." . " The output from the backup is as follows:\n" . $streamContents); } /*print_r($xbStatus); echo "\nWe got STDOUT: \n"; echo stream_get_contents($xbPipes[1]); echo "\nWe got STDERR: \n"; echo stream_get_contents($xbPipes[2]); */ $this->infolog->write("XtraBackup completed staging the backup with the following output:\n" . $streamContents, XBM_LOG_INFO); // Copy it to local machine $path = $snapshot->getPath(); // Fire up a netcat listener // Set the command we plan to run $ncBuilder = new netcatCommandBuilder(); $ncServer = $ncBuilder->getServerCommand($rbInfo['port']); $ncLogfile = $config['LOGS']['logdir'] . '/hosts/' . $hostInfo['hostname'] . '.netcat.log'; // Change to the dir where we want to put the files $ncCommand = ' cd ' . $path . ' ; '; // Check to see if there us a GNU tar - used to make XBM work on Solaris 11, since their tar doesn't work correctly $ncCommand .= ' if [ -f /usr/gnu/bin/tar ]; then TAR="/usr/gnu/bin/tar"; else TAR=tar; fi; '; // Now start the netcat listener and pipe through the auto-detected tar command $ncCommand .= $ncServer . ' | $TAR xvf - >> ' . $ncLogfile . ' 2>&1'; // Open the process with a stream to read from it // Set up how we'll interact with the IO file handlers of the process $ncDescriptors = array(0 => array('pipe', 'r'), 1 => array('pipe', 'w'), 2 => array('pipe', 'w')); // Open the netcat listener proc $this->infolog->write("Attempting to run netcat with command: " . $ncCommand, XBM_LOG_DEBUG); $ncProc = proc_open($ncCommand, $ncDescriptors, $ncPipes); if (!is_resource($ncProc)) { throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to start netcat with command: {$ncCommand} ."); } // Info output $this->infolog->write("Started Netcat (nc) listener on port " . $rbInfo['port'] . " to receive backup tar stream into directory {$path} ...", XBM_LOG_INFO); $ncClient = $ncBuilder->getClientCommand($config['SYSTEM']['xbm_hostname'], $rbInfo['port']); // Copy the backup back via the netcat listener $copyCommand = "ssh -o StrictHostKeyChecking=no -p " . $hostInfo['ssh_port'] . " " . $sbInfo['backup_user'] . "@" . $hostInfo['hostname'] . " 'cd " . $tempDir . "/deltas; tar cvf - . | " . $ncClient . " '"; // Set the state of the snapshot to COPYING $snapshot->setStatus('COPYING'); // Set up how we'll interact with the IO file handlers of the process $copyDescriptors = array(0 => array('file', '/dev/null', 'r'), 1 => array('pipe', 'w'), 2 => array('pipe', 'w')); // Start the xtrabackup process $this->infolog->write("Attempting to run copy with command: " . $copyCommand, XBM_LOG_DEBUG); $copyProc = proc_open($copyCommand, $copyDescriptors, $copyPipes); // Check that we launched OK. if (!is_resource($copyProc)) { throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to use ssh to start copy with: {$copyCommand} ."); } // Check the status of the backup every second... $streamContents = ''; stream_set_blocking($copyPipes[2], 0); do { $streamContents .= stream_get_contents($copyPipes[2]); if (!($copyStatus = proc_get_status($copyProc))) { throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: Unable to retrieve status on copy process."); } if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } while ($copyStatus['running']); if ($copyStatus['exitcode'] != 0) { throw new Exception('genericBackupTaker->takeIncrementalBackupSnapshot: ' . "Error: There was an error copying files - The process returned code " . $copyStatus['exitcode'] . "." . " The output from the backup is as follows:\n" . $streamContents); } /* For debugging... print_r($copyStatus); echo "\nWe got STDOUT: \n"; echo stream_get_contents($copyPipes[1]); echo "\nWe got STDERR: \n"; echo stream_get_contents($copyPipes[2]); */ // Close out the copy and netcat processes proc_close($copyProc); proc_close($ncProc); // Set the snapshot time to be now $snapshot->setSnapshotTime(date('Y-m-d H:i:s')); // Set the state of the snapshot to COMPLETED $snapshot->setStatus('COMPLETED'); $this->infolog->write("Completed copying the backup via netcat with the following output:\n" . $streamContents, XBM_LOG_INFO); } catch (Exception $e) { if (get_class($e) == 'KillException') { $this->infolog->write("The backup job was killed by an administrator. Aborting...", XBM_LOG_ERROR); } // If we had a netcat process, check to see if we need to kill it/clean up if (isset($ncProc) && is_resource($ncProc)) { proc_terminate($ncProc); } // Remove the snapshot files and mark it as failed. if ($config['SYSTEM']['cleanup_on_failure'] == true) { $this->infolog->write("Cleaning up files...", XBM_LOG_INFO); $snapshot->deleteFiles(); } else { $this->infolog->write("Skipping cleanup as cleanup_on_failure is turned off...", XBM_LOG_INFO); } $snapshot->setStatus('FAILED'); throw $e; } } catch (Exception $e) { // Clean up the running backup entry and rethrow error.. $runningBackup->finish(); throw $e; } // Clean up after ourselves.. // release our lock on the netcat port, scheduled backup, etc. $runningBackup->finish(); return $snapshot; }
function destroy() { // Validate this... if (!is_numeric($this->id)) { throw new Exception('scheduledBackup->destroy: ' . "Error: The ID for this object is not an integer."); } $queueManager = new queueManager(); $queueManager->setLogStream($this->log); // We need to take over all queues for this backup and make sure nothing is running. $queues = array('scheduledBackup:' . $this->id, 'retentionApply:' . $this->id, 'postProcess:' . $this->id); foreach ($queues as $queue) { $ticket = $queueManager->getTicketNumber($queue); if (!$queueManager->checkFrontOfQueue($queue, $ticket)) { throw new ProcessingException("Error: Cannot remove the Scheduled Backup Task as it is currently running."); } } // Check to see if anything is running for this scheduledBackup $runningBackupGetter = new runningBackupGetter(); $runningBackupGetter->setLogStream($this->log); $runningBackups = $runningBackupGetter->getByScheduledBackup($this); if (sizeOf($runningBackups) > 0) { throw new ProcessingException("Error: Cannot remove the Scheduled Backup Task as it is currently running."); } // Get all snapshots and destroy them.. $groups = $this->getSnapshotGroupsNewestToOldest(); foreach ($groups as $group) { $snapshots = $group->getAllSnapshotsNewestToOldest(); foreach ($snapshots as $snapshot) { $snapshot->destroy(); } } // If we have a materialized snapshot - destroy that too if ($latestMaterialized = $this->getMostRecentCompletedMaterializedSnapshot()) { $latestMaterialized->destroy(); } $conn = dbConnection::getInstance($this->log); // Remove DB the entries for this scheduledBackup $sql = "DELETE sb.*, sbp.* FROM scheduled_backups sb JOIN scheduled_backup_params sbp USING (scheduled_backup_id) WHERE scheduled_backup_id = " . $this->id; if (!$conn->query($sql)) { throw new DBException('scheduledBackup->setParam: ' . "Error: Query {$sql} \nFailed with MySQL Error: {$conn->error}"); } return; }
function takeScheduledBackupSnapshot(scheduledBackup $scheduledBackup) { global $config; $this->launchTime = time(); // Get the backup strategy for the scheduledBackup $sbInfo = $scheduledBackup->getInfo(); // Get the host of the backup $sbHost = $scheduledBackup->getHost(); // Get host info $hostInfo = $sbHost->getInfo(); // create a backupTakerFactory $takerFactory = new backupTakerFactory(); // use it to get the right object type for the backup strategy $backupTaker = $takerFactory->getBackupTakerByStrategy($sbInfo['strategy_code']); // Setup to write to host log $infolog = new logStream($config['LOGS']['logdir'] . '/hosts/' . $hostInfo['hostname'] . '.log', $this->infologVerbose, $config['LOGS']['level']); $this->setInfoLogStream($infolog); try { $msg = 'Initializing Scheduled Backup "' . $sbInfo['name'] . '" (ID #' . $sbInfo['scheduled_backup_id'] . ') for host: ' . $hostInfo['hostname'] . ' ... '; $this->infolog->write($msg, XBM_LOG_INFO); $msg = 'Using Backup Strategy: ' . $sbInfo['strategy_name']; $this->infolog->write($msg, XBM_LOG_INFO); // Create an entry for this backup job $jobGetter = new backupJobGetter(); $jobGetter->setLogStream($this->log); $job = $jobGetter->getNew($scheduledBackup); // Check to see if we can even start this party // First, take a number in the queues for the scheduledBackup and the host itself.. $queueManager = new queueManager(); $queueManager->setLogStream($this->log); // Set the global queueName for use later. $globalQueueName = 'scheduledBackup:GLOBAL'; // Take a number in the scheduledBackup queue for THIS backup... $schedQueueName = 'scheduledBackup:' . $sbInfo['scheduled_backup_id']; $schedQueueTicket = $queueManager->getTicketNumber($schedQueueName); // Take a number in the host queue... $hostQueueName = 'hostBackup:' . $sbInfo['host_id']; $hostQueueTicket = $queueManager->getTicketNumber($hostQueueName); // If we are not at the front of the scheduledBackup queue when we start up, then just exit // assume another job is running already for this scheduledBackup.. we dont queue up dupe backup jobs // we skip them if ($queueManager->checkFrontOfQueue($schedQueueName, $schedQueueTicket) == false) { // Release our tickets in the queues, then throw exception... $queueManager->releaseTicket($hostQueueTicket); $queueManager->releaseTicket($schedQueueTicket); $this->infolog->write("Detected this scheduled backup job is already running, exiting...", XBM_LOG_ERROR); throw new Exception('backupSnapshotTaker->takeScheduledBackupSnapshot: ' . "Error: Detected this scheduled backup job is already running."); } // Create this object now, so we don't recreate it a tonne of times in the loop $runningBackupGetter = new runningBackupGetter(); // Mark us as "QUEUED" while we figure out if we can launch... $job->setStatus('Queued'); $readyToRun = false; while ($readyToRun == false) { // If we are not at the front of the queue for this host, then sleep/wait until we are. if (!$queueManager->checkFrontOfQueue($hostQueueName, $hostQueueTicket)) { $this->infolog->write("There are jobs before this one in the queue for this host. Sleeping " . XBM_SLEEP_SECS . " before checking again...", XBM_LOG_INFO); for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) { if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } continue; } // We are at the front of the queue for this host... // Check to see how many backups are running for the host already... $runningBackups = $sbHost->getRunningBackups(); // If we are at or greater than max num of backups for the host, then sleep before we try again. if (sizeOf($runningBackups) >= $config['SYSTEM']['max_host_concurrent_backups']) { // Output to info log - this currently spits out every 30 secs (define is 30 at time of writing) // maybe it is too much $this->infolog->write("Found " . sizeOf($runningBackups) . " backup(s) running for this host out of a maximum of " . $config['SYSTEM']['max_host_concurrent_backups'] . " per host. Sleeping " . XBM_SLEEP_SECS . " before retry...", XBM_LOG_INFO); for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) { if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } continue; } // Only take a ticket in the global queue if we dont already have one // Wait until this point to prevent blocking up the GLOBAL queue when the host itself is blocked.. if (!isset($globalQueueTicket)) { $globalQueueTicket = $queueManager->getTicketNumber($globalQueueName); } // If we are not at the front of the queue for global backups, then sleep/wait until we are if (!$queueManager->checkFrontOfQueue($globalQueueName, $globalQueueTicket)) { $this->infolog->write("There are jobs before this one in the global backup queue. Sleeping " . XBM_SLEEP_SECS . " before checking again...", XBM_LOG_INFO); for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) { if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } continue; } // Now check to see the how many backups are running globally and if we should be allowed to run... $globalRunningBackups = $runningBackupGetter->getAll(); if (sizeOf($globalRunningBackups) >= $config['SYSTEM']['max_global_concurrent_backups']) { //output to info log -- currentl every 30 secs based on define at time of writing // maybe too much? $this->infolog->write("Found " . sizeOf($globalRunningBackups) . " backup(s) running out of a global maximum of " . $config['SYSTEM']['max_global_concurrent_backups'] . ". Sleeping " . XBM_SLEEP_SECS . " before retry...", XBM_LOG_INFO); for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) { if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } continue; } // If we made it to here - we are ready to run! $readyToRun = true; } // Populate the backupTaker with the relevant settings like log/infolog/Verbose, etc. $backupTaker->setLogStream($this->log); $backupTaker->setInfoLogStream($this->infolog); $backupTaker->setInfoLogVerbose($this->infologVerbose); $backupTaker->setLaunchTime($this->launchTime); $backupTaker->setTicketsToReleaseOnStart(array($hostQueueTicket, $globalQueueTicket)); // Kick off takeScheduledBackupSnapshot method of the actual backup taker $job->setStatus('Performing Backup'); $backupTaker->takeScheduledBackupSnapshot($job); // Release the ticket for running the backup.. $queueManager->releaseTicket($schedQueueTicket); $retentionQueueName = 'retentionApply:' . $sbInfo['scheduled_backup_id']; $retentionQueueTicket = $queueManager->getTicketNumber($retentionQueueName); // Proceed once we're at the start of the retention policy queue for this scheduled backup while (!$queueManager->checkFrontOfQueue($retentionQueueName, $retentionQueueTicket)) { $this->infolog->write('There is already a task applying retention policy for this scheduled backup. Sleeping ' . XBM_SLEEP_SECS . ' before retry...', XBM_LOG_INFO); for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) { if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } } // Apply the retention policy $this->infolog->write("Applying snapshot retention policy ...", XBM_LOG_INFO); $job->setStatus('Deleting Old Backups'); $backupTaker->applyRetentionPolicy($job); $this->infolog->write("Application of retention policy complete.", XBM_LOG_INFO); $queueManager->releaseTicket($retentionQueueTicket); // Perform any post processingA // Get ticket/queue $postProcessQueueName = 'postProcess:' . $sbInfo['scheduled_backup_id']; $postProcessQueueTicket = $queueManager->getTicketNumber($postProcessQueueName); while (!$queueManager->checkFrontOfQueue($postProcessQueueName, $postProcessQueueTicket)) { $this->infolog->write('There is already a task performing post processing for this scheduled backup. Sleeping ' . XBM_SLEEP_SECS . ' before retry...', XBM_LOG_INFO); for ($i = 0; $i <= XBM_SLEEP_SECS; $i++) { if ($job->isKilled()) { throw new KillException('The backup was killed by an administrator.'); } sleep(1); } } $this->infolog->write("Performing any post-processing necessary ...", XBM_LOG_INFO); $job->setStatus('Performing Post-Processing'); $backupTaker->postProcess($job); $this->infolog->write("Post-processing completed.", XBM_LOG_INFO); $queueManager->releaseTicket($postProcessQueueTicket); $this->infolog->write("Scheduled Backup Task Complete!", XBM_LOG_INFO); $job->setStatus('Completed'); } catch (KillException $e) { if (isset($job)) { $job->setStatus('Killed'); } $this->infolog->write('Exiting after the backup job was killed...', XBM_LOG_ERROR); throw $e; } catch (Exception $e) { if (isset($job)) { $job->setStatus('Failed'); } $this->infolog->write('An error occurred while trying to perform the backup. Proceeding to log some details to help debug...', XBM_LOG_ERROR); $this->infolog->write('Error Caught: ' . $e->getMessage(), XBM_LOG_ERROR); $this->infolog->write('Trace: ' . $e->getTraceAsString(), XBM_LOG_ERROR); throw $e; } return true; }