// end deletion.
// Handle download link
if (pb_backupbuddy::_GET('downloadlink_file') != '') {
$downloadSettings = $settings;
$downloadSettings['directory'] = $remotePath;
$link = pb_backupbuddy_destination_live::getFileURL($downloadSettings, base64_decode(pb_backupbuddy::_GET('downloadlink_file')));
pb_backupbuddy::alert('You may download this backup (' . base64_decode(pb_backupbuddy::_GET('downloadlink_file')) . ') with <a href="' . $link . '">this link</a>. The link is valid for one hour.');
echo '<br>';
}
// end download link.
$marker = null;
if ('' != pb_backupbuddy::_GET('marker')) {
// Jump to specific spot.
$marker = base64_decode(urldecode(pb_backupbuddy::_GET('marker')));
}
$files = pb_backupbuddy_destination_live::listFiles($settings, $remotePath, $marker);
if (!is_array($files)) {
pb_backupbuddy::alert('Error #892329b: ' . $files);
die;
}
/*
echo '<pre>';
print_r( $files );
echo '</pre>';
*/
$backup_list_temp = array();
foreach ((array) $files as $file) {
$last_modified = strtotime($file['LastModified']);
while (isset($backup_list_temp[$last_modified])) {
$last_modified += 0.1;
// Add .1 repeatedly until timestamp is free.
private static function _step_audit_remote_files($marker = null, $runningCount = 0)
{
if (time() - self::$_state['stats']['last_file_audit_finish'] < self::TIME_BETWEEN_FILE_AUDIT) {
pb_backupbuddy::status('details', 'Not enough time has passed since last file audit. Skipping for now. Minimum time: `' . self::TIME_BETWEEN_FILE_AUDIT . '` secs. Last ran ago: `' . (time() - self::$_state['stats']['last_file_audit_finish']) . '` secs.');
return true;
}
$deleteBatchSize = 100;
// Delete files in batches of this many files via deleteObjects via deleteFiles().
$serialDir = 'wp-content/uploads/backupbuddy_temp/SERIAL/';
// Include trailing slash.
$serialDirLen = strlen($serialDir);
if (false === self::_load_state()) {
return false;
}
if (false === self::_load_catalog()) {
return false;
}
if (false === self::_load_tables()) {
return false;
}
$destination_settings = self::get_destination_settings();
require_once pb_backupbuddy::plugin_path() . '/destinations/live/init.php';
if (null == $marker) {
// Only reset if NOT chunking (first pass). Was cause of a bug first few weeks of release resulting in process restarting after hitting 100% if audit step chunked.
self::$_state['stats']['last_file_audit_start'] = microtime(true);
// Audit start time.
}
self::$_stateObj->save();
$loopCount = 0;
$loopStart = microtime(true);
$keepLooping = true;
$totalListed = 0;
$totalTables = 0;
$serialSkips = 0;
$filesDeleted = 0;
$tablesDeleted = 0;
$last_save = microtime(true);
while (true === $keepLooping) {
$loopCount++;
pb_backupbuddy::status('details', 'Listing files starting at marker `' . $marker . '`.');
$files = pb_backupbuddy_destination_live::listFiles($destination_settings, $remotePath = '', $marker);
if (!is_array($files)) {
$error = 'Error #3279327: One or more errors encountered attempting to list remote files for auditing. Details: `' . print_r($files, true) . '`.';
pb_backupbuddy::status('error', $error);
backupbuddy_core::addNotification('live_error', 'BackupBuddy Stash Live Error', $error);
self::$_state['step']['last_status'] = 'Error: Unable to list remote files for audit.';
return false;
}
pb_backupbuddy::status('details', 'Listed `' . count($files) . '` files.');
$totalListed += count($files);
// Iterate through all remote files.
$pendingDelete = array();
$filesDeletedThisRound = 0;
foreach ($files as $file) {
// Skip all files in the SERIAL directory with underscore. Audit the rest.
if (substr($file['Key'], 0, $serialDirLen) == $serialDir) {
$totalTables++;
$basename = basename($file['Key']);
// Ignore underscore-prefixed live db data. Do not audit these. Skip.
if ('_' == substr($basename, 0, 1)) {
$serialSkips++;
continue;
}
// Ignore backupbuddy_dat.php metadata file and importbuddy.php files in database folder.
if ('backupbuddy_dat.php' == $basename || 'importbuddy.php' == $basename) {
continue;
}
// Verify no unexpected extra .sql files exist.
if (pb_backupbuddy::$options['log_level'] == '3') {
// Full logging enabled.
pb_backupbuddy::status('details', 'Auditing remotely found table (shown due to log level): `' . $basename . '`.');
}
$table_name = str_replace('.sql', '', $basename);
if (!isset(self::$_tables[$table_name])) {
pb_backupbuddy::status('details', 'Deleting unexpectedly remotely found table file: `' . $basename . '`.');
if (true !== ($delete_result = pb_backupbuddy_destination_live::deleteFile($destination_settings, array($file['Key'])))) {
pb_backupbuddy::status('error', 'Error #329030923: Unable to delete remote file. See log above for details. Details: `' . $deleteResult . '`.');
} else {
pb_backupbuddy::status('details', 'Deleted remote database file `' . $file['Key'] . '`.');
$tablesDeleted++;
}
}
continue;
}
if (!isset(self::$_catalog['/' . $file['Key']])) {
// Remotely stored file not found in local catalog. Delete remote.
$pendingDelete[] = $file['Key'];
// Process deletions.
if (count($pendingDelete) >= $deleteBatchSize) {
if (true !== ($delete_result = pb_backupbuddy_destination_live::deleteFile($destination_settings, $pendingDelete))) {
pb_backupbuddy::status('error', 'Error #4397347934: Unable to delete one or more remote files. See log above for details. Details: `' . print_r($delete_result, true) . '`. Clearing pendingDelete var for next batch.');
} else {
pb_backupbuddy::status('details', 'Deleted batch of `' . count($pendingDelete) . '` remote files. Cleaning pendingDelete var for next batch.');
$filesDeleted += count($pendingDelete);
$filesDeletedThisRound += count($pendingDelete);
}
$pendingDelete = array();
}
} else {
// Remotely stored file found in local catalog. Updated verified audit timestamp.
// Update 'v' key (for verified) with current timestamp to show it is verified as being on remote server.
self::$_catalog['/' . $file['Key']]['v'] = microtime(true);
}
}
// Process any remaining deletions.
if (count($pendingDelete) > 0) {
if (true !== ($delete_result = pb_backupbuddy_destination_live::deleteFile($destination_settings, $pendingDelete))) {
pb_backupbuddy::status('error', 'Error #373262793: Unable to delete one or more remote files. See log above for details. Details: `' . $deleteResult . '`. Clearing pendingDelete var for next batch.');
} else {
pb_backupbuddy::status('details', 'Deleted batch of `' . count($pendingDelete) . '` remote files.');
$filesDeleted += count($pendingDelete);
}
unset($pendingDelete);
}
pb_backupbuddy::status('details', 'Deleted `' . $filesDeletedThisRound . '` total files this round out of `' . count($files) . '` listed. Looped `' . $loopCount . '` times.');
// See if it's time to save 'v' key changes so far.
if (time() - $last_save > self::SAVE_SIGNATURES_EVERY_X_SECONDS) {
self::$_catalogObj->save();
//self::$_stateObj->save();
$last_save = microtime(true);
}
$filesListedMinusSkips = $totalListed - $serialSkips;
$total_files = $filesListedMinusSkips - $filesDeleted;
$totalTablesMinusSkips = $totalTables - $serialSkips;
// If files retrieves is >= to the list limit then there may be more files. Set marker and chunk.
if (count($files) < $destination_settings['max_filelist_keys']) {
// No more files remain.
$keepLooping = false;
self::$_catalogObj->save();
self::$_state['stats']['last_file_audit_finish'] = microtime(true);
// Audit finish time.
$runningCount += $total_files - $totalTablesMinusSkips;
pb_backupbuddy::status('details', 'No more files to check. Deleted `' . $filesDeleted . '` out of listed `' . $totalListed . '` (`' . $filesListedMinusSkips . '` files, Deleted `' . $tablesDeleted . '` tables out of `' . $totalTablesMinusSkips . '` total tables. `' . $serialSkips . '` skipped database/serial dir). `' . $total_files . '` files+tables.sql files after deletions. Files running count: `' . $runningCount . '`.');
if ($runningCount < self::$_state['stats']['files_total_count']) {
$message = 'Attention! Remote storage lists fewer files (' . $runningCount . ') than expected (' . self::$_state['stats']['files_total_count'] . '). More files may be pending transfer. Deleted: `' . $filesDeletedThisRound . '`.';
pb_backupbuddy::status('error', $message);
backupbuddy_core::addNotification('live_error', 'BackupBuddy Stash Live Error', $message);
}
return true;
} else {
// More files MAY remain.
pb_backupbuddy::status('details', 'More files remain to check. Deleted `' . $filesDeleted . '` total files this round so far. Files running count: `' . ($runningCount + $total_files - $totalTablesMinusSkips) . '`.');
$marker = end($files);
$marker = $marker['Key'];
reset($files);
// Do we have enough time to proceed or do we need to chunk?
$time_elapsed = microtime(true) - pb_backupbuddy::$start_time;
$time_remaining = $destination_settings['max_time'] - ($time_elapsed + self::TIME_WIGGLE_ROOM);
// Estimated time remaining before PHP times out. Unit: seconds.
$averageTimePerLoop = (microtime(true) - $loopStart) / $loopCount;
pb_backupbuddy::status('details', 'Time elapsed: `' . $time_elapsed . '`, estimated remaining: `' . $time_remaining . '`, average time needed per loop: `' . $averageTimePerLoop . '`. Max time setting: `' . $destination_settings['max_time'] . '`.');
if ($averageTimePerLoop >= $time_remaining) {
// Not enough time for another loop. Chunk.
$keepLooping = false;
self::$_catalogObj->save();
$runningCount += $total_files - $totalTablesMinusSkips;
pb_backupbuddy::status('details', 'Running out of time processing file audit. Took `' . (microtime(true) - $loopStart) . '` seconds to delete `' . $filesDeleted . '` out of listed `' . $totalListed . '` (`' . $filesListedMinusSkips . '` files, Deleted `' . $tablesDeleted . '` tables out of `' . $totalTablesMinusSkips . '` total tables. `' . $serialSkips . '` skipped database/serial dir). `' . ($filesListedMinusSkips - $filesDeleted) . '` files after deletions. Starting next at `' . $marker . '`. Files running count: `' . $runningCount . '`.');
return array('Auditing remote files', array($marker, $runningCount));
} else {
// Proceed looping in this PHP page load...
$keepLooping = true;
}
}
// end if more files may remain.
}
// End while.
// Made it here so we finished.
self::$_catalogObj->save();
self::$_state['stats']['last_file_audit_finish'] = microtime(true);
// Audit finish time.
// If not all files have uploaded, skip snapshot for now.
if (self::$_state['stats']['files_pending_send'] > 0 || self::$_state['stats']['tables_pending_send'] > 0) {
pb_backupbuddy::status('details', '`' . self::$_state['stats']['files_pending_send'] . '` files and `' . self::$_state['stats']['tables_pending_send'] . '` database tables are still pending transfer. Waiting for transfers to finish before creating Snapshot.');
self::$_state['stats']['wait_on_transfers_start'] = microtime(true);
backupbuddy_live::queue_step($step = 'wait_on_transfers', $args = array(), $skip_run_now = true);
return true;
}
return true;
}