public static function send($settings = array(), $files = array(), $send_id = '', $clear_uploads = false)
{
global $pb_backupbuddy_destination_errors;
if (!is_array($files)) {
$files = array($files);
}
if ($clear_uploads === false) {
// Uncomment the following line to override and always clear.
//$clear_uploads = true;
}
$itxapi_username = $settings['itxapi_username'];
$itxapi_password = $settings['itxapi_password'];
$db_archive_limit = $settings['db_archive_limit'];
$full_archive_limit = $settings['full_archive_limit'];
$files_archive_limit = $settings['files_archive_limit'];
$max_chunk_size = $settings['max_chunk_size'];
$remote_path = self::get_remote_path($settings['directory']);
// Has leading and trailng slashes.
if ($settings['ssl'] == '0') {
$disable_ssl = true;
} else {
$disable_ssl = false;
}
$multipart_id = $settings['_multipart_id'];
$multipart_counts = $settings['_multipart_counts'];
pb_backupbuddy::status('details', 'Stash remote path set to `' . $remote_path . '`.');
require_once dirname(__FILE__) . '/lib/class.itx_helper.php';
require_once dirname(dirname(__FILE__)) . '/_s3lib/aws-sdk/sdk.class.php';
// Stash API talk.
$stash = new ITXAPI_Helper(pb_backupbuddy_destination_stash::ITXAPI_KEY, pb_backupbuddy_destination_stash::ITXAPI_URL, $itxapi_username, $itxapi_password);
$manage_data = pb_backupbuddy_destination_stash::get_manage_data($settings);
if (!is_array($manage_data['credentials'])) {
pb_backupbuddy::status('error', 'Error #8484383b: Your authentication credentials for Stash failed. Verify your login and password to Stash. You may need to update the Stash destination settings. Perhaps you recently changed your password?');
return false;
}
// Wipe all current uploads.
if ($clear_uploads === true) {
pb_backupbuddy::status('details', 'Clearing any current uploads via Stash call to `abort-all`.');
$abort_url = $stash->get_upload_url(null, 'abort-all');
$request = new RequestCore($abort_url);
$response = $request->send_request(true);
}
// Process multipart transfer that we already initiated in a previous PHP load.
if ($multipart_id != '') {
// Multipart upload initiated and needs parts sent.
// Create S3 instance.
pb_backupbuddy::status('details', 'Creating Stash S3 instance.');
$s3 = new AmazonS3($settings['_multipart_upload_data']['credentials']);
// the key, secret, token
if ($disable_ssl === true) {
@$s3->disable_ssl(true);
}
pb_backupbuddy::status('details', 'Stash S3 instance created.');
$backup_type = str_replace('/', '', $settings['_multipart_backup_type_dir']);
// For use later by file limiting.
$this_part_number = $settings['_multipart_partnumber'] + 1;
pb_backupbuddy::status('details', 'Stash beginning upload of part `' . $this_part_number . '` of `' . count($settings['_multipart_counts']) . '` parts of file `' . $settings['_multipart_file'] . '` with multipart ID `' . $settings['_multipart_id'] . '`.');
$response = $s3->upload_part($settings['_multipart_upload_data']['bucket'], $settings['_multipart_upload_data']['object'], $settings['_multipart_id'], array('expect' => '100-continue', 'fileUpload' => $settings['_multipart_file'], 'partNumber' => $this_part_number, 'seekTo' => (int) $settings['_multipart_counts'][$settings['_multipart_partnumber']]['seekTo'], 'length' => (int) $settings['_multipart_counts'][$settings['_multipart_partnumber']]['length']));
if (!$response->isOK()) {
$this_error = 'Stash unable to upload file part for multipart upload `' . $settings['_multipart_id'] . '`. Details: `' . print_r($response, true) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
$uploaded_size = $response->header['_info']['size_upload'];
$uploaded_speed = $response->header['_info']['speed_upload'];
pb_backupbuddy::status('details', 'Uploaded size: ' . pb_backupbuddy::$format->file_size($uploaded_size) . ', Speed: ' . pb_backupbuddy::$format->file_size($uploaded_speed) . '/sec.');
}
// Load fileoptions to the send.
pb_backupbuddy::status('details', 'About to load fileoptions data.');
require_once pb_backupbuddy::plugin_path() . '/classes/fileoptions.php';
$fileoptions_obj = new pb_backupbuddy_fileoptions(backupbuddy_core::getLogDirectory() . 'fileoptions/send-' . $send_id . '.txt', $read_only = false, $ignore_lock = false, $create_file = false);
if (true !== ($result = $fileoptions_obj->is_ok())) {
pb_backupbuddy::status('error', __('Fatal Error #9034.2344848. Unable to access fileoptions data.', 'it-l10n-backupbuddy') . ' Error: ' . $result);
return false;
}
pb_backupbuddy::status('details', 'Fileoptions data loaded.');
$fileoptions =& $fileoptions_obj->options;
$update_status = 'Sent part ' . $this_part_number . ' of ' . count($settings['_multipart_counts']) . '.';
// Made it here so success sending part. Increment for next part to send.
$settings['_multipart_partnumber']++;
if (!isset($settings['_multipart_counts'][$settings['_multipart_partnumber']])) {
// No more parts exist for this file. Tell S3 the multipart upload is complete and move on.
pb_backupbuddy::status('details', 'Stash getting parts with etags to notify S3 of completed multipart send.');
$etag_parts = $s3->list_parts($settings['_multipart_upload_data']['bucket'], $settings['_multipart_upload_data']['object'], $settings['_multipart_id']);
pb_backupbuddy::status('details', 'Stash got parts list. Notifying S3 of multipart upload completion.');
$response = $s3->complete_multipart_upload($settings['_multipart_upload_data']['bucket'], $settings['_multipart_upload_data']['object'], $settings['_multipart_id'], $etag_parts);
if (!$response->isOK()) {
$this_error = 'Stash unable to notify S3 of completion of all parts for multipart upload `' . $settings['_multipart_id'] . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
pb_backupbuddy::status('details', 'Stash notified S3 of multipart completion.');
}
$backup_type_dir = $settings['_multipart_backup_type_dir'];
// Notify Stash API that things were succesful.
$done_url = $stash->get_upload_url($settings['_multipart_file'], 'done', $remote_path . $backup_type_dir . basename($settings['_multipart_file']));
pb_backupbuddy::status('details', 'Notifying Stash of completed multipart upload with done url `' . $done_url . '`.');
$request = new RequestCore($done_url);
$response = $request->send_request(true);
if (!$response->isOK()) {
$this_error = 'Error #756834682. Could not finalize Stash upload. Response code: `' . $response->get_response_code() . '`; Response body: `' . $response->get_response_body() . '`; Response headers: `' . $response->get_response_header() . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
// Good server response.
// See if we got an optional json response.
$upload_data = @json_decode($response->body, true);
if (isset($upload_data['error'])) {
$this_error = 'Stash error(s): `' . implode(' - ', $upload_data['error']) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
pb_backupbuddy::status('details', 'Stash success sending file `' . basename($settings['_multipart_file']) . '`. File uploaded via multipart across `' . $this_part_number . '` parts and reported to Stash as completed.');
}
pb_backupbuddy::status('details', 'Stash has no more parts left for this multipart upload. Clearing multipart instance variables.');
$settings['_multipart_partnumber'] = 0;
$settings['_multipart_id'] = '';
$settings['_multipart_file'] = '';
$settings['_multipart_upload_data'] = array();
$settings['_multipart_transferspeeds'][] = $uploaded_speed;
// Overall upload speed average.
$uploaded_speed = array_sum($settings['_multipart_transferspeeds']) / count($settings['_multipart_counts']);
pb_backupbuddy::status('details', 'Upload speed average of all chunks: `' . pb_backupbuddy::$format->file_size($uploaded_speed) . '`.');
$settings['_multipart_counts'] = array();
// Update stats.
$fileoptions['_multipart_status'] = $update_status;
$fileoptions['finish_time'] = time();
$fileoptions['status'] = 'success';
if (isset($uploaded_speed)) {
$fileoptions['write_speed'] = $uploaded_speed;
}
$fileoptions_obj->save();
unset($fileoptions);
}
delete_transient('pb_backupbuddy_stashquota_' . $settings['itxapi_username']);
// Delete quota transient since it probably has changed now.
// Schedule to continue if anything is left to upload for this multipart of any individual files.
if ($settings['_multipart_id'] != '' || count($files) > 0) {
pb_backupbuddy::status('details', 'Stash multipart upload has more parts left. Scheduling next part send.');
$schedule_result = backupbuddy_core::schedule_single_event(time(), pb_backupbuddy::cron_tag('destination_send'), array($settings, $files, $send_id));
if (true === $schedule_result) {
pb_backupbuddy::status('details', 'Next Stash chunk step cron event scheduled.');
} else {
pb_backupbuddy::status('error', 'Next Stash chunk step cron even FAILED to be scheduled.');
}
spawn_cron(time() + 150);
// Adds > 60 seconds to get around once per minute cron running limit.
update_option('_transient_doing_cron', 0);
// Prevent cron-blocking for next item.
return array($settings['_multipart_id'], 'Sent part ' . $this_part_number . ' of ' . count($settings['_multipart_counts']) . ' parts.');
}
}
// end if multipart continuation.
require_once pb_backupbuddy::plugin_path() . '/classes/fileoptions.php';
// Upload each file.
foreach ($files as $file_id => $file) {
// Determine backup type directory (if zip).
$backup_type_dir = '';
$backup_type = '';
if (stristr($file, '.zip') !== false) {
// If a zip try to determine backup type.
pb_backupbuddy::status('details', 'Stash: Zip file. Detecting backup type if possible.');
$serial = backupbuddy_core::get_serial_from_file($file);
// See if we can get backup type from fileoptions data.
$backup_options = new pb_backupbuddy_fileoptions(backupbuddy_core::getLogDirectory() . 'fileoptions/' . $serial . '.txt', $read_only = true, $ignore_lock = true);
if (true !== ($result = $backup_options->is_ok())) {
pb_backupbuddy::status('error', 'Unable to open fileoptions file `' . backupbuddy_core::getLogDirectory() . 'fileoptions/' . $serial . '.txt' . '`.');
} else {
if (isset($backup_options->options['integrity']['detected_type'])) {
pb_backupbuddy::status('details', 'Stash: Detected backup type as `' . $backup_options->options['integrity']['detected_type'] . '` via integrity check data.');
$backup_type_dir = $backup_options->options['integrity']['detected_type'] . '/';
$backup_type = $backup_options->options['integrity']['detected_type'];
}
}
// If still do not know backup type then attempt to deduce it from filename.
if ($backup_type == '') {
if (stristr($file, '-db-') !== false) {
pb_backupbuddy::status('details', 'Stash: Detected backup type as `db` via filename.');
$backup_type_dir = 'db/';
$backup_type = 'db';
} elseif (stristr($file, '-full-') !== false) {
pb_backupbuddy::status('details', 'Stash: Detected backup type as `full` via filename.');
$backup_type_dir = 'full/';
$backup_type = 'full';
} elseif (stristr($file, '-files-') !== false) {
pb_backupbuddy::status('details', 'Stash: Detected backup type as `files` via filename.');
$backup_type_dir = 'files/';
$backup_type = 'files';
} else {
pb_backupbuddy::status('details', 'Stash: Could not detect backup type via integrity details nor filename.');
}
}
}
// Interact with Stash API.
pb_backupbuddy::status('details', 'Determining Stash upload URL for `' . $file . '`.` with destination remote path `' . $remote_path . $backup_type_dir . basename($file) . '`.');
$upload_url = $stash->get_upload_url($file, 'request', $remote_path . $backup_type_dir . basename($file));
pb_backupbuddy::status('details', 'Determined upload url: `' . $upload_url . '`.');
$request = new RequestCore($upload_url);
pb_backupbuddy::status('details', 'Sending Stash API request.');
$response = $request->send_request(true);
// Validate response.
if (!$response->isOK()) {
$this_error = 'Stash request for upload credentials failed.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
if (!($upload_data = json_decode($response->body, true))) {
$this_error = 'Stash API did not give a valid JSON response.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
if (isset($upload_data['error'])) {
$this_error = 'Stash error(s): `' . implode(' - ', $upload_data['error']) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
// Create S3 instance.
pb_backupbuddy::status('details', 'Creating Stash S3 instance.');
$s3 = new AmazonS3($upload_data['credentials']);
// the key, secret, token
if ($disable_ssl === true) {
@$s3->disable_ssl(true);
}
pb_backupbuddy::status('details', 'Stash S3 instance created.');
// Handle chunking of file into a multipart upload (if applicable).
$file_size = filesize($file);
if ($max_chunk_size >= self::MINIMUM_CHUNK_SIZE && $file_size / 1024 / 1024 > $max_chunk_size) {
// minimum chunk size is 5mb. Anything under 5mb we will not chunk.
pb_backupbuddy::status('details', 'Stash file size of ' . pb_backupbuddy::$format->file_size($file_size) . ' exceeds max chunk size of ' . $max_chunk_size . 'MB set in settings for sending file as multipart upload.');
// Initiate multipart upload with S3.
pb_backupbuddy::status('details', 'Initiating Stash multipart upload.');
$response = $s3->initiate_multipart_upload($upload_data['bucket'], $upload_data['object'], array('encryption' => 'AES256'));
if (!$response->isOK()) {
$this_error = 'Stash was unable to initiate multipart upload.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
$upload_id = (string) $response->body->UploadId;
pb_backupbuddy::status('details', 'Stash initiated multipart upload with ID `' . $upload_id . '`.');
}
// Get chunk parts for multipart transfer.
pb_backupbuddy::status('details', 'Stash getting multipart counts.');
$parts = $s3->get_multipart_counts($file_size, $max_chunk_size * 1024 * 1024);
// Size of chunks expected to be in bytes.
$multipart_destination_settings = $settings;
$multipart_destination_settings['_multipart_id'] = $upload_id;
$multipart_destination_settings['_multipart_partnumber'] = 0;
$multipart_destination_settings['_multipart_file'] = $file;
$multipart_destination_settings['_multipart_counts'] = $parts;
$multipart_destination_settings['_multipart_upload_data'] = $upload_data;
$multipart_destination_settings['_multipart_backup_type_dir'] = $backup_type_dir;
pb_backupbuddy::status('details', 'Stash multipart settings to pass:'******'details', 'Stash scheduling send of next part(s).');
backupbuddy_core::schedule_single_event(time(), pb_backupbuddy::cron_tag('destination_send'), array($multipart_destination_settings, $files, $send_id));
spawn_cron(time() + 150);
// Adds > 60 seconds to get around once per minute cron running limit.
update_option('_transient_doing_cron', 0);
// Prevent cron-blocking for next item.
pb_backupbuddy::status('details', 'Stash scheduled send of next part(s). Done for this cycle.');
return array($upload_id, 'Starting send of ' . count($multipart_destination_settings['_multipart_counts']) . ' parts.');
} else {
// did not meet chunking criteria.
if ($max_chunk_size != '0') {
if ($file_size / 1024 / 1024 > self::MINIMUM_CHUNK_SIZE) {
pb_backupbuddy::status('details', 'File size of ' . pb_backupbuddy::$format->file_size($file_size) . ' is less than the max chunk size of ' . $max_chunk_size . 'MB; not chunking into multipart upload.');
} else {
pb_backupbuddy::status('details', 'File size of ' . pb_backupbuddy::$format->file_size($file_size) . ' is less than the minimum allowed chunk size of ' . self::MINIMUM_CHUNK_SIZE . 'MB; not chunking into multipart upload.');
}
} else {
pb_backupbuddy::status('details', 'Max chunk size set to zero so not chunking into multipart upload.');
}
}
// SEND file.
pb_backupbuddy::status('details', 'About to put (upload) object to Stash.');
$response = $s3->create_object($upload_data['bucket'], $upload_data['object'], array('fileUpload' => $file, 'encryption' => 'AES256'));
// Validate response. On failure notify Stash API that things went wrong.
if (!$response->isOK()) {
// Send FAILED.
pb_backupbuddy::status('details', 'Sending upload abort.');
$request = new RequestCore($abort_url);
$response = $request->send_request(true);
$this_error = 'Could not upload to Stash, attempt aborted.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
// Send SUCCESS.
pb_backupbuddy::status('details', 'Success uploading file to Stash storage. Notifying Stash API next. Upload details: `' . print_r($response, true) . '`.');
$uploaded_size = $response->header['_info']['size_upload'];
$uploaded_speed = $response->header['_info']['speed_upload'];
pb_backupbuddy::status('details', 'Uploaded size: ' . pb_backupbuddy::$format->file_size($uploaded_size) . ', Speed: ' . pb_backupbuddy::$format->file_size($uploaded_speed) . '/sec.');
}
delete_transient('pb_backupbuddy_stashquota_' . $settings['itxapi_username']);
// Delete quota transient since it probably has changed now.
// Notify Stash API that things were succesful.
$done_url = $stash->get_upload_url($file, 'done', $remote_path . $backup_type_dir . basename($file));
pb_backupbuddy::status('details', 'Notifying Stash of completed upload with done url `' . $done_url . '`.');
$request = new RequestCore($done_url);
$response = $request->send_request(true);
if (!$response->isOK()) {
$this_error = 'Error #247568834682. Could not finalize Stash upload. Response code: `' . $response->get_response_code() . '`; Response body: `' . $response->get_response_body() . '`; Response headers: `' . $response->get_response_header() . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
// Good server response.
// See if we got an optional json response.
$upload_data = @json_decode($response->body, true);
if (isset($upload_data['error'])) {
// Some kind of error.
$this_error = 'Stash error(s): `' . implode(' - ', $upload_data['error']) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
unset($files[$file_id]);
// Remove from list of files we have not sent yet.
pb_backupbuddy::status('details', 'Stash success sending file `' . basename($file) . '`. File uploaded and reported to Stash as completed.');
// Load destination fileoptions.
pb_backupbuddy::status('details', 'About to load fileoptions data.');
require_once pb_backupbuddy::plugin_path() . '/classes/fileoptions.php';
$fileoptions_obj = new pb_backupbuddy_fileoptions(backupbuddy_core::getLogDirectory() . 'fileoptions/send-' . $send_id . '.txt', $read_only = false, $ignore_lock = false, $create_file = false);
if (true !== ($result = $fileoptions_obj->is_ok())) {
pb_backupbuddy::status('error', __('Fatal Error #9034.84838. Unable to access fileoptions data.', 'it-l10n-backupbuddy') . ' Error: ' . $result);
return false;
}
pb_backupbuddy::status('details', 'Fileoptions data loaded.');
$fileoptions =& $fileoptions_obj->options;
// Save stats.
if (isset($uploaded_speed)) {
$fileoptions['write_speed'] = $uploaded_speed;
$fileoptions_obj->save();
}
//$fileoptions['finish_time'] = time();
//$fileoptions['status'] = 'success';
unset($fileoptions_obj);
}
}
// end foreach.
// BEGIN FILE LIMIT PROCESSING. Enforce archive limits if applicable.
if ($backup_type == 'full') {
$limit = $full_archive_limit;
pb_backupbuddy::status('details', 'Stash full backup archive limit of `' . $limit . '` of type `full` based on destination settings.');
} elseif ($backup_type == 'db') {
$limit = $db_archive_limit;
pb_backupbuddy::status('details', 'Stash database backup archive limit of `' . $limit . '` of type `db` based on destination settings.');
} elseif ($backup_type == 'files') {
$limit = $db_archive_limit;
pb_backupbuddy::status('details', 'Stash database backup archive limit of `' . $limit . '` of type `files` based on destination settings.');
} else {
$limit = 0;
pb_backupbuddy::status('warning', 'Warning #54854895. Stash was unable to determine backup type (reported: `' . $backup_type . '`) so archive limits NOT enforced for this backup.');
}
if ($limit > 0) {
pb_backupbuddy::status('details', 'Stash archive limit enforcement beginning.');
// S3 object for managing files.
$s3_manage = new AmazonS3($manage_data['credentials']);
if ($disable_ssl === true) {
@$s3_manage->disable_ssl(true);
}
// Get file listing.
$response_manage = $s3_manage->list_objects($manage_data['bucket'], array('prefix' => $manage_data['subkey'] . $remote_path . $backup_type_dir));
// list all the files in the subscriber account
// Create array of backups and organize by date
$prefix = backupbuddy_core::backup_prefix();
// List backups associated with this site by date.
$backups = array();
foreach ($response_manage->body->Contents as $object) {
$file = str_replace($manage_data['subkey'] . $remote_path . $backup_type_dir, '', $object->Key);
// Stash stores files in a directory per site so no need to check prefix here! if ( false !== strpos( $file, 'backup-' . $prefix . '-' ) ) { // if backup has this site prefix...
$backups[$file] = strtotime($object->LastModified);
}
arsort($backups);
pb_backupbuddy::status('details', 'Stash found `' . count($backups) . '` backups of this type when checking archive limits.');
if (count($backups) > $limit) {
pb_backupbuddy::status('details', 'More archives (' . count($backups) . ') than limit (' . $limit . ') allows. Trimming...');
$i = 0;
$delete_fail_count = 0;
foreach ($backups as $buname => $butime) {
$i++;
if ($i > $limit) {
pb_backupbuddy::status('details', 'Trimming excess file `' . $buname . '`...');
$response = $s3_manage->delete_object($manage_data['bucket'], $manage_data['subkey'] . $remote_path . $backup_type_dir . $buname);
if (!$response->isOK()) {
pb_backupbuddy::status('details', 'Unable to delete excess Stash file `' . $buname . '`. Details: `' . print_r($response, true) . '`.');
$delete_fail_count++;
}
}
}
pb_backupbuddy::status('details', 'Finished trimming excess backups.');
if ($delete_fail_count !== 0) {
$error_message = 'Stash remote limit could not delete ' . $delete_fail_count . ' backups.';
pb_backupbuddy::status('error', $error_message);
backupbuddy_core::mail_error($error_message);
}
}
pb_backupbuddy::status('details', 'Stash completed archive limiting.');
} else {
pb_backupbuddy::status('details', 'No Stash archive file limit to enforce.');
}
// End remote backup limit
if (isset($fileoptions_obj)) {
unset($fileoptions_obj);
}
// END FILE LIMIT PROCESSING.
// Success if we made it this far.
return true;
}
public static function send($settings = array(), $files = array(), $clear_uploads = false)
{
global $pb_backupbuddy_destination_errors;
if (!is_array($files)) {
$files = array($files);
}
if ($clear_uploads === false) {
// Uncomment the following line to override and always clear.
//$clear_uploads = true;
}
$itxapi_username = $settings['itxapi_username'];
$itxapi_password = $settings['itxapi_password'];
$db_archive_limit = $settings['db_archive_limit'];
$full_archive_limit = $settings['full_archive_limit'];
$max_chunk_size = $settings['max_chunk_size'];
$remote_path = self::get_remote_path($settings['directory']);
// Has leading and trailng slashes.
if ($settings['ssl'] == '0') {
$disable_ssl = true;
} else {
$disable_ssl = false;
}
$multipart_id = $settings['_multipart_id'];
$multipart_counts = $settings['_multipart_counts'];
pb_backupbuddy::status('details', 'Stash remote path set to `' . $remote_path . '`.');
require_once dirname(__FILE__) . '/lib/class.itx_helper.php';
require_once dirname(__FILE__) . '/lib/aws-sdk/sdk.class.php';
// Stash API talk.
$stash = new ITXAPI_Helper(pb_backupbuddy_destination_stash::ITXAPI_KEY, pb_backupbuddy_destination_stash::ITXAPI_URL, $itxapi_username, $itxapi_password);
$manage_data = pb_backupbuddy_destination_stash::get_manage_data($settings);
// Wipe all current uploads.
if ($clear_uploads === true) {
pb_backupbuddy::status('details', 'Clearing any current uploads via Stash call to `abort-all`.');
$abort_url = $stash->get_upload_url(null, 'abort-all');
$request = new RequestCore($abort_url);
//pb_backupbuddy::status('details', print_r( $request , true ) );
$response = $request->send_request(true);
}
// Process multipart transfer that we already initiated in a previous PHP load.
if ($multipart_id != '') {
// Multipart upload initiated and needs parts sent.
// Create S3 instance.
pb_backupbuddy::status('details', 'Creating Stash S3 instance.');
$s3 = new AmazonS3($settings['_multipart_upload_data']['credentials']);
// the key, secret, token
if ($disable_ssl === true) {
@$s3->disable_ssl(true);
}
pb_backupbuddy::status('details', 'Stash S3 instance created.');
$this_part_number = $settings['_multipart_partnumber'] + 1;
pb_backupbuddy::status('details', 'Stash beginning upload of part `' . $this_part_number . '` of `' . count($settings['_multipart_counts']) . '` parts of file `' . $settings['_multipart_file'] . '` with multipart ID `' . $settings['_multipart_id'] . '`.');
$response = $s3->upload_part($settings['_multipart_upload_data']['bucket'], $settings['_multipart_upload_data']['object'], $settings['_multipart_id'], array('expect' => '100-continue', 'fileUpload' => $settings['_multipart_file'], 'partNumber' => $this_part_number, 'seekTo' => (int) $settings['_multipart_counts'][$settings['_multipart_partnumber']]['seekTo'], 'length' => (int) $settings['_multipart_counts'][$settings['_multipart_partnumber']]['length']));
if (!$response->isOK()) {
$this_error = 'Stash unable to upload file part for multipart upload `' . $settings['_multipart_id'] . '`. Details: `' . print_r($response, true) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
// Update stats.
foreach (pb_backupbuddy::$options['remote_sends'] as $identifier => $remote_send) {
if (isset($remote_send['_multipart_id']) && $remote_send['_multipart_id'] == $multipart_id) {
// this item.
pb_backupbuddy::$options['remote_sends'][$identifier]['_multipart_status'] = 'Sent part ' . $this_part_number . ' of ' . count($settings['_multipart_counts']) . '.';
if ($this_part_number == count($settings['_multipart_counts'])) {
pb_backupbuddy::$options['remote_sends'][$identifier]['_multipart_status'] .= '<br>Success.';
pb_backupbuddy::$options['remote_sends'][$identifier]['finish_time'] = time();
}
pb_backupbuddy::save();
break;
}
}
// Made it here so success sending part. Increment for next part to send.
$settings['_multipart_partnumber']++;
if (!isset($settings['_multipart_counts'][$settings['_multipart_partnumber']])) {
// No more parts exist for this file. Tell S3 the multipart upload is complete and move on.
pb_backupbuddy::status('details', 'Stash getting parts with etags to notify S3 of completed multipart send.');
$etag_parts = $s3->list_parts($settings['_multipart_upload_data']['bucket'], $settings['_multipart_upload_data']['object'], $settings['_multipart_id']);
pb_backupbuddy::status('details', 'Stash got parts list. Notifying S3 of multipart upload completion.');
$response = $s3->complete_multipart_upload($settings['_multipart_upload_data']['bucket'], $settings['_multipart_upload_data']['object'], $settings['_multipart_id'], $etag_parts);
if (!$response->isOK()) {
$this_error = 'Stash unable to notify S3 of completion of all parts for multipart upload `' . $settings['_multipart_id'] . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
pb_backupbuddy::status('details', 'Stash notified S3 of multipart completion.');
}
// Notify Stash API that things were succesful.
$done_url = $stash->get_upload_url($settings['_multipart_file'], 'done', $remote_path . $settings['_multipart_backup_type_dir'] . basename($settings['_multipart_file']));
pb_backupbuddy::status('details', 'Notifying Stash of completed multipart upload with done url `' . $done_url . '`.');
$request = new RequestCore($done_url);
$response = $request->send_request(true);
if (!$response->isOK()) {
$this_error = 'Error #756834682. Could not finalize Stash upload. Response code: `' . $response->get_response_code() . '`; Response body: `' . $response->get_response_body() . '`; Response headers: `' . $response->get_response_header() . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
// Good server response.
// See if we got an optional json response.
$upload_data = @json_decode($response->body, true);
if (isset($upload_data['error'])) {
$this_error = 'Stash error(s): `' . implode(' - ', $upload_data['error']) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
pb_backupbuddy::status('details', 'Stash success sending file `' . basename($settings['_multipart_file']) . '`. File uploaded via multipart across `' . $this_part_number . '` parts and reported to Stash as completed.');
}
pb_backupbuddy::status('details', 'Stash has no more parts left for this multipart upload. Clearing multipart instance variables.');
$settings['_multipart_partnumber'] = 0;
$settings['_multipart_id'] = '';
$settings['_multipart_file'] = '';
$settings['_multipart_counts'] = array();
$settings['_multipart_upload_data'] = array();
}
delete_transient('pb_backupbuddy_stashquota_' . $settings['itxapi_username']);
// Delete quota transient since it probably has changed now.
// Schedule to continue if anything is left to upload for this multipart of any individual files.
if ($settings['_multipart_id'] != '' || count($files) > 0) {
pb_backupbuddy::status('details', 'Stash multipart upload has more parts left. Scheduling next part send.');
wp_schedule_single_event(time(), pb_backupbuddy::cron_tag('destination_send'), array($settings, $files, 'multipart', false));
spawn_cron(time() + 150);
// Adds > 60 seconds to get around once per minute cron running limit.
update_option('_transient_doing_cron', 0);
// Prevent cron-blocking for next item.
pb_backupbuddy::status('details', 'Stash scheduled send of next part(s). Done for this cycle.');
return array($settings['_multipart_id'], 'Sent ' . $this_part_number . ' of ' . count($multipart_destination_settings['_multipart_counts'] . ' parts.'));
}
}
// Upload each file.
foreach ($files as $file_id => $file) {
// Determine backup type directory (if zip).
$backup_type_dir = '';
$backup_type = '';
if (stristr($file, '.zip') !== false) {
// If a zip try to determine backup type.
pb_backupbuddy::status('details', 'Stash: Zip file. Detecting backup type if possible.');
$serial = pb_backupbuddy::$classes['core']->get_serial_from_file($file);
if (isset(pb_backupbuddy::$options['backups'][$serial]['integrity']['detected_type'])) {
pb_backupbuddy::status('details', 'Stash: Detected backup type as `' . pb_backupbuddy::$options['backups'][$serial]['integrity']['detected_type'] . '` via integrity check data.');
$backup_type_dir = pb_backupbuddy::$options['backups'][$serial]['integrity']['detected_type'] . '/';
$backup_type = pb_backupbuddy::$options['backups'][$serial]['integrity']['detected_type'];
} else {
if (stristr($file, '-db-') !== false) {
pb_backupbuddy::status('details', 'Stash: Detected backup type as `db` via filename.');
$backup_type_dir = 'db/';
$backup_type = 'db';
} elseif (stristr($file, '-full-') !== false) {
pb_backupbuddy::status('details', 'Stash: Detected backup type as `full` via filename.');
$backup_type_dir = 'full/';
$backup_type = 'full';
} else {
pb_backupbuddy::status('details', 'Stash: Could not detect backup type via integrity details nor filename.');
}
}
}
// Interact with Stash API.
pb_backupbuddy::status('details', 'Determining Stash upload URL for `' . $file . '`.` with destination remote path `' . $remote_path . $backup_type_dir . basename($file) . '`.');
$upload_url = $stash->get_upload_url($file, 'request', $remote_path . $backup_type_dir . basename($file));
pb_backupbuddy::status('details', 'Determined upload url: `' . $upload_url . '`.');
$request = new RequestCore($upload_url);
pb_backupbuddy::status('details', 'Sending Stash API request.');
$response = $request->send_request(true);
// Validate response.
if (!$response->isOK()) {
$this_error = 'Stash request for upload credentials failed.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
if (!($upload_data = json_decode($response->body, true))) {
$this_error = 'Stash API did not give a valid JSON response.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
if (isset($upload_data['error'])) {
$this_error = 'Stash error(s): `' . implode(' - ', $upload_data['error']) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
// Calculate meta data to send.
/*
$meta_array = array();
if ( stristr( $file, '.zip' ) !== false ) { // If a zip try to determine backup type.
pb_backupbuddy::status( 'details', 'Stash: Zip file. Detecting backup type if possible.' );
$serial = pb_backupbuddy::$classes['core']->get_serial_from_file( $file );
if ( isset( pb_backupbuddy::$options['backups'][$serial]['integrity']['detected_type'] ) ) {
pb_backupbuddy::status( 'details', 'Stash: Detected backup type as `' . pb_backupbuddy::$options['backups'][$serial]['integrity']['detected_type'] . '` via integrity check data.' );
$meta_array['backup_type'] = pb_backupbuddy::$options['backups'][$serial]['integrity']['detected_type'];
} else {
if ( stristr( $file, '-db-' ) !== false ) {
pb_backupbuddy::status( 'details', 'Stash: Detected backup type as `db` via filename.' );
$meta_array['backup_type'] = 'db';
} elseif ( stristr( $file, '-full-' ) !== false ) {
pb_backupbuddy::status( 'details', 'Stash: Detected backup type as `full` via filename.' );
$meta_array['backup_type'] = 'full';
} else {
pb_backupbuddy::status( 'details', 'Stash: Could not detect backup type via integrity details nor filename.' );
}
}
}
*/
// Create S3 instance.
pb_backupbuddy::status('details', 'Creating Stash S3 instance.');
$s3 = new AmazonS3($upload_data['credentials']);
// the key, secret, token
if ($disable_ssl === true) {
@$s3->disable_ssl(true);
}
pb_backupbuddy::status('details', 'Stash S3 instance created.');
// Handle chunking of file into a multipart upload (if applicable).
$file_size = filesize($file);
if ($max_chunk_size >= 5 && $file_size / 1024 / 1024 > $max_chunk_size) {
// minimum chunk size is 5mb. Anything under 5mb we will not chunk.
pb_backupbuddy::status('details', 'Stash file size of ' . $file_size / 1024 / 1024 . 'MB exceeds max chunk size of ' . $max_chunk_size . 'MB set in settings for sending file as multipart upload.');
// Initiate multipart upload with S3.
pb_backupbuddy::status('details', 'Initiating Stash multipart upload.');
$response = $s3->initiate_multipart_upload($upload_data['bucket'], $upload_data['object'], array('encryption' => 'AES256'));
if (!$response->isOK()) {
$this_error = 'Stash was unable to initiate multipart upload.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
$upload_id = (string) $response->body->UploadId;
pb_backupbuddy::status('details', 'Stash initiated multipart upload with ID `' . $upload_id . '`.');
}
// Get chunk parts for multipart transfer.
pb_backupbuddy::status('details', 'Stash getting multipart counts.');
$parts = $s3->get_multipart_counts($file_size, $max_chunk_size * 1024 * 1024);
// Size of chunks expected to be in bytes.
$multipart_destination_settings = $settings;
$multipart_destination_settings['_multipart_id'] = $upload_id;
$multipart_destination_settings['_multipart_partnumber'] = 0;
$multipart_destination_settings['_multipart_file'] = $file;
$multipart_destination_settings['_multipart_counts'] = $parts;
$multipart_destination_settings['_multipart_upload_data'] = $upload_data;
$multipart_destination_settings['_multipart_backup_type_dir'] = $backup_type_dir;
pb_backupbuddy::status('details', 'Stash multipart settings to pass:'******'details', 'Stash scheduling send of next part(s).');
wp_schedule_single_event(time(), pb_backupbuddy::cron_tag('destination_send'), array($multipart_destination_settings, $files, 'multipart', false));
spawn_cron(time() + 150);
// Adds > 60 seconds to get around once per minute cron running limit.
update_option('_transient_doing_cron', 0);
// Prevent cron-blocking for next item.
pb_backupbuddy::status('details', 'Stash scheduled send of next part(s). Done for this cycle.');
return array($upload_id, 'Starting send of ' . count($multipart_destination_settings['_multipart_counts']) . ' parts.');
} else {
if ($max_chunk_size != '0') {
pb_backupbuddy::status('details', 'File size of ' . $file_size / 1024 / 1024 . 'MB is less than the max chunk size of ' . $max_chunk_size . 'MB; not chunking into multipart upload.');
} else {
pb_backupbuddy::status('details', 'Max chunk size set to zero so not chunking into multipart upload.');
}
}
// SEND file.
pb_backupbuddy::status('details', 'About to put (upload) object to Stash.');
$response = $s3->create_object($upload_data['bucket'], $upload_data['object'], array('fileUpload' => $file, 'encryption' => 'AES256'));
// we can also utilize the multi-part-upload to create an object
// $response = $s3->create_mpu_object($upload_data['bucket'], $upload_data['object'], array('fileUpload'=>$upload_file));
// Validate response. On failure notify Stash API that things went wrong.
if (!$response->isOK()) {
pb_backupbuddy::status('details', 'Sending upload abort.');
$request = new RequestCore($abort_url);
$response = $request->send_request(true);
$this_error = 'Could not upload to Stash, attempt aborted.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
// pb_backupbuddy::status( 'details', 'Stash file upload speed: ' . ( $response->header['_info']['speed_upload'] / 1024 / 1024 ) . 'MB/sec. This number may be invalid for small file transfers.' );
pb_backupbuddy::status('details', 'Stash put success. Need to nofity Stash of upload completion. Details: `' . print_r($response, true) . '`.');
}
delete_transient('pb_backupbuddy_stashquota_' . $settings['itxapi_username']);
// Delete quota transient since it probably has changed now.
// Notify Stash API that things were succesful.
$done_url = $stash->get_upload_url($file, 'done', $remote_path . $backup_type_dir . basename($file));
pb_backupbuddy::status('details', 'Notifying Stash of completed upload with done url `' . $done_url . '`.');
$request = new RequestCore($done_url);
$response = $request->send_request(true);
if (!$response->isOK()) {
$this_error = 'Error #756834682. Could not finalize Stash upload. Response code: `' . $response->get_response_code() . '`; Response body: `' . $response->get_response_body() . '`; Response headers: `' . $response->get_response_header() . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
} else {
// Good server response.
// See if we got an optional json response.
$upload_data = @json_decode($response->body, true);
if (isset($upload_data['error'])) {
// Some kind of error.
$this_error = 'Stash error(s): `' . implode(' - ', $upload_data['error']) . '`.';
$pb_backupbuddy_destination_errors[] = $this_error;
pb_backupbuddy::status('error', $this_error);
return false;
}
unset($files[$file_id]);
// Remove from list of files we have not sent yet.
pb_backupbuddy::status('details', 'Stash success sending file `' . basename($file) . '`. File uploaded and reported to Stash as completed.');
}
// Enforce archive limits if applicable.
if ($backup_type == 'full') {
$limit = $full_archive_limit;
pb_backupbuddy::status('details', 'Stash full backup archive limit of `' . $limit . '` based on destination settings.');
} elseif ($backup_type == 'db') {
$limit = $db_archive_limit;
pb_backupbuddy::status('details', 'Stash database backup archive limit of `' . $limit . '` based on destination settings.');
} else {
$limit = 0;
pb_backupbuddy::status('error', 'Error #54854895. Stash was unable to determine backup type so archive limits NOT enforced for this backup.');
}
if ($limit > 0) {
pb_backupbuddy::status('details', 'Stash archive limit enforcement beginning.');
// S3 object for managing files.
$s3_manage = new AmazonS3($manage_data['credentials']);
if ($disable_ssl === true) {
@$s3_manage->disable_ssl(true);
}
// Get file listing.
$response_manage = $s3_manage->list_objects($manage_data['bucket'], array('prefix' => $manage_data['subkey'] . $remote_path . $backup_type_dir));
// list all the files in the subscriber account
// Create array of backups and organize by date
$prefix = pb_backupbuddy::$classes['core']->backup_prefix();
// List backups associated with this site by date.
$backups = array();
foreach ($response_manage->body->Contents as $object) {
$file = str_replace($manage_data['subkey'] . $remote_path . $backup_type_dir, '', $object->Key);
// Stash stores files in a directory per site so no need to check prefix here! if ( false !== strpos( $file, 'backup-' . $prefix . '-' ) ) { // if backup has this site prefix...
$backups[$file] = strtotime($object->LastModified);
//}
}
arsort($backups);
//error_log( 'backups: ' . print_r( $backups, true ) );
pb_backupbuddy::status('details', 'Stash found `' . count($backups) . '` backups of this type when checking archive limits.');
if (count($backups) > $limit) {
pb_backupbuddy::status('details', 'More archives (' . count($backups) . ') than limit (' . $limit . ') allows. Trimming...');
$i = 0;
$delete_fail_count = 0;
foreach ($backups as $buname => $butime) {
$i++;
if ($i > $limit) {
pb_backupbuddy::status('details', 'Trimming excess file `' . $buname . '`...');
$response = $s3_manage->delete_object($manage_data['bucket'], $manage_data['subkey'] . $remote_path . $backup_type_dir . $buname);
if (!$response->isOK()) {
pb_backupbuddy::status('details', 'Unable to delete excess Stash file `' . $buname . '`. Details: `' . print_r($response, true) . '`.');
$delete_fail_count++;
}
}
}
pb_backupbuddy::status('details', 'Finished trimming excess backups.');
if ($delete_fail_count !== 0) {
$error_message = 'Stash remote limit could not delete ' . $delete_fail_count . ' backups.';
pb_backupbuddy::status('error', $error_message);
pb_backupbuddy::$classes['core']->mail_error($error_message);
}
}
pb_backupbuddy::status('details', 'Stash completed archive limiting.');
} else {
pb_backupbuddy::status('details', 'No Stash archive file limit to enforce.');
}
// End remote backup limit
}
// end foreach.
// Success if we made it this far.
return true;
}
