public static function send($settings = array(), $files = array(), $send_id = '', $delete_after = false, $delete_remote_after = false) { global $pb_backupbuddy_destination_errors; if ('1' == $settings['disabled']) { $pb_backupbuddy_destination_errors[] = __('Error #48933: This destination is currently disabled. Enable it under this destination\'s Advanced Settings.', 'it-l10n-backupbuddy'); return false; } if (!is_array($files)) { $files = array($files); } pb_backupbuddy::status('details', 'Google Drive send() function started. Settings: `' . print_r($settings, true) . '`.'); self::$_timeStart = microtime(true); $settings = self::_normalizeSettings($settings); if (false === ($settings = self::_connect($settings))) { // $settings = return self::_error('Error #38923923: Unable to connect with Google Drive. See log for details.'); } $folderID = $settings['folderID']; if ('' == $folderID) { $folderID = 'root'; } $chunkSizeBytes = $settings['max_burst'] * 1024 * 1024; // Send X mb at a time to limit memory usage. foreach ($files as $file) { // Determine backup type for limiting later. $backup_type = ''; if (stristr($file, '-db-') !== false) { $backup_type = 'db'; } elseif (stristr($file, '-full-') !== false) { $backup_type = 'full'; } elseif (stristr($file, '-files-') !== false) { $backup_type = 'files'; } elseif (stristr($file, '-export-') !== false) { $backup_type = 'export'; } if (!file_exists($file)) { return self::_error('Error #37792: File selected to send not found: `' . $file . '`.'); } $fileSize = filesize($file); $fileinfo = pathinfo($file); $fileextension = $fileinfo['extension']; if ('zip' == $fileextension) { $mimeType = 'application/zip'; } elseif ('php' == $fileextension) { $mimeType = 'application/x-httpd-php'; } else { $mimeType = ''; } pb_backupbuddy::status('details', 'About to upload file `' . $file . '` of size `' . $fileSize . '` with mimetype `' . $mimeType . '` into folder `' . $folderID . '`. Internal chunk size of `' . $chunkSizeBytes . '` bytes.'); if ($fileSize > $chunkSizeBytes) { pb_backupbuddy::status('details', 'File size `' . pb_backupbuddy::$format->file_size($fileSize) . '` exceeds max burst size `' . $settings['max_burst'] . ' MB` so this will be sent in bursts. If time limit nears then send will be chunked across multiple PHP loads.'); $settings['_chunks_total'] = ceil($fileSize / $chunkSizeBytes); } if (0 == $settings['_chunks_total']) { $settings['_chunks_total'] = 1; } //Insert a file $driveFile = new Google_Service_Drive_DriveFile(); $driveFile->setTitle(basename($file)); $driveFile->setDescription('BackupBuddy file'); $driveFile->setMimeType($mimeType); // Set the parent folder. if ('root' != $folderID) { $parentsCollectionData = new Google_Service_Drive_ParentReference(); $parentsCollectionData->setId($folderID); $driveFile->setParents(array($parentsCollectionData)); } self::$_client->setDefer(true); try { $insertRequest = self::$_drive->files->insert($driveFile); } catch (Exception $e) { pb_backupbuddy::alert('Error #3232783268336: initiating upload. Details: ' . $e->getMessage()); return false; } // Handle getting resume information to see if resuming is still an option. $resumable = false; if ('' != $settings['_media_resumeUri']) { $headers = array('content-range' => 'bytes */' . $fileSize); $request = new Google_Http_Request($settings['_media_resumeUri'], 'PUT', $headers, ''); $response = self::$_client->getIo()->makeRequest($request); if (308 == $response->getResponseHttpCode()) { $range = $response->getResponseHeader('range'); if (!empty($range) && preg_match('/bytes=0-(\\d+)$/', $range, $matches)) { $resumable = true; pb_backupbuddy::status('details', 'Last send reported next byte to be `' . $settings['_media_progress'] . '`.'); $settings['_media_progress'] = $matches[1] + 1; pb_backupbuddy::status('details', 'Google Drive resuming is available. Google Drive reports next byte to be `' . $settings['_media_progress'] . '`. Range: `' . $range . '`.'); } } if (!$resumable) { pb_backupbuddy::status('details', 'Google Drive could not resume. Too much time may have passed or some other cause.'); } if ($settings['_media_progress'] >= $fileSize) { pb_backupbuddy::status('details', 'Google Drive resuming not needed. Remote file meets or exceeds file size. Completed.'); return true; } } // See https://developers.google.com/api-client-library/php/guide/media_upload try { $media = new Google_Http_MediaFileUpload(self::$_client, $insertRequest, $mimeType, null, true, $chunkSizeBytes); } catch (Exception $e) { pb_backupbuddy::alert('Error #3893273937: initiating upload. Details: ' . $e->getMessage()); return; } $media->setFileSize($fileSize); // Reset these internal variables. NOTE: These are by default private. Must modify MediaFileUpload.php to make this possible by setting these vars public. Thanks Google! if ('' != $settings['_media_resumeUri']) { $media->resumeUri = $settings['_media_resumeUri']; $media->progress = $settings['_media_progress']; } pb_backupbuddy::status('details', 'Opening file for sending in binary mode.'); $fs = fopen($file, 'rb'); // If chunked resuming then seek to the correct place in the file. if ('' != $settings['_media_progress'] && $settings['_media_progress'] > 0) { // Resuming send of a partially transferred file. if (0 !== fseek($fs, $settings['_media_progress'])) { // Go off the resume point as given by Google in case it didnt all make it. //$settings['resume_point'] ) ) { // Returns 0 on success. pb_backupbuddy::status('error', 'Error #3872733: Failed to seek file to resume point `' . $settings['_media_progress'] . '` via fseek().'); return false; } $prevPointer = $settings['_media_progress']; //$settings['resume_point']; } else { // New file send. $prevPointer = 0; } $needProcessChunking = false; // Set true if we need to spawn off resuming to a new PHP page load. $uploadStatus = false; while (!$uploadStatus && !feof($fs)) { $chunk = fread($fs, $chunkSizeBytes); pb_backupbuddy::status('details', 'Chunk of size `' . pb_backupbuddy::$format->file_size($chunkSizeBytes) . '` read into memory. Total bytes summed: `' . ($settings['_media_progress'] + strlen($chunk)) . '` of filesize: `' . $fileSize . '`.'); pb_backupbuddy::status('details', 'Sending burst file data next. If next message is not "Burst file data sent" then the send likely timed out. Try reducing burst size. Sending now...'); // Send chunk of data. try { $uploadStatus = $media->nextChunk($chunk); } catch (Exception $e) { global $pb_backupbuddy_destination_errors; $pb_backupbuddy_destination_errors[] = $e->getMessage(); $error = $e->getMessage(); pb_backupbuddy::status('error', 'Error #8239832: Error sending burst data. Details: `' . $error . '`.'); return false; } $settings['_chunks_sent']++; self::$_chunksSentThisRound++; pb_backupbuddy::status('details', 'Burst file data sent.'); $maxTime = $settings['max_time']; if ('' == $maxTime || !is_numeric($maxTime)) { pb_backupbuddy::status('details', 'Max time not set in settings so detecting server max PHP runtime.'); $maxTime = backupbuddy_core::detectMaxExecutionTime(); } //return; // Handle splitting up across multiple PHP page loads if needed. if (!feof($fs) && 0 != $maxTime) { // More data remains so see if we need to consider chunking to a new PHP process. // If we are within X second of reaching maximum PHP runtime then stop here so that it can be picked up in another PHP process... $totalSizeSent = self::$_chunksSentThisRound * $chunkSizeBytes; // Total bytes sent this PHP load. $bytesPerSec = $totalSizeSent / (microtime(true) - self::$_timeStart); $timeRemaining = $maxTime - (microtime(true) - self::$_timeStart + self::TIME_WIGGLE_ROOM); if ($timeRemaining < 0) { $timeRemaining = 0; } $bytesWeCouldSendWithTimeLeft = $bytesPerSec * $timeRemaining; pb_backupbuddy::status('details', 'Total sent: `' . pb_backupbuddy::$format->file_size($totalSizeSent) . '`. Speed (per sec): `' . pb_backupbuddy::$format->file_size($bytesPerSec) . '`. Time Remaining (w/ wiggle): `' . $timeRemaining . '`. Size that could potentially be sent with remaining time: `' . pb_backupbuddy::$format->file_size($bytesWeCouldSendWithTimeLeft) . '` with chunk size of `' . pb_backupbuddy::$format->file_size($chunkSizeBytes) . '`.'); if ($bytesWeCouldSendWithTimeLeft < $chunkSizeBytes) { // We can send more than a whole chunk (including wiggle room) so send another bit. pb_backupbuddy::status('message', 'Not enough time left (~`' . $timeRemaining . '`) with max time of `' . $maxTime . '` sec to send another chunk at `' . pb_backupbuddy::$format->file_size($bytesPerSec) . '` / sec. Ran for ' . round(microtime(true) - self::$_timeStart, 3) . ' sec. Proceeding to use chunking.'); @fclose($fs); // Tells next chunk where to pick up. if (isset($chunksTotal)) { $settings['_chunks_total'] = $chunksTotal; } // Grab these vars from the class. Note that we changed these vars from private to public to make chunked resuming possible. $settings['_media_resumeUri'] = $media->resumeUri; $settings['_media_progress'] = $media->progress; // Schedule cron. $cronTime = time(); $cronArgs = array($settings, $files, $send_id, $delete_after); $cronHashID = md5($cronTime . serialize($cronArgs)); $cronArgs[] = $cronHashID; $schedule_result = backupbuddy_core::schedule_single_event($cronTime, 'destination_send', $cronArgs); if (true === $schedule_result) { pb_backupbuddy::status('details', 'Next Site chunk step cron event scheduled.'); } else { pb_backupbuddy::status('error', 'Next Site chunk step cron even FAILED to be scheduled.'); } spawn_cron(time() + 150); // Adds > 60 seconds to get around once per minute cron running limit. update_option('_transient_doing_cron', 0); // Prevent cron-blocking for next item. return array($prevPointer, 'Sent part ' . $settings['_chunks_sent'] . ' of ~' . $settings['_chunks_total'] . ' parts.'); // filepointer location, elapsed time during the import } else { // End if. pb_backupbuddy::status('details', 'Not approaching limits.'); } } else { pb_backupbuddy::status('details', 'No more data remains (eg for chunking) so finishing up.'); if ('' != $send_id) { require_once pb_backupbuddy::plugin_path() . '/classes/fileoptions.php'; $fileoptions_obj = new pb_backupbuddy_fileoptions(backupbuddy_core::getLogDirectory() . 'fileoptions/send-' . $send_id . '.txt', $read_only = false, $ignore_lock = false, $create_file = false); if (true !== ($result = $fileoptions_obj->is_ok())) { pb_backupbuddy::status('error', __('Fatal Error #9034.397237. Unable to access fileoptions data.', 'it-l10n-backupbuddy') . ' Error: ' . $result); return false; } pb_backupbuddy::status('details', 'Fileoptions data loaded.'); $fileoptions =& $fileoptions_obj->options; $fileoptions['_multipart_status'] = 'Sent part ' . $settings['_chunks_sent'] . ' of ~' . $settings['_chunks_total'] . ' parts.'; $fileoptions['finish_time'] = microtime(true); $fileoptions['status'] = 'success'; $fileoptions_obj->save(); unset($fileoptions_obj); } } } fclose($fs); self::$_client->setDefer(false); if (false == $uploadStatus) { global $pb_backupbuddy_destination_errors; $pb_backupbuddy_destination_errors[] = 'Error #84347474 sending. Details: ' . $uploadStatus; return false; } else { // Success. if (true === $delete_remote_after) { self::deleteFile($settings, $uploadStatus->id); } } } // end foreach. $db_archive_limit = $settings['db_archive_limit']; $full_archive_limit = $settings['full_archive_limit']; $files_archive_limit = $settings['files_archive_limit']; // BEGIN FILE LIMIT PROCESSING. Enforce archive limits if applicable. if ($backup_type == 'full') { $limit = $full_archive_limit; } elseif ($backup_type == 'db') { $limit = $db_archive_limit; } elseif ($backup_type == 'files') { $limit = $files_archive_limit; } else { $limit = 0; pb_backupbuddy::status('warning', 'Warning #34352453244. Google Drive was unable to determine backup type (reported: `' . $backup_type . '`) so archive limits NOT enforced for this backup.'); } pb_backupbuddy::status('details', 'Google Drive database backup archive limit of `' . $limit . '` of type `' . $backup_type . '` based on destination settings.'); if ($limit > 0) { pb_backupbuddy::status('details', 'Google Drive archive limit enforcement beginning.'); // Get file listing. $searchCount = 1; $remoteFiles = array(); while (count($remoteFiles) == 0 && $searchCount < 5) { pb_backupbuddy::status('details', 'Checking archive limits. Attempt ' . $searchCount . '.'); $remoteFiles = pb_backupbuddy_destination_gdrive::listFiles($settings, "title contains 'backup-' AND title contains '-" . $backup_type . "-' AND '" . $folderID . "' IN parents AND trashed=false"); //"title contains 'backup' and trashed=false" ); sleep(1); $searchCount++; } // List backups associated with this site by date. $backups = array(); $prefix = backupbuddy_core::backup_prefix(); foreach ($remoteFiles as $remoteFile) { if ('application/vnd.google-apps.folder' == $remoteFile->mimeType) { // Ignore folders. continue; } if (strpos($remoteFile->originalFilename, 'backup-' . $prefix . '-') !== false) { // Appears to be a backup file for this site. $backups[$remoteFile->id] = strtotime($remoteFile->modifiedDate); } } arsort($backups); pb_backupbuddy::status('details', 'Google Drive found `' . count($backups) . '` backups of this type when checking archive limits.'); if (count($backups) > $limit) { pb_backupbuddy::status('details', 'More archives (' . count($backups) . ') than limit (' . $limit . ') allows. Trimming...'); $i = 0; $delete_fail_count = 0; foreach ($backups as $buname => $butime) { $i++; if ($i > $limit) { pb_backupbuddy::status('details', 'Trimming excess file `' . $buname . '`...'); if (true !== self::deleteFile($settings, $buname)) { pb_backupbuddy::status('details', 'Unable to delete excess Google Drive file `' . $buname . '`. Details: `' . print_r($pb_backupbuddy_destination_errors, true) . '`.'); $delete_fail_count++; } } } pb_backupbuddy::status('details', 'Finished trimming excess backups.'); if ($delete_fail_count !== 0) { $error_message = 'Google Drive remote limit could not delete ' . $delete_fail_count . ' backups.'; pb_backupbuddy::status('error', $error_message); backupbuddy_core::mail_error($error_message); } } pb_backupbuddy::status('details', 'Google Drive completed archive limiting.'); } else { pb_backupbuddy::status('details', 'No Google Drive archive file limit to enforce.'); } // End remote backup limit // Made it this far then success. return true; }
public static function send($settings = array(), $files = array(), $send_id = '', $delete_remote_after = false) { self::$_timeStart = microtime(true); $settings = self::_normalizeSettings($settings); if (false === ($settings = self::_connect($settings))) { $error = 'Unable to connect with Google Drive. See log for details.'; echo $error; pb_backupbuddy::status('error', $error); return false; } $chunkSizeBytes = $settings['max_burst'] * 1024 * 1024; // Send X mb at a time to limit memory usage. foreach ($files as $file) { $fileSize = filesize($file); $fileinfo = pathinfo($file); $fileextension = $fileinfo['extension']; if ('zip' == $fileextension) { $mimeType = 'application/zip'; } elseif ('php' == $fileextension) { $mimeType = 'application/x-httpd-php'; } else { $mimeType = ''; } pb_backupbuddy::status('details', 'About to upload file `' . $file . '` of size `' . $fileSize . '` with mimetype `' . $mimeType . '`. Internal chunk size of `' . $chunkSizeBytes . '` bytes.'); //Insert a file $driveFile = new Google_Service_Drive_DriveFile(); $driveFile->setTitle(basename($file)); $driveFile->setDescription('BackupBuddy file'); $driveFile->setMimeType($mimeType); self::$_client->setDefer(true); try { $insertRequest = self::$_drive->files->insert($driveFile); } catch (Exception $e) { pb_backupbuddy::alert('Error #3232783268336: initiating upload. Details: ' . $e->getMessage()); return; } // See https://developers.google.com/api-client-library/php/guide/media_upload try { $media = new Google_Http_MediaFileUpload(self::$_client, $insertRequest, $mimeType, null, true, $chunkSizeBytes); } catch (Exception $e) { pb_backupbuddy::alert('Error #3893273937: initiating upload. Details: ' . $e->getMessage()); return; } $media->setFileSize($fileSize); pb_backupbuddy::status('details', 'Opening file for sending in binary mode.'); $fs = fopen($file, 'rb'); // If chunked resuming then seek to the correct place in the file. if ('' != $settings['resume_point'] && $settings['resume_point'] > 0) { // Resuming send of a partially transferred file. if (0 !== fseek($fs, $settings['resume_point'])) { // Returns 0 on success. pb_backupbuddy::status('error', 'Error #3872733: Failed to seek file to resume point `' . $settings['resume_point'] . '` via fseek().'); return false; } $prevPointer = $settings['resume_point']; } else { // New file send. $prevPointer = 0; } $needProcessChunking = false; // Set true if we need to spawn off resuming to a new PHP page load. $uploadStatus = false; while (!$uploadStatus && !feof($fs)) { $chunk = fread($fs, $chunkSizeBytes); $uploadStatus = $media->nextChunk($chunk); // Handle splitting up across multiple PHP page loads if needed. if (!feof($fs) && 0 != $settings['max_time']) { // More data remains so see if we need to consider chunking to a new PHP process. // If we are within X second of reaching maximum PHP runtime then stop here so that it can be picked up in another PHP process... if (microtime(true) - self::$_timeStart + self::TIME_WIGGLE_ROOM >= $settings['max_time']) { pb_backupbuddy::status('message', 'Approaching limit of available PHP chunking time of `' . $settings['max_time'] . '` sec. Ran for ' . round(microtime(true) - self::$_timeStart, 3) . ' sec. Proceeding to use chunking.'); @fclose($fs); // Tells next chunk where to pick up. $settings['resume_point'] = $prevPointer; if (isset($chunksTotal)) { $settings['chunks_total'] = $chunksTotal; } // Schedule cron. $cronTime = time(); $cronArgs = array($settings, $files, $send_id, $delete_after = false); $cronHashID = md5($cronTime . serialize($cronArgs)); $cronArgs[] = $cronHashID; $schedule_result = backupbuddy_core::schedule_single_event($cronTime, pb_backupbuddy::cron_tag('destination_send'), $cronArgs); if (true === $schedule_result) { pb_backupbuddy::status('details', 'Next Site chunk step cron event scheduled.'); } else { pb_backupbuddy::status('error', 'Next Site chunk step cron even FAILED to be scheduled.'); } spawn_cron(time() + 150); // Adds > 60 seconds to get around once per minute cron running limit. update_option('_transient_doing_cron', 0); // Prevent cron-blocking for next item. return array($prevPointer, 'Sent part ' . $settings['chunks_sent'] . ' of ~' . $settings['chunks_total'] . ' parts.'); // filepointer location, elapsed time during the import } else { // End if. pb_backupbuddy::status('details', 'Not approaching time limit.'); } } else { pb_backupbuddy::status('details', 'No more data remains (eg for chunking) so finishing up.'); } } fclose($fs); self::$_client->setDefer(false); if (false == $uploadStatus) { global $pb_backupbuddy_destination_errors; $pb_backupbuddy_destination_errors[] = 'Error #84347474 sending. Details: ' . $uploadStatus; return false; } else { // Success. if (true === $delete_remote_after) { self::deleteFile($settings, $uploadStatus->id); } } } // end foreach. // Made it this far then success. return true; }