protected function _addFile($isVirtual, &$sourceNameOrData, $targetName)
{
if ($isVirtual) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "-- Adding {$targetName} to archive (virtual data)");
} else {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "-- Adding {$targetName} to archive (source: {$sourceNameOrData})");
}
$configuration = AEFactory::getConfiguration();
$timer = AEFactory::getTimer();
// Initialize archive file pointer
$fp = null;
// Initialize inode change timestamp
$filectime = 0;
$processingFile = $configuration->get('volatile.engine.archiver.processingfile', false);
if (!$processingFile) {
// Uncache data
$configuration->set('volatile.engine.archiver.sourceNameOrData', null);
$configuration->set('volatile.engine.archiver.unc_len', null);
$configuration->set('volatile.engine.archiver.resume', null);
$configuration->set('volatile.engine.archiver.processingfile', false);
// See if it's a directory
$isDir = $isVirtual ? false : is_dir($sourceNameOrData);
// See if it's a symlink (w/out dereference)
$isSymlink = false;
if ($this->_symlink_store_target && !$isVirtual) {
$isSymlink = is_link($sourceNameOrData);
}
// Get real size before compression
if ($isVirtual) {
$fileSize = akstringlen($sourceNameOrData);
$filectime = time();
} else {
if ($isSymlink) {
$fileSize = akstringlen(@readlink($sourceNameOrData));
} else {
// Is the file readable?
if (!is_readable($sourceNameOrData)) {
// Unreadable files won't be recorded in the archive file
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
return false;
} else {
// Really, REALLY check if it is readable (PHP sometimes lies, dammit!)
$myfp = @fopen($sourceNameOrData, 'rb');
if ($myfp === false) {
// Unreadable file, skip it.
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
return false;
}
@fclose($myfp);
}
// Get the filesize and modification time
$fileSize = $isDir ? 0 : @filesize($sourceNameOrData);
$filectime = $isDir ? 0 : @filemtime($sourceNameOrData);
}
}
// Decide if we will compress
if ($isDir || $isSymlink) {
// don't compress directories and symlinks...
$compressionMethod = 0;
} else {
// always compress files using gzip
$compressionMethod = 1;
}
// Fix stored name for directories
$storedName = $targetName;
$storedName .= $isDir ? "/" : "";
// Get file permissions
$perms = $isVirtual ? 0755 : @fileperms($sourceNameOrData);
// Get file type
if (!$isDir && !$isSymlink) {
$fileType = 1;
} elseif ($isSymlink) {
$fileType = 2;
} elseif ($isDir) {
$fileType = 0;
}
// Create the Entity Description Block Data
$headerData = pack('v', akstringlen($storedName)) . $storedName . pack('c', $fileType) . pack('c', $compressionMethod) . pack('V', $fileSize) . pack('V', $perms) . pack('V', $filectime);
// Create and write the Entity Description Block Header
$decryptedSize = akstringlen($headerData);
$headerData = AEUtilEncrypt::AESEncryptCBC($headerData, $this->password, 128);
$encryptedSize = akstringlen($headerData);
$headerData = $this->_fileHeader . pack('v', $encryptedSize) . pack('v', $decryptedSize) . $headerData;
// Do we have enough space to store the header?
if ($this->_useSplitZIP) {
// Compare to free part space
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space <= akstringlen($headerData)) {
// Not enough space on current part, create new part
if (!$this->_createNewPart()) {
$this->setError('Could not create new JPS part file ' . basename($this->_dataFileName));
return false;
}
}
}
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file '{$this->_dataFileName}' for append!");
return;
}
// Write the header data
$this->_fwrite($fp, $headerData);
// Cache useful information about the file
$configuration->set('volatile.engine.archiver.sourceNameOrData', $sourceNameOrData);
$configuration->set('volatile.engine.archiver.unc_len', $fileSize);
// Update global stats
$this->_fileCount++;
$this->_uncompressedSize += $fileSize;
} else {
$isDir = false;
$isSymlink = false;
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file '{$this->_dataFileName}' for append!");
return;
}
}
// Symlink: Single step, one block, uncompressed
if ($isSymlink) {
$data = @readlink($sourceNameOrData);
$this->_writeEncryptedBlock($fp, $data);
$this->_compressedSize += akstringlen($data);
if ($this->getError()) {
return;
}
} elseif ($isVirtual) {
// Loop in 64Kb blocks
while (strlen($sourceNameOrData) > 0) {
$data = substr($sourceNameOrData, 0, 65535);
if (akstringlen($data) < akstringlen($sourceNameOrData)) {
$sourceNameOrData = substr($sourceNameOrData, 65535);
} else {
$sourceNameOrData = '';
}
$data = gzcompress($data);
$data = substr(substr($data, 0, -4), 2);
$this->_writeEncryptedBlock($fp, $data);
$this->_compressedSize += akstringlen($data);
if ($this->getError()) {
return;
}
}
} else {
// Get resume information of required
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
$sourceNameOrData = $configuration->get('volatile.engine.archiver.sourceNameOrData', '');
$fileSize = $configuration->get('volatile.engine.archiver.unc_len', 0);
$resume = $configuration->get('volatile.engine.archiver.resume', 0);
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "(cont) Source: {$sourceNameOrData} - Size: {$fileSize} - Resume: {$resume}");
}
// Open the file
$zdatafp = @fopen($sourceNameOrData, "rb");
if ($zdatafp === FALSE) {
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
@fclose($fp);
return false;
}
// Seek to the resume point if required
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
// Seek to new offset
$seek_result = @fseek($zdatafp, $resume);
if ($seek_result === -1) {
// What?! We can't resume!
$this->setError(sprintf('Could not resume packing of file %s. Your archive is damaged!', $sourceNameOrData));
@fclose($zdatafp);
@fclose($fp);
return false;
}
// Doctor the uncompressed size to match the remainder of the data
$fileSize = $fileSize - $resume;
}
while (!feof($zdatafp) && $timer->getTimeLeft() > 0 && $fileSize > 0) {
$zdata = @fread($zdatafp, AKEEBA_CHUNK);
$fileSize -= min(akstringlen($zdata), AKEEBA_CHUNK);
$zdata = gzcompress($zdata);
$zdata = substr(substr($zdata, 0, -4), 2);
$this->_writeEncryptedBlock($fp, $zdata);
$this->_compressedSize += akstringlen($zdata);
if ($this->getError()) {
@fclose($zdatafp);
@fclose($fp);
return;
}
}
// WARNING!!! The extra $fileSize != 0 check is necessary as PHP won't reach EOF for 0-byte files.
if (!feof($zdatafp) && $fileSize != 0) {
// We have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
@fclose($zdatafp);
@fclose($fp);
return true;
} else {
$configuration->set('volatile.engine.archiver.resume', null);
$configuration->set('volatile.engine.archiver.processingfile', false);
}
@fclose($zdatafp);
}
}
/**
* Creates a dummy file of a given size. Remember to give the filesize
* query parameter in bytes!
*/
public function partsize()
{
$timer = AEFactory::getTimer();
$blocks = $this->input->get('blocks', 1, 'int');
$result = $this->createTempFile($blocks);
if ($result) {
// Save the setting
if ($blocks > 200) {
$blocks = 16383;
// Over 25Mb = 2Gb minus 128Kb limit (safe setting for PHP not running on 64-bit Linux)
}
$profile_id = AEPlatform::getInstance()->get_active_profile();
$config = AEFactory::getConfiguration();
$config->set('engine.archiver.common.part_size', $blocks * 128 * 1024);
AEPlatform::getInstance()->save_configuration($profile_id);
}
// Enforce the min exec time
$timer->enforce_min_exec_time(false);
return $result;
}
/**
* Performs one more step of dumping database data
* @return type
*/
protected function stepDatabaseDump()
{
// Initialize local variables
$db = $this->getDB();
if ($this->getError()) {
return;
}
if (!is_object($db) || $db === false) {
$this->setError(__CLASS__ . '::_run() Could not connect to database?!');
return;
}
$outData = '';
// Used for outputting INSERT INTO commands
$this->enforceSQLCompatibility();
// Apply MySQL compatibility option
if ($this->getError()) {
return;
}
// Touch SQL dump file
$nada = "";
$this->writeline($nada);
// Get this table's information
$tableName = $this->nextTable;
$this->setStep($tableName);
$this->setSubstep('');
$tableAbstract = trim($this->table_name_map[$tableName]);
$dump_records = $this->tables_data[$tableName]['dump_records'];
// If it is the first run, find number of rows and get the CREATE TABLE command
if ($this->nextRange == 0) {
if ($this->getError()) {
return;
}
$outCreate = '';
if (is_array($this->tables_data[$tableName])) {
if (array_key_exists('create', $this->tables_data[$tableName])) {
$outCreate = $this->tables_data[$tableName]['create'];
}
}
if (empty($outCreate) && !empty($tableName)) {
// The CREATE command wasn't cached. Time to create it. The $type and $dependencies
// variables will be thrown away.
$type = 'table';
$dependencies = array();
$outCreate = $this->get_create($tableAbstract, $tableName, $type, $dependencies);
}
// Create drop statements if required (the key is defined by the scripting engine)
$configuration = AEFactory::getConfiguration();
if (AEUtilScripting::getScriptingParameter('db.dropstatements', 0)) {
if (array_key_exists('create', $this->tables_data[$tableName])) {
// @todo This looks cheesy...
$dropStatement = $this->createDrop($this->tables_data[$tableName]['create']);
} else {
$type = 'table';
$createStatement = $this->get_create($tableAbstract, $tableName, $type, $dependencies);
$dropStatement = $this->createDrop($createStatement);
}
if (!empty($dropStatement)) {
$dropStatement .= "\n";
if (!$this->writeDump($dropStatement)) {
return;
}
}
}
// Write the CREATE command after any DROP command which might be necessary.
if (!$this->writeDump($outCreate)) {
return;
}
if ($dump_records) {
// We are dumping data from a table, get the row count
$this->getRowCount($tableAbstract);
} else {
// We should not dump any data
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Skipping dumping data of " . $tableAbstract);
$this->maxRange = 0;
$this->nextRange = 1;
$outData = '';
$numRows = 0;
}
// Output any data preamble commands, e.g. SET IDENTITY_INSERT for SQL Server
if ($dump_records && AEUtilScripting::getScriptingParameter('db.dropstatements', 0)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Writing data dump preamble for " . $tableAbstract);
$preamble = $this->getDataDumpPreamble($tableAbstract, $tableName, $this->maxRange);
if (!empty($preamble)) {
if (!$this->writeDump($preamble)) {
return;
}
}
}
}
// Check if we have more work to do on this table
$configuration = AEFactory::getConfiguration();
$batchsize = intval($configuration->get('engine.dump.common.batchsize', 1000));
if ($batchsize <= 0) {
$batchsize = 1000;
}
if ($this->nextRange < $this->maxRange) {
$timer = AEFactory::getTimer();
// Get the number of rows left to dump from the current table
$sql = $db->getQuery(true)->select('*')->from($db->nameQuote($tableAbstract));
if ($this->nextRange == 0) {
// First run, get a cursor to all records
$db->setQuery($sql, 0, $batchsize);
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Beginning dump of " . $tableAbstract);
} else {
// Subsequent runs, get a cursor to the rest of the records
$db->setQuery($sql, $this->nextRange, $batchsize);
$this->setSubstep($this->nextRange . ' / ' . $this->maxRange);
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Continuing dump of " . $tableAbstract . " from record #{$this->nextRange}");
}
$this->query = '';
$numRows = 0;
$use_abstract = AEUtilScripting::getScriptingParameter('db.abstractnames', 1);
$filters = AEFactory::getFilters();
$mustFilter = $filters->hasFilterType('dbobject', 'children');
try {
$cursor = $db->query();
} catch (Exception $exc) {
$db->resetErrors();
$cursor = null;
}
while (is_array($myRow = $db->fetchAssoc()) && $numRows < $this->maxRange - $this->nextRange) {
$this->createNewPartIfRequired();
$numRows++;
$numOfFields = count($myRow);
// On MS SQL Server there's always a RowNumber pseudocolumn added at the end, screwing up the backup (GRRRR!)
if ($db->getDriverType() == 'mssql') {
$numOfFields--;
}
// If row-level filtering is enabled, please run the filtering
if ($mustFilter) {
$isFiltered = $filters->isFiltered(array('table' => $tableAbstract, 'row' => $myRow), $configuration->get('volatile.database.root', '[SITEDB]'), 'dbobject', 'children');
if ($isFiltered) {
continue;
}
}
if (!$this->extendedInserts || $this->extendedInserts && empty($this->query)) {
$newQuery = true;
$fieldList = $this->getFieldListSQL(array_keys($myRow), $numOfFields);
if ($numOfFields > 0) {
$this->query = "INSERT INTO " . $db->nameQuote(!$use_abstract ? $tableName : $tableAbstract) . " {$fieldList} VALUES ";
}
} else {
// On other cases, just mark that we should add a comma and start a new VALUES entry
$newQuery = false;
}
$outData = '(';
// Step through each of the row's values
$fieldID = 0;
// Used in running backup fix
$isCurrentBackupEntry = false;
// Fix 1.2a - NULL values were being skipped
if ($numOfFields > 0) {
foreach ($myRow as $value) {
// The ID of the field, used to determine placement of commas
$fieldID++;
if ($fieldID > $numOfFields) {
// This is required for SQL Server backups, do NOT remove!
continue;
}
// Fix 2.0: Mark currently running backup as successful in the DB snapshot
if ($tableAbstract == '#__ak_stats') {
if ($fieldID == 1) {
// Compare the ID to the currently running
$statistics = AEFactory::getStatistics();
$isCurrentBackupEntry = $value == $statistics->getId();
} elseif ($fieldID == 6) {
// Treat the status field
$value = $isCurrentBackupEntry ? 'complete' : $value;
}
}
// Post-process the value
if (is_null($value)) {
$outData .= "NULL";
// Cope with null values
} else {
// Accommodate for runtime magic quotes
$value = @get_magic_quotes_runtime() ? stripslashes($value) : $value;
$value = $db->Quote($value);
if ($this->postProcessValues) {
$value = $this->postProcessQuotedValue($value);
}
$outData .= $value;
}
if ($fieldID < $numOfFields) {
$outData .= ', ';
}
}
}
$outData .= ')';
if ($numOfFields) {
// If it's an existing query and we have extended inserts
if ($this->extendedInserts && !$newQuery) {
// Check the existing query size
$query_length = strlen($this->query);
$data_length = strlen($outData);
if ($query_length + $data_length > $this->packetSize) {
// We are about to exceed the packet size. Write the data so far.
$this->query .= ";\n";
if (!$this->writeDump($this->query)) {
return;
}
// Then, start a new query
$this->query = '';
$this->query = "INSERT INTO " . $db->nameQuote(!$use_abstract ? $tableName : $tableAbstract) . " VALUES ";
$this->query .= $outData;
} else {
// We have room for more data. Append $outData to the query.
$this->query .= ', ';
$this->query .= $outData;
}
} elseif ($this->extendedInserts && $newQuery) {
// Append the data to the INSERT statement
$this->query .= $outData;
// Let's see the size of the dumped data...
$query_length = strlen($this->query);
if ($query_length >= $this->packetSize) {
// This was a BIG query. Write the data to disk.
$this->query .= ";\n";
if (!$this->writeDump($this->query)) {
return;
}
// Then, start a new query
$this->query = '';
}
} else {
// Append the data to the INSERT statement
$this->query .= $outData;
// Write the data to disk.
$this->query .= ";\n";
if (!$this->writeDump($this->query)) {
return;
}
// Then, start a new query
$this->query = '';
}
}
$outData = '';
unset($myRow);
// Check for imminent timeout
if ($timer->getTimeLeft() <= 0) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Breaking dump of {$tableAbstract} after {$numRows} rows; will continue on next step");
break;
}
}
$db->freeResult($cursor);
// Advance the _nextRange pointer
$this->nextRange += $numRows != 0 ? $numRows : 1;
$this->setStep($tableName);
$this->setSubstep($this->nextRange . ' / ' . $this->maxRange);
}
// Finalize any pending query
// WARNING! If we do not do that now, the query will be emptied in the next operation and all
// accumulated data will go away...
if (!empty($this->query)) {
$this->query .= ";\n";
if (!$this->writeDump($this->query)) {
return;
}
$this->query = '';
}
// Check for end of table dump (so that it happens inside the same operation)
if (!($this->nextRange < $this->maxRange)) {
// Tell the user we are done with the table
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Done dumping " . $tableAbstract);
// Output any data preamble commands, e.g. SET IDENTITY_INSERT for SQL Server
if ($dump_records && AEUtilScripting::getScriptingParameter('db.dropstatements', 0)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Writing data dump epilogue for " . $tableAbstract);
$epilogue = $this->getDataDumpEpilogue($tableAbstract, $tableName, $this->maxRange);
if (!empty($epilogue)) {
if (!$this->writeDump($epilogue)) {
return;
}
}
}
if (count($this->tables) == 0) {
// We have finished dumping the database!
AEUtilLogger::WriteLog(_AE_LOG_INFO, "End of database detected; flushing the dump buffers...");
$null = null;
$this->writeDump($null);
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Database has been successfully dumped to SQL file(s)");
$this->setState('postrun');
$this->setStep('');
$this->setSubstep('');
$this->nextTable = '';
$this->nextRange = 0;
} elseif (count($this->tables) != 0) {
// Switch tables
$this->nextTable = array_shift($this->tables);
$this->nextRange = 0;
$this->setStep($this->nextTable);
$this->setSubstep('');
}
}
}
public function deleteRemoteFiles()
{
$id = $this->getState('id', -1);
$part = $this->getState('part', -1);
$ret = array('error' => false, 'finished' => false, 'id' => $id, 'part' => $part);
// Gather the necessary information to perform the delete
$stat = AEPlatform::getInstance()->get_statistics($id);
$remoteFilename = $stat['remote_filename'];
$rfparts = explode('://', $remoteFilename);
$engine = AEFactory::getPostprocEngine($rfparts[0]);
$remote_filename = $rfparts[1];
// Load the correct backup profile
AEPlatform::getInstance()->load_configuration($stat['profile_id']);
$config = AEFactory::getConfiguration();
// Start timing ourselves
$timer = AEFactory::getTimer();
// The core timer object
$start = $timer->getRunningTime();
// Mark the start of this download
$break = false;
// Don't break the step
while ($timer->getTimeLeft() && !$break && $part < $stat['multipart']) {
// Get the remote filename
$basename = basename($remote_filename);
$extension = strtolower(str_replace(".", "", strrchr($basename, ".")));
if ($part > 0) {
$new_extension = substr($extension, 0, 1) . sprintf('%02u', $part);
} else {
$new_extension = $extension;
}
$filename = $basename . '.' . $new_extension;
$remote_filename = substr($remote_filename, 0, -strlen($extension)) . $new_extension;
// Do we have to initialize the process?
if ($part == -1) {
// Init
$part = 0;
}
// Try to delete the part
$required_time = 1.0;
$result = $engine->delete($remote_filename);
if (!$result) {
$ret['error'] = JText::_('REMOTEFILES_ERR_CANTDELETE') . $engine->getWarning();
return $ret;
return;
} else {
// Successful delete
$end = $timer->getRunningTime();
$part++;
}
// Do we predict that we have enough time?
$required_time = max(1.1 * ($end - $start), $required_time);
if ($timer->getTimeLeft() < $required_time) {
$break = true;
}
$start = $end;
}
if ($part >= $stat['multipart']) {
// Just finished!
$stat['remote_filename'] = '';
AEPlatform::getInstance()->set_or_update_statistics($id, $stat, $engine);
$ret['finished'] = true;
return $ret;
} else {
// More work to do...
$ret['id'] = $id;
$ret['part'] = $part;
return $ret;
}
}
protected function _run()
{
AEUtilLogger::openLog($this->tag);
set_error_handler('akeebaBackupErrorHandler');
// Maybe we're already done or in an error state?
if ($this->getError() || $this->getState() == 'postrun') {
return;
}
// Set running state
$this->setState('running');
// Initialize operation counter
$registry = AEFactory::getConfiguration();
$registry->set('volatile.operation_counter', 0);
// Advance step counter
$stepCounter = $registry->get('volatile.step_counter', 0);
$registry->set('volatile.step_counter', ++$stepCounter);
// Log step start number
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, '====== Starting Step number ' . $stepCounter . ' ======');
if (defined('AKEEBADEBUG')) {
$root = AEPlatform::getInstance()->get_site_root();
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, 'Site root: ' . $root);
}
$timer = AEFactory::getTimer();
$finished = false;
$error = false;
$breakFlag = false;
// BREAKFLAG is optionally passed by domains to force-break current operation
// Apply an infinite time limit if required
if ($registry->get('akeeba.tuning.settimelimit', 0)) {
if (function_exists('set_time_limit')) {
set_time_limit(0);
}
}
// Loop until time's up, we're done or an error occurred, or BREAKFLAG is set
$this->array_cache = null;
while ($timer->getTimeLeft() > 0 && !$finished && !$error && !$breakFlag) {
// Reset the break flag
$registry->set('volatile.breakflag', false);
// Do we have to switch domains? This only happens if there is no active
// domain, or the current domain has finished
$have_to_switch = false;
if ($this->class == '') {
$have_to_switch = true;
} else {
$object = AEFactory::getDomainObject($this->class);
if (!is_object($object)) {
$have_to_switch = true;
} else {
if (!in_array('getState', get_class_methods($object))) {
$have_to_switch = true;
} else {
if ($object->getState() == 'finished') {
$have_to_switch = true;
}
}
}
}
// Switch domain if necessary
if ($have_to_switch) {
if (!AEFactory::getConfiguration()->get('akeeba.tuning.nobreak.domains', 0)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Kettenrad :: BREAKING STEP BEFORE SWITCHING DOMAIN");
$registry->set('volatile.breakflag', true);
}
$object = null;
// Free last domain
if (empty($this->domain_chain)) {
// Aw, we're done! No more domains to run.
$this->setState('postrun');
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Kettenrad :: No more domains to process");
$this->array_cache = null;
//restore_error_handler();
return;
}
// Shift the next definition off the stack
$this->array_cache = null;
$new_definition = array_shift($this->domain_chain);
if (array_key_exists('class', $new_definition)) {
$this->domain = $new_definition['domain'];
$this->class = $new_definition['class'];
// Get a working object
$object = AEFactory::getDomainObject($this->class);
$object->setup($this->_parametersArray);
} else {
AEUtilLogger::WriteLog(_AE_LOG_WARNING, "Kettenrad :: No class defined trying to switch domains. The backup will crash.");
$this->domain = null;
$this->class = null;
}
} else {
if (!is_object($object)) {
$object = AEFactory::getDomainObject($this->class);
}
}
// Tick the object
$result = $object->tick();
// Propagate errors
$this->propagateFromObject($object);
// Advance operation counter
$currentOperationNumber = $registry->get('volatile.operation_counter', 0);
$currentOperationNumber++;
$registry->set('volatile.operation_counter', $currentOperationNumber);
// Process return array
$this->setDomain($this->domain);
$this->setStep($result['Step']);
$this->setSubstep($result['Substep']);
// Check for BREAKFLAG
$breakFlag = $registry->get('volatile.breakflag', false);
// Process errors
$error = false;
if ($this->getError()) {
$error = true;
}
// Check if the backup procedure should finish now
$finished = $error ? true : !$result['HasRun'];
// Log operation end
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, '----- Finished operation ' . $currentOperationNumber . ' ------');
}
// while
// Log the result
if (!$error) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Successful Smart algorithm on " . get_class($object));
} else {
AEUtilLogger::WriteLog(_AE_LOG_ERROR, "Failed Smart algorithm on " . get_class($object));
}
// Log if we have to do more work or not
if (!is_object($object)) {
AEUtilLogger::WriteLog(_AE_LOG_WARNING, "Kettenrad :: Empty object found when processing domain '" . $this->domain . "'. This should never happen.");
} else {
if ($object->getState() == 'running') {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Kettenrad :: More work required in domain '" . $this->domain . "'");
// We need to set the break flag for the part processing to not batch successive steps
$registry->set('volatile.breakflag', true);
} elseif ($object->getState() == 'finished') {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Kettenrad :: Domain '" . $this->domain . "' has finished.");
$registry->set('volatile.breakflag', false);
}
}
// Log step end
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, '====== Finished Step number ' . $stepCounter . ' ======');
if (!$registry->get('akeeba.tuning.nobreak.domains', 0)) {
// Force break between steps
$registry->set('volatile.breakflag', true);
}
//restore_error_handler();
}
/**
* Try to pack some files in the $file_list, restraining ourselves not to reach the max
* number of files or max fragment size while doing so. If this process is over and we are
* left without any more files, reset $done_scanning to false in order to instruct the class
* to scan for more files.
*
* @return bool True if there were files packed, false otherwise (empty filelist)
*/
private function pack_files()
{
// Get a reference to the archiver and the timer classes
$archiver = AEFactory::getArchiverEngine();
$timer = AEFactory::getTimer();
$configuration = AEFactory::getConfiguration();
// If post-processing after part creation is enabled, make sure we do post-process each part before moving on
if ($configuration->get('engine.postproc.common.after_part', 0)) {
if (!empty($archiver->finishedPart)) {
$filename = array_shift($archiver->finishedPart);
AEUtilLogger::WriteLog(_AE_LOG_INFO, 'Preparing to post process ' . basename($filename));
$post_proc = AEFactory::getPostprocEngine();
$result = $post_proc->processPart($filename);
$this->propagateFromObject($post_proc);
if ($result === false) {
$this->setWarning('Failed to process file ' . basename($filename));
} else {
AEUtilLogger::WriteLog(_AE_LOG_INFO, 'Successfully processed file ' . basename($filename));
}
// Should we delete the file afterwards?
if ($configuration->get('engine.postproc.common.delete_after', false) && $post_proc->allow_deletes && $result !== false) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, 'Deleting already processed file ' . basename($filename));
AEPlatform::getInstance()->unlink($filename);
}
if ($post_proc->break_after && $result !== false) {
$configuration->set('volatile.breakflag', true);
return true;
}
// This is required to let the backup continue even after a post-proc failure
$this->resetErrors();
$this->setState('running');
}
}
// If the archiver has work to do, make sure it finished up before continuing
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Continuing file packing from previous step");
$result = $archiver->addFile('', '', '');
$this->propagateFromObject($archiver);
if ($this->getError()) {
return false;
}
// If that was the last step, mark a file done
if (!$configuration->get('volatile.engine.archiver.processingfile', false)) {
$this->progressMarkFileDone();
}
}
// Did it finish, or does it have more work to do?
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
// More work to do. Let's just tell our parent that we finished up successfully.
return true;
}
// Normal file backup loop; we keep on processing the file list, packing files as we go.
if (count($this->file_list) == 0) {
// No files left to pack -- This should never happen! We catch this condition at the end of this method!
$this->done_scanning = false;
$this->progressMarkFolderDone();
return false;
} else {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Packing files");
$packedSize = 0;
$numberOfFiles = 0;
list($usec, $sec) = explode(" ", microtime());
$opStartTime = (double) $usec + (double) $sec;
while (count($this->file_list) > 0) {
$file = @array_shift($this->file_list);
$size = 0;
if (file_exists($file)) {
$size = @filesize($file);
}
// Anticipatory file size algorithm
if ($numberOfFiles > 0 && $size > AELargeFileThreshold) {
if (!AEFactory::getConfiguration()->get('akeeba.tuning.nobreak.beforelargefile', 0)) {
// If the file is bigger than the big file threshold, break the step
// to avoid potential timeouts
$this->setBreakFlag();
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Breaking step _before_ large file: " . $file . " - size: " . $size);
// Push the file back to the list.
array_unshift($this->file_list, $file);
// Mark that we are not done packing files
$this->done_scanning = true;
return true;
}
}
// Proactive potential timeout detection
// Rough estimation of packing speed in bytes per second
list($usec, $sec) = explode(" ", microtime());
$opEndTime = (double) $usec + (double) $sec;
if ($opEndTime - $opStartTime == 0) {
$_packSpeed = 0;
} else {
$_packSpeed = $packedSize / ($opEndTime - $opStartTime);
}
// Estimate required time to pack next file. If it's the first file of this operation,
// do not impose any limitations.
$_reqTime = $_packSpeed - 0.01 <= 0 ? 0 : $size / $_packSpeed;
// Do we have enough time?
if ($timer->getTimeLeft() < $_reqTime) {
if (!AEFactory::getConfiguration()->get('akeeba.tuning.nobreak.proactive', 0)) {
array_unshift($this->file_list, $file);
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Proactive step break - file: " . $file . " - size: " . $size . " - req. time " . sprintf('%2.2f', $_reqTime));
$this->setBreakFlag();
$this->done_scanning = true;
return true;
}
}
$packedSize += $size;
$numberOfFiles++;
$ret = $archiver->addFile($file, $this->remove_path_prefix, $this->path_prefix);
// If no more processing steps are required, mark a done file
if (!$configuration->get('volatile.engine.archiver.processingfile', false)) {
$this->progressMarkFileDone();
}
// Error propagation
$this->propagateFromObject($archiver);
if ($this->getError()) {
return false;
}
// If this was the first file of the fragment and it exceeded the fragment's capacity,
// break the step. Continuing with more operations after packing such a big file is
// increasing the risk to hit a timeout.
if ($packedSize > AELargeFileThreshold && $numberOfFiles == 1) {
if (!AEFactory::getConfiguration()->get('akeeba.tuning.nobreak.afterlargefile', 0)) {
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Breaking step *after* large file: " . $file . " - size: " . $size);
$this->setBreakFlag();
return true;
}
}
// If we have to continue processing the file, break the file packing loop forcibly
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
return true;
}
}
$this->done_scanning = count($this->file_list) > 0;
if (!$this->done_scanning) {
$this->progressMarkFolderDone();
}
return true;
}
}
/**
* The public interface to an engine part. This method takes care for
* calling the correct method in order to perform the initialisation -
* run - finalisation cycle of operation and return a proper reponse array.
* @return array A Reponse Array
*/
public final function tick($nesting = 0)
{
$configuration = AEFactory::getConfiguration();
$timer = AEFactory::getTimer();
// Call the right action method, depending on engine part state
switch ($this->getState()) {
case "init":
$this->_prepare();
$breakFlag = $configuration->set('volatile.breakflag', false);
break;
case "prepared":
$this->_run();
break;
case "running":
$this->_run();
break;
case "postrun":
$this->_finalize();
$breakFlag = $configuration->set('volatile.breakflag', false);
break;
}
// If there is still time, we are not finished and there is no break flag set, re-run the tick()
// method.
$breakFlag = $configuration->get('volatile.breakflag', false);
if (!in_array($this->getState(), array('finished', 'error')) && $timer->getTimeLeft() > 0 && !$breakFlag && $nesting < 20 && $this->nest_logging) {
// Nesting is only applied if $this->nest_logging == true (currently only Kettenrad has this)
$nesting++;
if ($this->nest_logging) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "*** Batching successive steps (nesting level {$nesting})");
}
$out = $this->tick($nesting);
} else {
// Return the output array
$out = $this->_makeReturnTable();
// Things to do for nest-logged parts (currently, only Kettenrad is)
if ($this->nest_logging) {
if ($breakFlag) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "*** Engine steps batching: Break flag detected.");
}
// Reset the break flag
$configuration->set('volatile.breakflag', false);
// Log that we're breaking the step
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "*** Batching of engine steps finished. I will now return control to the caller.");
// Enforce minimum execution time
$timer = AEFactory::getTimer();
$timer->enforce_min_exec_time(true);
}
}
// Send a Return Table back to the caller
return $out;
}
public function downloadToServer()
{
if (!$this->_hasAdequateInformation()) {
$this->setError(JText::_('S3IMPORT_ERR_NOTENOUGHINFO'));
return false;
}
// Gather the necessary information to perform the download
$part = JFactory::getApplication()->getUserState('com_akeeba.s3import.part', -1);
$frag = JFactory::getApplication()->getUserState('com_akeeba.s3import.frag', -1);
$remoteFilename = $this->getState('file', '');
$s3 = $this->_getS3Object();
// Get the number of parts and total size from the session, or –if not there– fetch it
$totalparts = JFactory::getApplication()->getUserState('com_akeeba.s3import.totalparts', -1);
$totalsize = JFactory::getApplication()->getUserState('com_akeeba.s3import.totalsize', -1);
if ($totalparts < 0 || $part < 0 && $frag < 0) {
$filePrefix = substr($remoteFilename, 0, -3);
$allFiles = $s3->getBucket($this->getState('s3bucket'), $filePrefix);
$totalsize = 0;
if (count($allFiles)) {
foreach ($allFiles as $name => $file) {
$totalsize += $file['size'];
}
}
JFactory::getApplication()->setUserState('com_akeeba.s3import.totalparts', count($allFiles));
JFactory::getApplication()->setUserState('com_akeeba.s3import.totalsize', $totalsize);
JFactory::getApplication()->setUserState('com_akeeba.s3import.donesize', 0);
$totalparts = JFactory::getApplication()->getUserState('com_akeeba.s3import.totalparts', -1);
}
// Start timing ourselves
$timer = AEFactory::getTimer();
// The core timer object
$start = $timer->getRunningTime();
// Mark the start of this download
$break = false;
// Don't break the step
while ($timer->getRunningTime() < 10 && !$break && $part < $totalparts) {
// Get the remote and local filenames
$basename = basename($remoteFilename);
$extension = strtolower(str_replace(".", "", strrchr($basename, ".")));
if ($part > 0) {
$new_extension = substr($extension, 0, 1) . sprintf('%02u', $part);
} else {
$new_extension = $extension;
}
$filename = $basename . '.' . $new_extension;
$remote_filename = substr($remoteFilename, 0, -strlen($extension)) . $new_extension;
// Figure out where on Earth to put that file
$local_file = AEFactory::getConfiguration()->get('akeeba.basic.output_directory') . '/' . basename($remote_filename);
// Do we have to initialize the process?
if ($part == -1) {
// Currently downloaded size
JFactory::getApplication()->setUserState('com_akeeba.s3import.donesize', 0);
// Init
$part = 0;
}
// Do we have to initialize the file?
if ($frag == -1) {
// Delete and touch the output file
AEPlatform::getInstance()->unlink($local_file);
$fp = @fopen($local_file, 'wb');
if ($fp !== false) {
@fclose($fp);
}
// Init
$frag = 0;
}
// Calculate from and length
$length = 1048576;
// That's wrong: the first byte is byte 0, not byte 1!!!
//$from = $frag * $length + 1;
$from = $frag * $length;
$to = $length + $from;
if ($from == 0) {
$from = 1;
}
// Try to download the first frag
$temp_file = $local_file . '.tmp';
@unlink($temp_file);
$required_time = 1.0;
$result = $s3->getObject($this->getState('s3bucket', ''), $remote_filename, $temp_file, $from, $to);
if (!$result) {
// Failed download
@unlink($temp_file);
if (($part < $totalparts || $totalparts == 1 && $part == 0) && $frag == 0) {
// Failure to download the part's beginning = failure to download. Period.
$this->setError(JText::_('S3IMPORT_ERR_NOTFOUND'));
return false;
} elseif ($part >= $totalparts) {
// Just finished! Create a stats record.
$multipart = $totalparts;
$multipart--;
$filetime = time();
// Create a new backup record
$record = array('description' => JText::_('DISCOVER_LABEL_IMPORTEDDESCRIPTION'), 'comment' => '', 'backupstart' => date('Y-m-d H:i:s', $filetime), 'backupend' => date('Y-m-d H:i:s', $filetime + 1), 'status' => 'complete', 'origin' => 'backend', 'type' => 'full', 'profile_id' => 1, 'archivename' => basename($remoteFilename), 'absolute_path' => dirname($local_file) . '/' . basename($remoteFilename), 'multipart' => $multipart, 'tag' => 'backend', 'filesexist' => 1, 'remote_filename' => '', 'total_size' => $totalsize);
$id = null;
$id = AEPlatform::getInstance()->set_or_update_statistics($id, $record, $this);
return null;
} else {
// Since this is a staggered download, consider this normal and go to the next part.
$part++;
$frag = -1;
}
}
// Add the currently downloaded frag to the total size of downloaded files
if ($result) {
clearstatcache();
$filesize = (int) @filesize($temp_file);
$total = JFactory::getApplication()->getUserState('com_akeeba.s3import.donesize', 0);
$total += $filesize;
JFactory::getApplication()->setUserState('com_akeeba.s3import.donesize', $total);
}
// Successful download, or have to move to the next part.
if ($result) {
// Append the file
$fp = @fopen($local_file, 'ab');
if ($fp === false) {
// Can't open the file for writing
@unlink($temp_file);
$this->setError(JText::_('S3IMPORT_ERR_CANTWRITE'));
return false;
}
$tf = fopen($temp_file, 'rb');
while (!feof($tf)) {
$data = fread($tf, 262144);
fwrite($fp, $data);
}
fclose($tf);
fclose($fp);
@unlink($temp_file);
$frag++;
}
// Advance the frag pointer and mark the end
$end = $timer->getRunningTime();
// Do we predict that we have enough time?
$required_time = max(1.1 * ($end - $start), $required_time);
if ($required_time > 10 - $end + $start) {
$break = true;
}
$start = $end;
}
// Pass the id, part, frag in the request so that the view can grab it
$this->setState('part', $part);
$this->setState('frag', $frag);
JFactory::getApplication()->setUserState('com_akeeba.s3import.part', $part);
JFactory::getApplication()->setUserState('com_akeeba.s3import.frag', $frag);
if ($part >= $totalparts) {
// Just finished! Create a new backup record
$record = array('description' => JText::_('DISCOVER_LABEL_IMPORTEDDESCRIPTION'), 'comment' => '', 'backupstart' => date('Y-m-d H:i:s'), 'backupend' => date('Y-m-d H:i:s', time() + 1), 'status' => 'complete', 'origin' => 'backend', 'type' => 'full', 'profile_id' => 1, 'archivename' => basename($remoteFilename), 'absolute_path' => dirname($local_file) . '/' . basename($remoteFilename), 'multipart' => $totalparts, 'tag' => 'backend', 'filesexist' => 1, 'remote_filename' => '', 'total_size' => $totalsize);
$id = null;
$id = AEPlatform::getInstance()->set_or_update_statistics($id, $record, $this);
return null;
}
return true;
}
/**
* The most basic file transaction: add a single entry (file or directory) to
* the archive.
*
* @param bool $isVirtual If true, the next parameter contains file data instead of a file name
* @param string $sourceNameOrData Absolute file name to read data from or the file data itself is $isVirtual is true
* @param string $targetName The (relative) file name under which to store the file in the archive
* @return True on success, false otherwise
* @since 1.2.1
* @access protected
* @abstract
*/
protected function _addFile($isVirtual, &$sourceNameOrData, $targetName)
{
static $configuration;
$isDir = false;
$isSymlink = false;
if (is_null($isVirtual)) {
$isVirtual = false;
}
$compressionMethod = 0;
if ($isVirtual) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "-- Adding {$targetName} to archive (virtual data)");
} else {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "-- Adding {$targetName} to archive (source: {$sourceNameOrData})");
}
if (!$configuration) {
$configuration =& AEFactory::getConfiguration();
}
$timer =& AEFactory::getTimer();
// Initialize archive file pointer
$fp = null;
// Initialize inode change timestamp
$filectime = 0;
if (!$configuration->get('volatile.engine.archiver.processingfile', false)) {
// Uncache data -- WHY DO THAT?!
/**
$configuration->set('volatile.engine.archiver.sourceNameOrData', null);
$configuration->set('volatile.engine.archiver.unc_len', null);
$configuration->set('volatile.engine.archiver.resume', null);
$configuration->set('volatile.engine.archiver.processingfile',false);
/**/
// See if it's a directory
$isDir = $isVirtual ? false : is_dir($sourceNameOrData);
// See if it's a symlink (w/out dereference)
$isSymlink = false;
if ($this->_symlink_store_target && !$isVirtual) {
$isSymlink = is_link($sourceNameOrData);
}
// Get real size before compression
if ($isVirtual) {
$fileSize = akstringlen($sourceNameOrData);
$filectime = time();
} else {
if ($isSymlink) {
$fileSize = akstringlen(@readlink($sourceNameOrData));
} else {
// Is the file readable?
if (!is_readable($sourceNameOrData) && !$isDir) {
// Unreadable files won't be recorded in the archive file
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
return false;
}
// Get the filesize
$fileSize = $isDir ? 0 : @filesize($sourceNameOrData);
$filectime = $isDir ? 0 : @filemtime($sourceNameOrData);
}
}
// Decide if we will compress
if ($isDir || $isSymlink) {
$compressionMethod = 0;
// don't compress directories...
} else {
// Do we have plenty of memory left?
$memLimit = ini_get("memory_limit");
if (strstr($memLimit, 'M')) {
$memLimit = (int) $memLimit * 1048576;
} elseif (strstr($totalRAM, 'K')) {
$memLimit = (int) $memLimit * 1024;
} elseif (strstr($memLimit, 'G')) {
$memLimit = (int) $memLimit * 1073741824;
} else {
$memLimit = (int) $memLimit;
}
if (is_numeric($memLimit) && $memLimit < 0) {
$memLimit = "";
}
// 1.2a3 -- Rare case with memory_limit < 0, e.g. -1Mb!
if ($memLimit == "" || $fileSize >= _AKEEBA_COMPRESSION_THRESHOLD) {
// No memory limit, or over 1Mb files => always compress up to 1Mb files (otherwise it times out)
$compressionMethod = $fileSize <= _AKEEBA_COMPRESSION_THRESHOLD ? 1 : 0;
} elseif (function_exists("memory_get_usage")) {
// PHP can report memory usage, see if there's enough available memory; Joomla! alone eats about 5-6Mb! This code is called on files <= 1Mb
$memLimit = $this->_return_bytes($memLimit);
$availableRAM = $memLimit - memory_get_usage();
$compressionMethod = $availableRAM / 2.5 >= $fileSize ? 1 : 0;
} else {
// PHP can't report memory usage, compress only files up to 512Kb (conservative approach) and hope it doesn't break
$compressionMethod = $fileSize <= 524288 ? 1 : 0;
}
}
$compressionMethod = function_exists("gzcompress") ? $compressionMethod : 0;
$storedName = $targetName;
/* "Entity Description BLock" segment. */
$unc_len =& $fileSize;
// File size
$storedName .= $isDir ? "/" : "";
if ($compressionMethod == 1) {
if ($isVirtual) {
$udata =& $sourceNameOrData;
} else {
// Get uncompressed data
$udata = @file_get_contents($sourceNameOrData);
// PHP > 4.3.0 saves us the trouble
}
if ($udata === FALSE) {
// Unreadable file, skip it.
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
return false;
} else {
// Proceed with compression
$zdata = @gzcompress($udata);
if ($zdata === false) {
// If compression fails, let it behave like no compression was available
$c_len =& $unc_len;
$compressionMethod = 0;
} else {
unset($udata);
$zdata = substr(substr($zdata, 0, -4), 2);
$c_len = akstringlen($zdata);
}
}
} else {
$c_len = $unc_len;
// Test for unreadable files
if (!$isVirtual && !$isSymlink && !$isDir) {
$myfp = @fopen($sourceNameOrData, 'rb');
if ($myfp === false) {
// Unreadable file, skip it.
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
return false;
}
@fclose($myfp);
}
}
$this->_compressedSize += $c_len;
// Update global data
$this->_uncompressedSize += $fileSize;
// Update global data
$this->_fileCount++;
// Get file permissions
$perms = 0755;
if (!$isVirtual) {
if (@file_exists($sourceNameOrData)) {
if (@is_file($sourceNameOrData) || @is_link($sourceNameOrData)) {
if (@is_readable($sourceNameOrData)) {
$perms = @fileperms($sourceNameOrData);
}
}
}
}
// Calculate Entity Description Block length
$blockLength = 21 + akstringlen($storedName);
if ($filectime > 0) {
$blockLength += 8;
}
// If we need to store the file mod date
// Get file type
if (!$isDir && !$isSymlink) {
$fileType = 1;
} elseif ($isSymlink) {
$fileType = 2;
} elseif ($isDir) {
$fileType = 0;
}
// If it's a split ZIP file, we've got to make sure that the header can fit in the part
if ($this->_useSplitZIP) {
// Compare to free part space
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space <= $blockLength) {
// Not enough space on current part, create new part
if (!$this->_createNewPart()) {
$this->setError('Could not create new JPA part file ' . basename($this->_dataFileName));
return false;
}
}
}
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file '{$this->_dataFileName}' for append!");
return false;
}
$this->_fwrite($fp, $this->_fileHeader);
// Entity Description Block header
if ($this->getError()) {
return false;
}
$this->_fwrite($fp, pack('v', $blockLength));
// Entity Description Block header length
$this->_fwrite($fp, pack('v', akstringlen($storedName)));
// Length of entity path
$this->_fwrite($fp, $storedName);
// Entity path
$this->_fwrite($fp, pack('C', $fileType));
// Entity type
$this->_fwrite($fp, pack('C', $compressionMethod));
// Compression method
$this->_fwrite($fp, pack('V', $c_len));
// Compressed size
$this->_fwrite($fp, pack('V', $unc_len));
// Uncompressed size
$this->_fwrite($fp, pack('V', $perms));
// Entity permissions
// Timestamp Extra Field, only for files
if ($filectime > 0) {
$this->_fwrite($fp, "");
// Extra Field Identifier
$this->_fwrite($fp, pack('v', 8));
// Extra Field Length
$this->_fwrite($fp, pack('V', $filectime));
// Timestamp
}
// Cache useful information about the file
if (!$isDir && !$isSymlink && !$isVirtual) {
$configuration->set('volatile.engine.archiver.unc_len', $unc_len);
$configuration->set('volatile.engine.archiver.sourceNameOrData', $sourceNameOrData);
}
} else {
// If we are continuing file packing we have an uncompressed, non-virtual file.
// We need to set up these variables so as not to throw any PHP notices.
$isDir = false;
$isSymlink = false;
$isVirtual = false;
$compressionMethod = 0;
// Create a file pointer to the archive file
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file '{$this->_dataFileName}' for append!");
return false;
}
}
/* "File data" segment. */
if ($compressionMethod == 1) {
if (!$this->_useSplitZIP) {
// Just dump the compressed data
$this->_fwrite($fp, $zdata);
if ($this->getError()) {
@fclose($fp);
return false;
}
} else {
// Split ZIP. Check if we need to split the part in the middle of the data.
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space >= akstringlen($zdata)) {
// Write in one part
$this->_fwrite($fp, $zdata);
if ($this->getError()) {
@fclose($fp);
return false;
}
} else {
$bytes_left = akstringlen($zdata);
while ($bytes_left > 0) {
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
// Split between parts - Write first part
$this->_fwrite($fp, $zdata, min(akstringlen($zdata), $free_space));
if ($this->getError()) {
@fclose($fp);
return false;
}
// Get the rest of the data
$bytes_left = akstringlen($zdata) - $free_space;
if ($bytes_left > 0) {
// Create new part
@fclose($fp);
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new JPA part file ' . basename($this->_dataFileName));
return false;
} else {
// Close the old data file
@fclose($fp);
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
return false;
}
}
$zdata = substr($zdata, -$bytes_left);
}
}
}
}
unset($zdata);
} elseif (!$isDir && !$isSymlink) {
if ($isVirtual) {
if (!$this->_useSplitZIP) {
// Just dump the data
$this->_fwrite($fp, $sourceNameOrData);
if ($this->getError()) {
@fclose($fp);
return false;
}
} else {
// Split JPA. Check if we need to split the part in the middle of the data.
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space >= akstringlen($sourceNameOrData)) {
// Write in one part
$this->_fwrite($fp, $sourceNameOrData);
if ($this->getError()) {
return false;
}
} else {
$bytes_left = akstringlen($sourceNameOrData);
while ($bytes_left > 0) {
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
// Split between parts - Write first part
$this->_fwrite($fp, $sourceNameOrData, min(akstringlen($sourceNameOrData), $free_space));
if ($this->getError()) {
@fclose($fp);
return false;
}
// Get the rest of the data
$rest_size = akstringlen($sourceNameOrData) - $free_space;
if ($rest_size > 0) {
// Create new part
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new JPA part file ' . basename($this->_dataFileName));
@fclose($fp);
return false;
} else {
// Close the old data file
@fclose($fp);
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
return false;
}
}
$zdata = substr($sourceNameOrData, -$rest_size);
}
$bytes_left = $rest_size;
}
// end while
}
}
} else {
// IMPORTANT! Only this case can be spanned across steps: uncompressed, non-virtual data
// Load cached data if we're resumming file packing
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
$sourceNameOrData = $configuration->get('volatile.engine.archiver.sourceNameOrData', '');
$unc_len = $configuration->get('volatile.engine.archiver.unc_len', 0);
$resume = $configuration->get('volatile.engine.archiver.resume', 0);
}
// Copy the file contents, ignore directories
$zdatafp = @fopen($sourceNameOrData, "rb");
if ($zdatafp === FALSE) {
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
@fclose($fp);
return false;
} else {
// Seek to the resume point if required
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
// Seek to new offset
$seek_result = @fseek($zdatafp, $resume);
if ($seek_result === -1) {
// What?! We can't resume!
$this->setError(sprintf('Could not resume packing of file %s. Your archive is damaged!', $sourceNameOrData));
@fclose($zdatafp);
@fclose($fp);
return false;
}
// Doctor the uncompressed size to match the remainder of the data
$unc_len = $unc_len - $resume;
}
if (!$this->_useSplitZIP) {
while (!feof($zdatafp) && $timer->getTimeLeft() > 0 && $unc_len > 0) {
$zdata = fread($zdatafp, AKEEBA_CHUNK);
$this->_fwrite($fp, $zdata, min(akstringlen($zdata), AKEEBA_CHUNK));
$unc_len -= min(akstringlen($zdata), AKEEBA_CHUNK);
if ($this->getError()) {
@fclose($zdatafp);
@fclose($fp);
return false;
}
}
// WARNING!!! The extra $unc_len != 0 check is necessary as PHP won't reach EOF for 0-byte files.
if (!feof($zdatafp) && $unc_len != 0) {
// We have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
@fclose($zdatafp);
@fclose($fp);
return true;
}
} else {
// Split JPA - Do we have enough space to host the whole file?
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space >= $unc_len) {
// Yes, it will fit inside this part, do quick copy
while (!feof($zdatafp) && $timer->getTimeLeft() > 0 && $unc_len > 0) {
$zdata = fread($zdatafp, AKEEBA_CHUNK);
$this->_fwrite($fp, $zdata, min(akstringlen($zdata), AKEEBA_CHUNK));
//$unc_len -= min(akstringlen($zdata), AKEEBA_CHUNK);
$unc_len -= AKEEBA_CHUNK;
if ($this->getError()) {
@fclose($zdatafp);
@fclose($fp);
return false;
}
}
//if(!feof($zdatafp) && ($unc_len != 0))
if (!feof($zdatafp) && $unc_len > 0) {
// We have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
@fclose($zdatafp);
@fclose($fp);
return true;
}
} else {
// No, we'll have to split between parts. We'll loop until we run
// out of space.
while (!feof($zdatafp) && $timer->getTimeLeft() > 0) {
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
// Find optimal chunk size
$chunk_size_primary = min(AKEEBA_CHUNK, $free_space);
if ($chunk_size_primary <= 0) {
$chunk_size_primary = max(AKEEBA_CHUNK, $free_space);
}
// Calculate if we have to read some more data (smaller chunk size)
// and how many times we must read w/ the primary chunk size
$chunk_size_secondary = $free_space % $chunk_size_primary;
$loop_times = ($free_space - $chunk_size_secondary) / $chunk_size_primary;
// Read and write with the primary chunk size
for ($i = 1; $i <= $loop_times; $i++) {
$zdata = fread($zdatafp, $chunk_size_primary);
$this->_fwrite($fp, $zdata, min(akstringlen($zdata), $chunk_size_primary));
//$unc_len -= min(akstringlen($zdata), $chunk_size_primary);
$unc_len -= $chunk_size_primary;
if ($this->getError()) {
@fclose($zdatafp);
@fclose($fp);
return false;
}
// Do we have enough time to proceed?
//if( (!feof($zdatafp)) && ($unc_len != 0) && ($timer->getTimeLeft() <= 0) ) {
if (!feof($zdatafp) && $unc_len >= 0 && $timer->getTimeLeft() <= 0) {
// No, we have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
@fclose($zdatafp);
@fclose($fp);
return true;
}
}
// Read and write w/ secondary chunk size, if non-zero
if ($chunk_size_secondary > 0) {
$zdata = fread($zdatafp, $chunk_size_secondary);
$this->_fwrite($fp, $zdata, min(akstringlen($zdata), $chunk_size_secondary));
//$unc_len -= min(akstringlen($zdata), $chunk_size_secondary);
$unc_len -= $chunk_size_secondary;
if ($this->getError()) {
@fclose($zdatafp);
@fclose($fp);
return false;
}
}
// Do we have enough time to proceed?
//if( (!feof($zdatafp)) && ($unc_len != 0) && ($timer->getTimeLeft() <= 0) ) {
if (!feof($zdatafp) && $unc_len >= 0 && $timer->getTimeLeft() <= 0) {
// No, we have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
// ...and create a new part as well
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new JPA part file ' . basename($this->_dataFileName));
@fclose($zdatafp);
@fclose($fp);
return false;
}
// ...then, return
@fclose($zdatafp);
@fclose($fp);
return true;
}
// Create new JPA part, but only if we'll have more data to write
//if(!feof($zdatafp) && ($unc_len != 0) && ($unc_len > 0) )
if (!feof($zdatafp) && $unc_len > 0) {
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new JPA part file ' . basename($this->_dataFileName));
@fclose($zdatafp);
@fclose($fp);
return false;
} else {
// Close the old data file
fclose($fp);
// We have created the part. If the user asked for immediate post-proc, break step now.
if ($configuration->get('engine.postproc.common.after_part', 0)) {
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
$configuration->set('volatile.breakflag', true);
@fclose($zdatafp);
@fclose($fp);
return true;
}
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
@fclose($zdatafp);
return false;
}
}
}
}
// end while
}
}
@fclose($zdatafp);
}
}
} elseif ($isSymlink) {
$this->_fwrite($fp, @readlink($sourceNameOrData));
}
@fclose($fp);
//AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "DEBUG -- Added $targetName to archive");
// Uncache data
$configuration->set('volatile.engine.archiver.sourceNameOrData', null);
$configuration->set('volatile.engine.archiver.unc_len', null);
$configuration->set('volatile.engine.archiver.resume', null);
$configuration->set('volatile.engine.archiver.processingfile', false);
// ... and return TRUE = success
return TRUE;
}
/**
* The most basic file transaction: add a single entry (file or directory) to
* the archive.
*
* @param bool $isVirtual If true, the next parameter contains file data instead of a file name
* @param string $sourceNameOrData Absolute file name to read data from or the file data itself is $isVirtual is true
* @param string $targetName The (relative) file name under which to store the file in the archive
*
* @return True on success, false otherwise
*/
protected function _addFile($isVirtual, &$sourceNameOrData, $targetName)
{
static $configuration;
// Note down the starting disk number for Split ZIP archives
if ($this->_useSplitZIP) {
$starting_disk_number_for_this_file = $this->_currentFragment - 1;
} else {
$starting_disk_number_for_this_file = 0;
}
if (!$configuration) {
$configuration = AEFactory::getConfiguration();
}
if (!$configuration->get('volatile.engine.archiver.processingfile', false)) {
// See if it's a directory
$isDir = $isVirtual ? false : is_dir($sourceNameOrData);
// See if it's a symlink (w/out dereference)
$isSymlink = false;
if ($this->_symlink_store_target && !$isVirtual) {
$isSymlink = is_link($sourceNameOrData);
}
// Get real size before compression
if ($isVirtual) {
$fileSize = function_exists('mb_strlen') ? mb_strlen($sourceNameOrData, '8bit') : strlen($sourceNameOrData);
} else {
if ($isSymlink) {
$fileSize = function_exists('mb_strlen') ? mb_strlen(@readlink($sourceNameOrData), '8bit') : strlen(@readlink($sourceNameOrData));
} else {
$fileSize = $isDir ? 0 : @filesize($sourceNameOrData);
}
}
// Get last modification time to store in archive
$ftime = $isVirtual ? time() : @filemtime($sourceNameOrData);
// Decide if we will compress
if ($isDir || $isSymlink) {
$compressionMethod = 0;
// don't compress directories...
} else {
// Do we have plenty of memory left?
$memLimit = ini_get("memory_limit");
if (strstr($memLimit, 'M')) {
$memLimit = (int) $memLimit * 1048576;
} elseif (strstr($memLimit, 'K')) {
$memLimit = (int) $memLimit * 1024;
} elseif (strstr($memLimit, 'G')) {
$memLimit = (int) $memLimit * 1073741824;
} else {
$memLimit = (int) $memLimit;
}
if ($memLimit == "" || $fileSize >= _AKEEBA_COMPRESSION_THRESHOLD) {
// No memory limit, or over 1Mb files => always compress up to 1Mb files (otherwise it times out)
$compressionMethod = $fileSize <= _AKEEBA_COMPRESSION_THRESHOLD ? 8 : 0;
} elseif (function_exists("memory_get_usage")) {
// PHP can report memory usage, see if there's enough available memory; Joomla! alone eats about 5-6Mb! This code is called on files <= 1Mb
$memLimit = $this->_return_bytes($memLimit);
$availableRAM = $memLimit - memory_get_usage();
$compressionMethod = $availableRAM / 2.5 >= $fileSize ? 8 : 0;
} else {
// PHP can't report memory usage, compress only files up to 512Kb (conservative approach) and hope it doesn't break
$compressionMethod = $fileSize <= 524288 ? 8 : 0;
}
}
$compressionMethod = function_exists("gzcompress") ? $compressionMethod : 0;
$storedName = $targetName;
if ($isVirtual) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, ' Virtual add:' . $storedName . ' (' . $fileSize . ') - ' . $compressionMethod);
}
/* "Local file header" segment. */
$unc_len = $fileSize;
// File size
if (!$isDir) {
// Get CRC for regular files, not dirs
if ($isVirtual) {
$crc = crc32($sourceNameOrData);
} else {
$crcCalculator = new AECRC32CalcClass();
$crc = $crcCalculator->crc32_file($sourceNameOrData, $this->AkeebaPackerZIP_CHUNK_SIZE);
// This is supposed to be the fast way to calculate CRC32 of a (large) file.
unset($crcCalculator);
// If the file was unreadable, $crc will be false, so we skip the file
if ($crc === false) {
$this->setWarning('Could not calculate CRC32 for ' . $sourceNameOrData);
return false;
}
}
} else {
if ($isSymlink) {
$crc = crc32(@readlink($sourceNameOrData));
} else {
// Dummy CRC for dirs
$crc = 0;
$storedName .= "/";
$unc_len = 0;
}
}
// If we have to compress, read the data in memory and compress it
if ($compressionMethod == 8) {
// Get uncompressed data
if ($isVirtual) {
$udata =& $sourceNameOrData;
} else {
$udata = @file_get_contents($sourceNameOrData);
// PHP > 4.3.0 saves us the trouble
}
if ($udata === false) {
// Unreadable file, skip it. Normally, we should have exited on CRC code above
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
return false;
} else {
// Proceed with compression
$zdata = @gzcompress($udata);
if ($zdata === false) {
// If compression fails, let it behave like no compression was available
$c_len = $unc_len;
$compressionMethod = 0;
} else {
unset($udata);
$zdata = substr(substr($zdata, 0, -4), 2);
$c_len = function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata);
}
}
} else {
$c_len = $unc_len;
}
/* Get the hex time. */
$dtime = dechex($this->_unix2DosTime($ftime));
if ((function_exists('mb_strlen') ? mb_strlen($dtime, '8bit') : strlen($dtime)) < 8) {
$dtime = "00000000";
}
$hexdtime = chr(hexdec($dtime[6] . $dtime[7])) . chr(hexdec($dtime[4] . $dtime[5])) . chr(hexdec($dtime[2] . $dtime[3])) . chr(hexdec($dtime[0] . $dtime[1]));
// Get current data file size
//clearstatcache();
//$old_offset = @filesize( $this->_dataFileName );
// If it's a split ZIP file, we've got to make sure that the header can fit in the part
if ($this->_useSplitZIP) {
// Get header size, taking into account any extra header necessary
$header_size = 30 + (function_exists('mb_strlen') ? mb_strlen($storedName, '8bit') : strlen($storedName));
// Compare to free part space
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space <= $header_size) {
// Not enough space on current part, create new part
if (!$this->_createNewPart()) {
$this->setError('Could not create new ZIP part file ' . basename($this->_dataFileName));
return false;
}
}
}
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
return false;
}
$seek_result = @fseek($fp, 0, SEEK_END);
$old_offset = $seek_result == -1 ? false : @ftell($fp);
if ($old_offset === false) {
@clearstatcache();
$old_offset = @filesize($this->_dataFileName);
}
// Get the file name length in bytes
if (function_exists('mb_strlen')) {
$fn_length = mb_strlen($storedName, '8bit');
} else {
$fn_length = strlen($storedName);
}
$this->_fwrite($fp, $this->_fileHeader);
/* Begin creating the ZIP data. */
if (!$isSymlink) {
$this->_fwrite($fp, "");
/* Version needed to extract. */
} else {
$this->_fwrite($fp, "\n");
/* Version needed to extract. */
}
$this->_fwrite($fp, pack('v', 2048));
/* General purpose bit flag. Bit 11 set = use UTF-8 encoding for filenames & comments */
$this->_fwrite($fp, $compressionMethod == 8 ? "" : "");
/* Compression method. */
$this->_fwrite($fp, $hexdtime);
/* Last modification time/date. */
$this->_fwrite($fp, pack('V', $crc));
/* CRC 32 information. */
if (!isset($c_len)) {
$c_len = $unc_len;
}
$this->_fwrite($fp, pack('V', $c_len));
/* Compressed filesize. */
$this->_fwrite($fp, pack('V', $unc_len));
/* Uncompressed filesize. */
$this->_fwrite($fp, pack('v', $fn_length));
/* Length of filename. */
$this->_fwrite($fp, pack('v', 0));
/* Extra field length. */
$this->_fwrite($fp, $storedName);
/* File name. */
// Cache useful information about the file
if (!$isDir && !$isSymlink && !$isVirtual) {
$configuration->set('volatile.engine.archiver.unc_len', $unc_len);
$configuration->set('volatile.engine.archiver.hexdtime', $hexdtime);
$configuration->set('volatile.engine.archiver.crc', $crc);
$configuration->set('volatile.engine.archiver.c_len', $c_len);
$configuration->set('volatile.engine.archiver.fn_length', $fn_length);
$configuration->set('volatile.engine.archiver.old_offset', $old_offset);
$configuration->set('volatile.engine.archiver.storedName', $storedName);
$configuration->set('volatile.engine.archiver.sourceNameOrData', $sourceNameOrData);
}
} else {
// Since we are continuing archiving, it's an uncompressed regular file. Set up the variables.
$compressionMethod = 1;
$isDir = false;
$isSymlink = false;
$unc_len = $configuration->get('volatile.engine.archiver.unc_len');
$hexdtime = $configuration->get('volatile.engine.archiver.hexdtime');
$crc = $configuration->get('volatile.engine.archiver.crc');
$c_len = $configuration->get('volatile.engine.archiver.c_len');
$fn_length = $configuration->get('volatile.engine.archiver.fn_length');
$old_offset = $configuration->get('volatile.engine.archiver.old_offset');
$storedName = $configuration->get('volatile.engine.archiver.storedName');
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
return false;
}
}
/* "File data" segment. */
if ($compressionMethod == 8) {
// Just dump the compressed data
if (!$this->_useSplitZIP) {
$this->_fwrite($fp, $zdata);
if ($this->getError()) {
return;
}
} else {
// Split ZIP. Check if we need to split the part in the middle of the data.
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space >= (function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata))) {
// Write in one part
$this->_fwrite($fp, $zdata);
if ($this->getError()) {
return;
}
} else {
$bytes_left = function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata);
while ($bytes_left > 0) {
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
// Split between parts - Write a part
$this->_fwrite($fp, $zdata, min(function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata), $free_space));
if ($this->getError()) {
return;
}
// Get the rest of the data
$bytes_left = (function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata)) - $free_space;
if ($bytes_left > 0) {
// Create new part
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new ZIP part file ' . basename($this->_dataFileName));
return false;
} else {
// Close the old data file
fclose($fp);
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
return false;
}
}
$zdata = substr($zdata, -$bytes_left);
}
}
}
}
unset($zdata);
} elseif (!($isDir || $isSymlink)) {
// Virtual file, just write the data!
if ($isVirtual) {
// Just dump the data
if (!$this->_useSplitZIP) {
$this->_fwrite($fp, $sourceNameOrData);
if ($this->getError()) {
return;
}
} else {
// Split ZIP. Check if we need to split the part in the middle of the data.
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space >= (function_exists('mb_strlen') ? mb_strlen($sourceNameOrData, '8bit') : strlen($sourceNameOrData))) {
// Write in one part
$this->_fwrite($fp, $sourceNameOrData);
if ($this->getError()) {
return;
}
} else {
$bytes_left = function_exists('mb_strlen') ? mb_strlen($sourceNameOrData, '8bit') : strlen($sourceNameOrData);
while ($bytes_left > 0) {
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
// Split between parts - Write first part
$this->_fwrite($fp, $sourceNameOrData, min(function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata), $free_space));
if ($this->getError()) {
return;
}
// Get the rest of the data
$rest_size = (function_exists('mb_strlen') ? mb_strlen($sourceNameOrData, '8bit') : strlen($sourceNameOrData)) - $free_space;
if ($rest_size > 0) {
// Create new part if required
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new ZIP part file ' . basename($this->_dataFileName));
return false;
} else {
// Close the old data file
fclose($fp);
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
return false;
}
}
// Get the rest of the compressed data
$zdata = substr($sourceNameOrData, -$rest_size);
}
$bytes_left = $rest_size;
}
}
}
} else {
// IMPORTANT! Only this case can be spanned across steps: uncompressed, non-virtual data
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
$sourceNameOrData = $configuration->get('volatile.engine.archiver.sourceNameOrData', '');
$unc_len = $configuration->get('volatile.engine.archiver.unc_len', 0);
$resume = $configuration->get('volatile.engine.archiver.resume', 0);
}
// Copy the file contents, ignore directories
$zdatafp = @fopen($sourceNameOrData, "rb");
if ($zdatafp === false) {
$this->setWarning('Unreadable file ' . $sourceNameOrData . '. Check permissions');
return false;
} else {
$timer = AEFactory::getTimer();
// Seek to the resume point if required
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
// Seek to new offset
$seek_result = @fseek($zdatafp, $resume);
if ($seek_result === -1) {
// What?! We can't resume!
$this->setError(sprintf('Could not resume packing of file %s. Your archive is damaged!', $sourceNameOrData));
return false;
}
// Doctor the uncompressed size to match the remainder of the data
$unc_len = $unc_len - $resume;
}
if (!$this->_useSplitZIP) {
// For non Split ZIP, just dump the file very fast
while (!feof($zdatafp) && $timer->getTimeLeft() > 0 && $unc_len > 0) {
$zdata = fread($zdatafp, AKEEBA_CHUNK);
$this->_fwrite($fp, $zdata, min(function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata), AKEEBA_CHUNK));
$unc_len -= AKEEBA_CHUNK;
if ($this->getError()) {
return;
}
}
if (!feof($zdatafp) && $unc_len != 0) {
// We have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
return true;
}
} else {
// Split ZIP - Do we have enough space to host the whole file?
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
if ($free_space >= $unc_len) {
// Yes, it will fit inside this part, do quick copy
while (!feof($zdatafp) && $timer->getTimeLeft() > 0 && $unc_len > 0) {
$zdata = fread($zdatafp, AKEEBA_CHUNK);
$this->_fwrite($fp, $zdata, min(function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata), AKEEBA_CHUNK));
$unc_len -= AKEEBA_CHUNK;
if ($this->getError()) {
return;
}
}
if (!feof($zdatafp) && $unc_len != 0) {
// We have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
return true;
}
} else {
// No, we'll have to split between parts. We'll loop until we run
// out of space.
while (!feof($zdatafp) && $timer->getTimeLeft() > 0) {
// No, we'll have to split between parts. Write the first part
// Find optimal chunk size
clearstatcache();
$current_part_size = @filesize($this->_dataFileName);
$free_space = $this->_fragmentSize - ($current_part_size === false ? 0 : $current_part_size);
$chunk_size_primary = min(AKEEBA_CHUNK, $free_space);
if ($chunk_size_primary <= 0) {
$chunk_size_primary = max(AKEEBA_CHUNK, $free_space);
}
// Calculate if we have to read some more data (smaller chunk size)
// and how many times we must read w/ the primary chunk size
$chunk_size_secondary = $free_space % $chunk_size_primary;
$loop_times = ($free_space - $chunk_size_secondary) / $chunk_size_primary;
// Read and write with the primary chunk size
for ($i = 1; $i <= $loop_times; $i++) {
$zdata = fread($zdatafp, $chunk_size_primary);
$this->_fwrite($fp, $zdata, min(function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata), $chunk_size_primary));
$unc_len -= $chunk_size_primary;
if ($this->getError()) {
return;
}
// Do we have enough time to proceed?
if (!feof($zdatafp) && $unc_len != 0 && $timer->getTimeLeft() <= 0) {
// No, we have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
return true;
}
}
// Read and write w/ secondary chunk size, if non-zero
if ($chunk_size_secondary > 0) {
$zdata = fread($zdatafp, $chunk_size_secondary);
$this->_fwrite($fp, $zdata, min(function_exists('mb_strlen') ? mb_strlen($zdata, '8bit') : strlen($zdata), $chunk_size_secondary));
$unc_len -= $chunk_size_secondary;
if ($this->getError()) {
return;
}
}
// Do we have enough time to proceed?
if (!feof($zdatafp) && $unc_len != 0 && $timer->getTimeLeft() <= 0) {
// No, we have to break, or we'll time out!
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
// ...and create a new part as well
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new ZIP part file ' . basename($this->_dataFileName));
return false;
}
// ...then, return
return true;
}
// Create new ZIP part, but only if we'll have more data to write
if (!feof($zdatafp) && $unc_len > 0) {
// Create new ZIP part
if (!$this->_createNewPart()) {
// Die if we couldn't create the new part
$this->setError('Could not create new ZIP part file ' . basename($this->_dataFileName));
return false;
} else {
// Close the old data file
fclose($fp);
// We have created the part. If the user asked for immediate post-proc, break step now.
if ($configuration->get('engine.postproc.common.after_part', 0)) {
$resume = @ftell($zdatafp);
$configuration->set('volatile.engine.archiver.resume', $resume);
$configuration->set('volatile.engine.archiver.processingfile', true);
$configuration->set('volatile.breakflag', true);
@fclose($zdatafp);
@fclose($fp);
return true;
}
// Open data file for output
$fp = @fopen($this->_dataFileName, "ab");
if ($fp === false) {
$this->setError("Could not open archive file {$this->_dataFileName} for append!");
return false;
}
}
}
}
// end while
}
}
fclose($zdatafp);
}
}
} elseif ($isSymlink) {
$this->_fwrite($fp, @readlink($sourceNameOrData));
}
// Done with data file.
fclose($fp);
// Open the central directory file for append
$fp = @fopen($this->_ctrlDirFileName, "ab");
if ($fp === false) {
$this->setError("Could not open Central Directory temporary file for append!");
return false;
}
$this->_fwrite($fp, $this->_ctrlDirHeader);
if (!$isSymlink) {
$this->_fwrite($fp, "");
/* Version made by (always set to 2.0). */
$this->_fwrite($fp, "");
/* Version needed to extract */
$this->_fwrite($fp, pack('v', 2048));
/* General purpose bit flag */
$this->_fwrite($fp, $compressionMethod == 8 ? "" : "");
/* Compression method. */
} else {
// Symlinks get special treatment
$this->_fwrite($fp, "");
/* Version made by (version 2.0 with UNIX extensions). */
$this->_fwrite($fp, "\n");
/* Version needed to extract */
$this->_fwrite($fp, pack('v', 2048));
/* General purpose bit flag */
$this->_fwrite($fp, "");
/* Compression method. */
}
$this->_fwrite($fp, $hexdtime);
/* Last mod time/date. */
$this->_fwrite($fp, pack('V', $crc));
/* CRC 32 information. */
$this->_fwrite($fp, pack('V', $c_len));
/* Compressed filesize. */
if ($compressionMethod == 0) {
// When we are not compressing, $unc_len is being reduced to 0 while backing up.
// With this trick, we always store the correct length, as in this case the compressed
// and uncompressed length is always the same.
$this->_fwrite($fp, pack('V', $c_len));
/* Uncompressed filesize. */
} else {
// When compressing, the uncompressed length differs from compressed length
// and this line writes the correct value.
$this->_fwrite($fp, pack('V', $unc_len));
/* Uncompressed filesize. */
}
$this->_fwrite($fp, pack('v', $fn_length));
/* Length of filename. */
$this->_fwrite($fp, pack('v', 0));
/* Extra field length. */
$this->_fwrite($fp, pack('v', 0));
/* File comment length. */
$this->_fwrite($fp, pack('v', $starting_disk_number_for_this_file));
/* Disk number start. */
$this->_fwrite($fp, pack('v', 0));
/* Internal file attributes. */
if (!$isSymlink) {
$this->_fwrite($fp, pack('V', $isDir ? 0x41ff0010 : 0.0));
/* External file attributes - 'archive' bit set. */
} else {
// For SymLinks we store UNIX file attributes
$this->_fwrite($fp, " €ÿ¡");
/* External file attributes for Symlink. */
}
$this->_fwrite($fp, pack('V', $old_offset));
/* Relative offset of local header. */
$this->_fwrite($fp, $storedName);
/* File name. */
/* Optional extra field, file comment goes here. */
// Finished with Central Directory
fclose($fp);
// Finaly, increase the file counter by one
$this->_totalFileEntries++;
// Uncache data
$configuration->set('volatile.engine.archiver.sourceNameOrData', null);
$configuration->set('volatile.engine.archiver.unc_len', null);
$configuration->set('volatile.engine.archiver.resume', null);
$configuration->set('volatile.engine.archiver.hexdtime', null);
$configuration->set('volatile.engine.archiver.crc', null);
$configuration->set('volatile.engine.archiver.c_len', null);
$configuration->set('volatile.engine.archiver.fn_length', null);
$configuration->set('volatile.engine.archiver.old_offset', null);
$configuration->set('volatile.engine.archiver.storedName', null);
$configuration->set('volatile.engine.archiver.sourceNameOrData', null);
$configuration->set('volatile.engine.archiver.processingfile', false);
// ... and return TRUE = success
return true;
}
private function apply_remote_quotas()
{
$this->setStep('Applying remote storage quotas');
$this->setSubstep('');
// Make sure we are enabled
$config = AEFactory::getConfiguration();
$enableRemote = $config->get('akeeba.quota.remote', 0);
if (!$enableRemote) {
return true;
}
// Get the list of files to kill
if (empty($this->remote_files_killlist)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, 'Applying remote file quotas');
$this->remote_files_killlist = $this->get_remote_quotas();
if (empty($this->remote_files_killlist)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, 'No remote files to apply quotas to were found');
return true;
}
}
// Remove the files
$timer = AEFactory::getTimer();
while ($timer->getRunningTime() && count($this->remote_files_killlist)) {
$filename = array_shift($this->remote_files_killlist);
list($engineName, $path) = explode('://', $filename);
$engine = AEFactory::getPostprocEngine($engineName);
if (!$engine->can_delete) {
continue;
}
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Removing {$filename}");
$result = $engine->delete($path);
if (!$result) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Removal failed: " . $engine->getWarning());
}
}
// Return false if we have more work to do or true if we're done
if (count($this->remote_files_killlist)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Remote file removal will continue in the next step");
return false;
} else {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Remote file quotas applied successfully");
return true;
}
}
/**
* Implements the _run() abstract method
*/
protected function _run()
{
// Check if we are already done
if ($this->getState() == 'postrun') {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, __CLASS__." :: Already finished");
$this->setStep("");
$this->setSubstep("");
return;
}
// Mark ourselves as still running (we will test if we actually do towards the end ;) )
$this->setState('running');
// Check if we are still adding a database dump part to the archive, or if
// we have to post-process a part
if( AEUtilScripting::getScriptingParameter('db.saveasname','normal') != 'output' )
{
$archiver =& AEFactory::getArchiverEngine();
$configuration =& AEFactory::getConfiguration();
if($configuration->get('engine.postproc.common.after_part',0))
{
if(!empty($archiver->finishedPart))
{
$filename = array_shift($archiver->finishedPart);
AEUtilLogger::WriteLog(_AE_LOG_INFO, 'Preparing to post process '.basename($filename));
$post_proc =& AEFactory::getPostprocEngine();
$result = $post_proc->processPart( $filename );
$this->propagateFromObject($post_proc);
if($result === false)
{
$this->setWarning('Failed to process file '.basename($filename));
}
else
{
AEUtilLogger::WriteLog(_AE_LOG_INFO, 'Successfully processed file '.basename($filename));
}
// Should we delete the file afterwards?
if(
$configuration->get('engine.postproc.common.delete_after',false)
&& $post_proc->allow_deletes
&& ($result !== false)
)
{
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, 'Deleting already processed file '.basename($filename));
AEPlatform::unlink($filename);
}
if($post_proc->break_after) {
$configuration->set('volatile.breakflag', true);
return;
}
}
}
if($configuration->get('volatile.engine.archiver.processingfile',false))
{
// We had already started archiving the db file, but it needs more time
$finished = true;
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Continuing adding the SQL dump part to the archive");
$archiver->addFile(null,null,null);
$this->propagateFromObject($archiver);
if($this->getError()) return;
$finished = !$configuration->get('volatile.engine.archiver.processingfile',false);
if($finished)
{
$this->getNextDumpPart();
}
else
{
return;
}
}
}
// Initialize local variables
$db =& $this->getDB();
if($this->getError()) return;
if( !is_object($db) || ($db === false) )
{
$this->setError(__CLASS__.'::_run() Could not connect to database?!');
return;
}
$outData = ''; // Used for outputting INSERT INTO commands
$this->enforceSQLCompatibility(); // Apply MySQL compatibility option
if($this->getError()) return;
// Touch SQL dump file
$nada = "";
$this->writeline($nada);
// Get this table's information
$tableName = $this->nextTable;
$tableAbstract = trim( $this->table_name_map[$tableName] );
$dump_records = $this->tables_data[$tableName]['dump_records'];
// If it is the first run, find number of rows and get the CREATE TABLE command
if( $this->nextRange == 0 )
{
if($this->getError()) return;
$outCreate = '';
if(is_array($this->tables_data[$tableName])) {
if(array_key_exists('create', $this->tables_data[$tableName])) {
$outCreate = $this->tables_data[$tableName]['create'];
}
}
if(empty($outCreate) && !empty($tableName)) {
// The CREATE command wasn't cached. Time to create it. The $type and $dependencies
// variables will be thrown away.
$type = 'table';
$outCreate = $this->get_create($tableAbstract, $tableName, $type, $dependencies);
}
// Write the CREATE command
if(!$this->writeDump($outCreate)) return;
// Create drop statements if required (the key is defined by the scripting engine)
$configuration =& AEFactory::getConfiguration();
if( AEUtilScripting::getScriptingParameter('db.dropstatements',0) )
{
if(array_key_exists('create', $this->tables_data[$tableName])) {
// @todo This looks cheesy...
$dropStatement = $this->createDrop($this->tables_data[$tableName]['create']);
} else {
$type = 'table';
$createStatement = $this->get_create($tableAbstract, $tableName, $type, $dependencies);
$dropStatement = $this->createDrop($createStatement);
}
if(!empty($dropStatement))
{
if(!$this->writeDump($outCreate)) return;
}
}
if( $dump_records )
{
// We are dumping data from a table, get the row count
$this->getRowCount( $tableAbstract );
}
else
{
// We should not dump any data
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Skipping dumping data of " . $tableAbstract);
$this->maxRange = 0;
$this->nextRange = 1;
$outData = '';
$numRows = 0;
}
}
// Check if we have more work to do on this table
$configuration =& AEFactory::getConfiguration();
$batchsize = intval($configuration->get('engine.dump.common.batchsize', 1000));
if($batchsize <= 0) $batchsize = 1000;
if( ($this->nextRange < $this->maxRange) )
{
$timer =& AEFactory::getTimer();
// Get the number of rows left to dump from the current table
$sql = "SELECT * FROM `$tableAbstract`";
if( $this->nextRange == 0 )
{
// First run, get a cursor to all records
$db->setQuery( $sql, 0, $batchsize );
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Beginning dump of " . $tableAbstract);
}
else
{
// Subsequent runs, get a cursor to the rest of the records
$db->setQuery( $sql, $this->nextRange, $batchsize );
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Continuing dump of " . $tableAbstract . " from record #{$this->nextRange}");
}
$this->query = '';
$numRows = 0;
$use_abstract = AEUtilScripting::getScriptingParameter('db.abstractnames', 1);
while( is_array($myRow = $db->loadAssoc(false)) && ( $numRows < ($this->maxRange - $this->nextRange) ) ) {
$this->createNewPartIfRequired();
$numRows++;
$numOfFields = count( $myRow );
if(
(!$this->extendedInserts) || // Add header on simple INSERTs, or...
( $this->extendedInserts && empty($this->query) ) //...on extended INSERTs if there are no other data, yet
)
{
$newQuery = true;
if( $numOfFields > 0 ) $this->query = "INSERT INTO `" . (!$use_abstract ? $tableName : $tableAbstract) . "` VALUES ";
}
else
{
// On other cases, just mark that we should add a comma and start a new VALUES entry
$newQuery = false;
}
$outData = '(';
// Step through each of the row's values
$fieldID = 0;
// Used in running backup fix
$isCurrentBackupEntry = false;
// Fix 1.2a - NULL values were being skipped
if( $numOfFields > 0 ) foreach( $myRow as $value )
{
// The ID of the field, used to determine placement of commas
$fieldID++;
// Fix 2.0: Mark currently running backup as successful in the DB snapshot
if($tableAbstract == '#__ak_stats')
{
if($fieldID == 1)
{
// Compare the ID to the currently running
$statistics =& AEFactory::getStatistics();
$isCurrentBackupEntry = ($value == $statistics->getId());
}
elseif ($fieldID == 6)
{
// Treat the status field
$value = $isCurrentBackupEntry ? 'complete' : $value;
}
}
// Post-process the value
if( is_null($value) )
{
$outData .= "NULL"; // Cope with null values
} else {
// Accommodate for runtime magic quotes
$value = @get_magic_quotes_runtime() ? stripslashes( $value ) : $value;
$outData .= $db->Quote($value);
}
if( $fieldID < $numOfFields ) $outData .= ', ';
} // foreach
$outData .= ')';
if( $numOfFields )
{
// If it's an existing query and we have extended inserts
if($this->extendedInserts && !$newQuery)
{
// Check the existing query size
$query_length = strlen($this->query);
$data_length = strlen($outData);
if( ($query_length + $data_length) > $this->packetSize )
{
// We are about to exceed the packet size. Write the data so far.
$this->query .= ";\n";
if(!$this->writeDump($this->query)) return;
// Then, start a new query
$this->query = '';
$this->query = "INSERT INTO `" . (!$use_abstract ? $tableName : $tableAbstract) . "` VALUES ";
$this->query .= $outData;
}
else
{
// We have room for more data. Append $outData to the query.
$this->query .= ', ';
$this->query .= $outData;
}
}
elseif($this->extendedInserts && $newQuery)
// If it's a brand new insert statement in an extended INSERTs set
{
// Append the data to the INSERT statement
$this->query .= $outData;
// Let's see the size of the dumped data...
$query_length = strlen($this->query);
if($query_length >= $this->packetSize)
{
// This was a BIG query. Write the data to disk.
$this->query .= ";\n";
if(!$this->writeDump($this->query)) return;
// Then, start a new query
$this->query = '';
}
}
else
// It's a normal (not extended) INSERT statement
{
// Append the data to the INSERT statement
$this->query .= $outData;
// Write the data to disk.
$this->query .= ";\n";
if(!$this->writeDump($this->query)) return;
// Then, start a new query
$this->query = '';
}
}
$outData = '';
// Check for imminent timeout
if( $timer->getTimeLeft() <= 0 ) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Breaking dump of $tableAbstract after $numRows rows; will continue on next step");
break;
}
} // for (all rows left)
// Advance the _nextRange pointer
$this->nextRange += ($numRows != 0) ? $numRows : 1;
$this->setStep($tableName);
$this->setSubstep($this->nextRange . ' / ' . $this->maxRange);
} // if more work on the table
// Finalize any pending query
// WARNING! If we do not do that now, the query will be emptied in the next operation and all
// accumulated data will go away...
if(!empty($this->query))
{
$this->query .= ";\n";
if(!$this->writeDump($this->query)) return;
$this->query = '';
}
// Check for end of table dump (so that it happens inside the same operation)
if( !($this->nextRange < $this->maxRange) )
{
// Tell the user we are done with the table
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Done dumping " . $tableAbstract);
if(count($this->tables) == 0)
{
// We have finished dumping the database!
AEUtilLogger::WriteLog(_AE_LOG_INFO, "End of database detected; flushing the dump buffers...");
$null = null;
$this->writeDump($null);
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Database has been successfully dumped to SQL file(s)");
$this->setState('postrun');
$this->setStep('');
$this->setSubstep('');
$this->nextTable = '';
$this->nextRange = 0;
} elseif(count($this->tables) != 0) {
// Switch tables
$this->nextTable = array_shift( $this->tables );
$this->nextRange = 0;
$this->setStep($this->nextTable);
$this->setSubstep('');
}
}
$null = null;
$this->writeline($null);
}
/**
* Try to add some files from the $file_list into the archive
*
* @return boolean True if there were files packed, false otherwise
* (empty filelist or fatal error)
*/
private function pack_files()
{
// Get a reference to the archiver and the timer classes
$archiver = AEFactory::getArchiverEngine();
$timer = AEFactory::getTimer();
$configuration = AEFactory::getConfiguration();
// If post-processing after part creation is enabled, make sure we do post-process each part before moving on
if ($configuration->get('engine.postproc.common.after_part', 0) && !empty($archiver->finishedPart)) {
if ($this->postProcessDonePartFile($archiver, $configuration)) {
return true;
}
}
// If the archiver has work to do, make sure it finished up before continuing
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Continuing file packing from previous step");
$result = $archiver->addFile('', '', '');
$this->propagateFromObject($archiver);
if ($this->getError()) {
return false;
}
// If that was the last step for packing this file, mark a file done
if (!$configuration->get('volatile.engine.archiver.processingfile', false)) {
$this->progressMarkFileDone();
}
}
// Did it finish, or does it have more work to do?
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
// More work to do. Let's just tell our parent that we finished up successfully.
return true;
}
// Normal file backup loop; we keep on processing the file list, packing files as we go.
if (count($this->file_list) == 0) {
// No files left to pack. Return true and let the engine loop
$this->progressMarkFolderDone();
return true;
} else {
AEUtilLogger::WriteLog(_AE_LOG_DEBUG, "Packing files");
$packedSize = 0;
$numberOfFiles = 0;
list($usec, $sec) = explode(" ", microtime());
$opStartTime = (double) $usec + (double) $sec;
$largeFileThreshold = AEFactory::getConfiguration()->get('engine.scan.common.largefile', 10485760);
while (count($this->file_list) > 0) {
$file = @array_shift($this->file_list);
$size = 0;
if (file_exists($file)) {
$size = @filesize($file);
}
// Anticipatory file size algorithm
if ($numberOfFiles > 0 && $size > $largeFileThreshold) {
if (!AEFactory::getConfiguration()->get('akeeba.tuning.nobreak.beforelargefile', 0)) {
// If the file is bigger than the big file threshold, break the step
// to avoid potential timeouts
$this->setBreakFlag();
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Breaking step _before_ large file: " . $file . " - size: " . $size);
// Push the file back to the list.
array_unshift($this->file_list, $file);
// Return true and let the engine loop
return true;
}
}
// Proactive potential timeout detection
// Rough estimation of packing speed in bytes per second
list($usec, $sec) = explode(" ", microtime());
$opEndTime = (double) $usec + (double) $sec;
if ($opEndTime - $opStartTime == 0) {
$_packSpeed = 0;
} else {
$_packSpeed = $packedSize / ($opEndTime - $opStartTime);
}
// Estimate required time to pack next file. If it's the first file of this operation,
// do not impose any limitations.
$_reqTime = $_packSpeed - 0.01 <= 0 ? 0 : $size / $_packSpeed;
// Do we have enough time?
if ($timer->getTimeLeft() < $_reqTime) {
if (!AEFactory::getConfiguration()->get('akeeba.tuning.nobreak.proactive', 0)) {
array_unshift($this->file_list, $file);
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Proactive step break - file: " . $file . " - size: " . $size . " - req. time " . sprintf('%2.2f', $_reqTime));
$this->setBreakFlag();
return true;
}
}
$packedSize += $size;
$numberOfFiles++;
$ret = $archiver->addFile($file, $this->remove_path_prefix, $this->path_prefix);
// If no more processing steps are required, mark a done file
if (!$configuration->get('volatile.engine.archiver.processingfile', false)) {
$this->progressMarkFileDone();
}
// Error propagation
$this->propagateFromObject($archiver);
if ($this->getError()) {
return false;
}
// If this was the first file packed and we've already gone past
// the large file size threshold break the step. Continuing with
// more operations after packing such a big file is increasing
// the risk to hit a timeout.
if ($packedSize > $largeFileThreshold && $numberOfFiles == 1) {
if (!AEFactory::getConfiguration()->get('akeeba.tuning.nobreak.afterlargefile', 0)) {
AEUtilLogger::WriteLog(_AE_LOG_INFO, "Breaking step *after* large file: " . $file . " - size: " . $size);
$this->setBreakFlag();
return true;
}
}
// If we have to continue processing the file, break the file packing loop forcibly
if ($configuration->get('volatile.engine.archiver.processingfile', false)) {
return true;
}
}
// True if we have more files, false if we're done packing
return count($this->file_list) > 0;
}
}
public function execute()
{
// Load the language files
$paths = array(JPATH_ADMINISTRATOR, JPATH_ROOT);
$jlang = JFactory::getLanguage();
$jlang->load('com_akeeba', $paths[0], 'en-GB', true);
$jlang->load('com_akeeba', $paths[1], 'en-GB', true);
$jlang->load('com_akeeba' . '.override', $paths[0], 'en-GB', true);
$jlang->load('com_akeeba' . '.override', $paths[1], 'en-GB', true);
// Get the backup profile and description
$profile = $this->input->get('profile', 1, 'int');
$description = $this->input->get('description', 'Command-line backup', 'string');
$overrides = $this->getOption('override', array(), false);
if (!empty($overrides)) {
$override_message = "\nConfiguration variables overriden in the command line:\n";
$override_message .= implode(', ', array_keys($overrides));
$override_message .= "\n";
} else {
$override_message = "";
}
$debugmessage = '';
if ($this->input->get('debug', -1, 'int') != -1) {
if (!defined('AKEEBADEBUG')) {
define('AKEEBADEBUG', 1);
}
$debugmessage = "*** DEBUG MODE ENABLED ***\n";
}
$version = AKEEBA_VERSION;
$date = AKEEBA_DATE;
$start_backup = time();
$memusage = $this->memUsage();
$phpversion = PHP_VERSION;
$phpenvironment = PHP_SAPI;
$phpos = PHP_OS;
if ($this->input->get('quiet', -1, 'int') == -1) {
$year = gmdate('Y');
echo <<<ENDBLOCK
Akeeba Backup CLI {$version} ({$date})
Copyright (C) 2010-{$year} Nicholas K. Dionysopoulos
-------------------------------------------------------------------------------
Akeeba Backup is Free Software, distributed under the terms of the GNU General
Public License version 3 or, at your option, any later version.
This program comes with ABSOLUTELY NO WARRANTY as per sections 15 & 16 of the
license. See http://www.gnu.org/licenses/gpl-3.0.html for details.
-------------------------------------------------------------------------------
You are using PHP {$phpversion} ({$phpenvironment})
{$debugmessage}
Starting a new backup with the following parameters:
Profile ID {$profile}
Description "{$description}"
{$override_message}
Current memory usage: {$memusage}
ENDBLOCK;
}
// Attempt to use an infinite time limit, in case you are using the PHP CGI binary instead
// of the PHP CLI binary. This will not work with Safe Mode, though.
$safe_mode = true;
if (function_exists('ini_get')) {
$safe_mode = ini_get('safe_mode');
}
if (!$safe_mode && function_exists('set_time_limit')) {
if ($this->input->get('quiet', -1, 'int') == -1) {
echo "Unsetting time limit restrictions.\n";
}
@set_time_limit(0);
} elseif (!$safe_mode) {
if ($this->input->get('quiet', -1, 'int') == -1) {
echo "Could not unset time limit restrictions; you may get a timeout error\n";
}
} else {
if ($this->input->get('quiet', -1, 'int') == -1) {
echo "You are using PHP's Safe Mode; you may get a timeout error\n";
}
}
if ($this->input->get('quiet', -1, 'int') == -1) {
echo "\n";
}
// Log some paths
if ($this->input->get('quiet', -1, 'int') == -1) {
echo "Site paths determined by this script:\n";
echo "JPATH_BASE : " . JPATH_BASE . "\n";
echo "JPATH_ADMINISTRATOR : " . JPATH_ADMINISTRATOR . "\n\n";
}
// Load the engine
$factoryPath = JPATH_ADMINISTRATOR . '/components/com_akeeba/akeeba/factory.php';
define('JPATH_COMPONENT_ADMINISTRATOR', JPATH_ADMINISTRATOR . '/components/com_akeeba');
define('AKEEBAROOT', JPATH_ADMINISTRATOR . '/components/com_akeeba/akeeba');
if (!file_exists($factoryPath)) {
echo "ERROR!\n";
echo "Could not load the backup engine; file does not exist. Technical information:\n";
echo "Path to " . basename(__FILE__) . ": " . __DIR__ . "\n";
echo "Path to factory file: {$factoryPath}\n";
die("\n");
} else {
try {
require_once $factoryPath;
} catch (Exception $e) {
echo "ERROR!\n";
echo "Backup engine returned an error. Technical information:\n";
echo "Error message:\n\n";
echo $e->getMessage() . "\n\n";
echo "Path to " . basename(__FILE__) . ":" . __DIR__ . "\n";
echo "Path to factory file: {$factoryPath}\n";
die("\n");
}
}
// Forced CLI mode settings
define('AKEEBA_PROFILE', $profile);
define('AKEEBA_BACKUP_ORIGIN', 'cli');
// Force loading CLI-mode translation class
$dummy = new AEUtilTranslate();
// Load the profile
AEPlatform::getInstance()->load_configuration($profile);
// Reset Kettenrad and its storage
AECoreKettenrad::reset(array('maxrun' => 0));
AEUtilTempvars::reset(AKEEBA_BACKUP_ORIGIN);
// Setup
$kettenrad = AEFactory::getKettenrad();
$options = array('description' => $description, 'comment' => '');
if (!empty($overrides)) {
AEPlatform::getInstance()->configOverrides = $overrides;
}
$kettenrad->setup($options);
// Dummy array so that the loop iterates once
$array = array('HasRun' => 0, 'Error' => '');
$warnings_flag = false;
while ($array['HasRun'] != 1 && empty($array['Error'])) {
// Recycle the database conenction to minimise problems with database timeouts
$db = AEFactory::getDatabase();
$db->close();
$db->open();
AEUtilLogger::openLog(AKEEBA_BACKUP_ORIGIN);
AEUtilLogger::WriteLog(true, '');
// Apply overrides in the command line
if (!empty($overrides)) {
$config = AEFactory::getConfiguration();
foreach ($overrides as $key => $value) {
$config->set($key, $value);
}
}
// Apply engine optimization overrides
$config = AEFactory::getConfiguration();
$config->set('akeeba.tuning.min_exec_time', 0);
$config->set('akeeba.tuning.nobreak.beforelargefile', 1);
$config->set('akeeba.tuning.nobreak.afterlargefile', 1);
$config->set('akeeba.tuning.nobreak.proactive', 1);
$config->set('akeeba.tuning.nobreak.finalization', 1);
$config->set('akeeba.tuning.settimelimit', 0);
$config->set('akeeba.tuning.nobreak.domains', 0);
$kettenrad->tick();
AEFactory::getTimer()->resetTime();
$array = $kettenrad->getStatusArray();
AEUtilLogger::closeLog();
$time = date('Y-m-d H:i:s \\G\\M\\TO (T)');
$memusage = $this->memUsage();
$warnings = "no warnings issued (good)";
$stepWarnings = false;
if (!empty($array['Warnings'])) {
$warnings_flag = true;
$warnings = "POTENTIAL PROBLEMS DETECTED; " . count($array['Warnings']) . " warnings issued (see below).\n";
foreach ($array['Warnings'] as $line) {
$warnings .= "\t{$line}\n";
}
$stepWarnings = true;
$kettenrad->resetWarnings();
}
if ($this->input->get('quiet', -1, 'int') == -1 || $stepWarnings) {
echo <<<ENDSTEPINFO
Last Tick : {$time}
Domain : {$array['Domain']}
Step : {$array['Step']}
Substep : {$array['Substep']}
Memory used : {$memusage}
Warnings : {$warnings}
ENDSTEPINFO;
}
}
// Clean up
AEUtilTempvars::reset(AKEEBA_BACKUP_ORIGIN);
if (!empty($array['Error'])) {
echo "An error has occurred:\n{$array['Error']}\n\n";
$exitCode = 2;
} else {
if ($this->input->get('quiet', -1, 'int') == -1) {
echo "Backup job finished successfully after approximately " . $this->timeago($start_backup, time(), '', false) . "\n";
}
$exitCode = 0;
}
if ($warnings_flag && $this->input->get('quiet', -1, 'int') == -1) {
$exitCode = 1;
echo "\n" . str_repeat('=', 79) . "\n";
echo "!!!!! W A R N I N G !!!!!\n\n";
echo "Akeeba Backup issued warnings during the backup process. You have to review them\n";
echo "and make sure that your backup has completed successfully. Always test a backup with\n";
echo "warnings to make sure that it is working properly, by restoring it to a local server.\n";
echo "DO NOT IGNORE THIS MESSAGE! AN UNTESTED BACKUP IS AS GOOD AS NO BACKUP AT ALL.\n";
echo "\n" . str_repeat('=', 79) . "\n";
} elseif ($warnings_flag) {
$exitCode = 1;
}
if ($this->input->get('quiet', -1, 'int') == -1) {
echo "Peak memory usage: " . $this->peakMemUsage() . "\n\n";
}
$this->close($exitCode);
}
/**
* Get the a reference to the Akeeba Engine's timer
* @return AECoreTimer
*/
public static function &getTimer()
{
// TODO I should create another Timer, since I could have problems with backup settings
// ie steps too close => backup error. I can't use the same settings for find that error :)
return AEFactory::getTimer();
}
