/** * Get an SplFileObject representing the file to be read. * * @return \SplFileObject The log file object. */ protected function getFile() { if (!isset($this->file)) { $this->file = new \SplFileObject($this->path, 'r'); $this->file->fseek(0, SEEK_END); $this->cursor = $this->file->ftell(); } return $this->file; }
/** * @return int */ public function getTotal() { if ($this->total === null) { $this->csvFile->fseek(0); $lines = 0; while (!$this->csvFile->eof()) { $lines += substr_count($this->csvFile->fread(8192), $this->lineSeparator); } $this->total = (int) $lines; } return $this->total; }
/** * @inheritdoc */ public function getFile() { if ($this->file === null) { try { $this->file = new TempFile(); $this->file->fwrite($this->data); $this->file->fseek(0); } catch (\RuntimeException $e) { throw new TransportException($e->getMessage(), null, $e); } } return $this->file; }
/** * Reads the latest snapshot of the given aggregate identifier. * * @param string $type the aggregate's type * @param string $identifier the aggregate's identifier * @return SerializedDomainEventDataInterface The latest snapshot of the given aggregate identifier * * @throws EventStoreException when reading the <code>snapshotEventFile</code> or reading the <code>eventFile</code> failed */ public function readSnapshotEvent($type, $identifier) { $snapshotEvent = null; $fileSystemSnapshotEvent = $this->readLastSnapshotEntry(); if (null !== $fileSystemSnapshotEvent) { $this->eventFile->fseek($fileSystemSnapshotEvent['bytesToSkip']); $actuallySkipped = $this->eventFile->ftell(); if ($actuallySkipped !== $fileSystemSnapshotEvent['bytesToSkip']) { throw new EventStoreException(sprintf("The skip operation did not actually skip the expected amount of bytes. " . "The event log of aggregate of type %s and identifier %s might be corrupt.", $type, $identifier)); } $snapshotEvent = $fileSystemSnapshotEvent['snapshotEvent']; } return $snapshotEvent; }
/** * Open the file and optionally restore the position * * @return void */ private function _openFile() { $this->_fieldMap = array(); $this->_fieldMapCount = 0; if (!file_exists($this->_filename)) { $this->_file = false; return; } try { $this->_file = new \SplFileObject($this->_filename, 'r'); $firstline = trim(\MUtil_Encoding::removeBOM($this->_file->current(), "\r\n")); if ($firstline) { $this->_fieldMap = call_user_func($this->_splitFunction, $firstline); $this->_fieldMapCount = count($this->_fieldMap); // Check for fields, do not run when empty if (0 === $this->_fieldMapCount) { $this->_file = false; return; } } // Restore old file position if any if (null !== $this->_filepos) { $this->_file->fseek($this->_filepos, SEEK_SET); } // Always move to next, even if there was no first line $this->next(); } catch (\Exception $e) { $this->_file = false; } }
/** * Adds a single line to a CSV document * * @param string[]|string $row a string, an array or an object implementing to '__toString' method * * @return static */ public function insertOne($row) { if (!is_array($row)) { $row = str_getcsv($row, $this->delimiter, $this->enclosure, $this->escape); } $row = $this->formatRow($row); $this->validateRow($row); if (is_null($this->csv)) { $this->csv = $this->getIterator(); } $this->csv->fputcsv($row, $this->delimiter, $this->enclosure); if ("\n" !== $this->newline) { $this->csv->fseek(-1, SEEK_CUR); $this->csv->fwrite($this->newline); } return $this; }
function modifyFileLine($filename, $linenum, $lineText) { $fp = new SplFileObject($filename); $fp->seek($linenum); $line = $fp->current(); $fp->fseek($linenum, SEEK_CUR); $fp->fwrite($lineText); }
/** * フィールドの配列をCSVの行として書き出します。 * @param \SplFileObject $file * @param string[] $fields */ protected function putCSVRecord(\SplFileObject $file, array $fields) { $file->fputcsv(array_map(function (string $field) : string { return preg_replace('/[\\x00-\\x09\\x11\\x7F]+/u', '', strtr($field, ["\r\n" => "\r\n", "\r" => "\r\n", "\n" => "\r\n"])); }, $fields)); $file->fseek(-1, SEEK_CUR); $file->fwrite("\r\n"); }
/** * @param bool $skipColumns * * @return void */ public function rewind($skipColumns = true) { $this->csvFile->fseek(0); if ($skipColumns) { $this->getFile()->fseek($this->getCsvMeta()->getColumnsOffset()); return; } }
private function content(SplFileObject $file) { $result = ""; $file->fseek(0); while (!$file->eof()) { $result .= $file->fread(1024); } return $result; }
/** * Returns coverage object from file. * * @param \SplFileObject $coverageFile Coverage file. * * @return \PHP_CodeCoverage|CodeCoverage */ private function getCoverageObject(\SplFileObject $coverageFile) { if ('<?php' === $coverageFile->fread(5)) { return include $coverageFile->getRealPath(); } $coverageFile->fseek(0); // the PHPUnit 3.x and below return unserialize($coverageFile->fread($coverageFile->getSize())); }
/** * @covers TextFile\Writer\ErasingWriter::write */ public function testWriteMiddleNewLine() { $prependingWriter = new PrependingWriter(); $filePath = $this->createTestFileFromFixtures('complex_singleline_file.txt'); $file = new \SplFileObject($filePath, 'r+'); $file->fseek(5); $prependingWriter->write($file, 'test', true); $file->rewind(); $this->assertEquals('firsttest', trim($file->current())); $file->next(); $this->assertEquals('secondthirdfourthfifth', trim($file->current())); }
/** * Retrieves the Metadata for a Key/Value pair * * Optionally allows a specific position offset in the file * Return Array * - klen: The length of the Key * - vlen: The length of the Value * - length: The total combined length of the Key and the Value * * @param integer $position An offset to seek to in a file * * @return array|bool */ protected function getMetadata($position = null) { if (!is_null($position)) { $this->file->fseek($position); } $metadata = $this->file->fread(8); if ($metadata) { list(, $klen, $vlen) = unpack('N*', $metadata); return (object) ['klen' => $klen, 'vlen' => $vlen, 'length' => $klen + $vlen]; } return false; }
/** * This method merges the data in another array with this array. * * @access public */ public function merge() { $arrays = func_get_args(); $this->writer->fseek($this->writer->ftell()); foreach ($arrays as $values) { if (is_array($values)) { foreach ($values as $value) { $this->writer->fwrite(serialize($value) . PHP_EOL); } } $this->length += count($values); } }
/** * {@inheritdoc} */ public function write(\SplFileObject $file, $text, $newLineAtEnd = false) { $originalSeek = $file->ftell(); // Refresh file size clearstatcache($file->getFilename()); if ($file->getSize() - $file->ftell() > 0) { $contentAfter = $file->fread($file->getSize() - $file->ftell()); } else { $contentAfter = ''; } $file->fseek($originalSeek); $file->fwrite($text . ($newLineAtEnd ? PHP_EOL : '')); $file->fwrite($contentAfter); return $file; }
/** * It skips the file headers * @return boolean * @access private * @final */ private final function skipHead() { if ($this->fileObject->getSize() < 0) { return false; } $this->fileObject->rewind(); $this->fileObject->fseek(7, SEEK_CUR); $mainHead = $this->fileObject->fread(7); if (ord($mainHead[2]) != 0x73) { return false; } $headSize = $this->getBytes($mainHead, 5, 2); $this->fileObject->fseek($headSize - 7, SEEK_CUR); unset($mainHead, $headSize); return true; }
/** * Get raw image data from the SplFileObject. * * @return string * String representation of the raw image binary data */ public function getRaw() { if (!is_null($this->raw)) { return $this->raw; } // Save our current position $position = $this->object->ftell(); $this->object->rewind(); $raw = ''; while ($this->object->valid()) { $raw .= $this->object->fgets(); } // Go back to our original position $this->object->fseek($position); return $this->raw = $raw; }
/** * @param \SplFileObject $file * * @return \Generator */ private function readLines($file) { $pos = -1; $currentLine = ''; while (-1 !== $file->fseek($pos, SEEK_END)) { $char = $file->fgetc(); if (PHP_EOL === $char) { (yield $currentLine); $currentLine = ''; } else { $currentLine = $char . $currentLine; } $pos--; } if (strlen($currentLine) > 0) { (yield $currentLine); } }
/** * Open the file and optionally restore the position * * @return void */ private function _openFile() { if ($this->_firstLineContainsHeaders) { $this->_fieldMap = array(); $this->_fieldMapCount = 0; } if (!file_exists($this->_filename)) { $this->_file = false; return; } try { $this->_file = new \SplFileObject($this->_filename, 'r'); $this->_file->setFlags($this->_fileFlags); $this->_skipBom(); // Read first line for headers when necessary $firstline = $this->_firstLineContainsHeaders ? $this->_file->current() : false; if ($firstline) { $this->_fieldMap = $this->_recode($firstline); $this->_fieldMapCount = count($this->_fieldMap); // Check for fields, do not run when empty if (0 === $this->_fieldMapCount) { $this->_file = false; return; } } // Restore old file position if any if (null !== $this->_filepos) { $this->_file->fseek($this->_filepos, SEEK_SET); // This is apparently needed, because otherwise the import may skip lines $this->_file->current(); } // Always move to next, even if there was no first line $this->next(); } catch (\Exception $e) { $this->_file = false; } }
<?php $fname = tempnam('/tmp', 'foobar'); file_put_contents($fname, 'herpderp'); $spl = new SplFileObject($fname, 'r'); $spl->fseek($spl->getSize() - 4); var_dump($spl->fgets());
/** * Takes the file objec for a history file and seeks to the beginning of the tempData-section. * @param SplFileObject $historyFileObject Uncompiled history-file * @return int The bytesize of the tempData-section*/ private static function seekHistoryFileTempData($historyFileObject) { $historyFileObject->fseek(-4, SEEK_END); $tempDataSize = unpack('N', $historyFileObject->fread(4))[1]; $historyFileObject->fseek(-4 - $tempDataSize, SEEK_END); return $tempDataSize; }
/** * @param int $limit * * @return Task[] * * @throws \RuntimeException */ protected function fetchTasks($limit) { $logger = Logger::getLogger('file-queue'); $statusFileObject = $this->getStatusFileObject(); if (!$statusFileObject->flock(LOCK_EX)) { throw new \RuntimeException("Can't get lock for status file {$this->getStatusFile()}"); } $seek = $statusFileObject->fgets(); if ($seek === false) { throw new \RuntimeException("Can't get seek from status file {$this->getStatusFile()}"); } if (!$seek) { $seek = 0; } $seek = (int) $seek; $tasksFileObject = new \SplFileObject($this->getTasksFile(), 'r'); if ($tasksFileObject->fseek($seek) !== 0) { throw new \RuntimeException("Can't seek to {$seek} in tasks file {$this->getTasksFile()}"); } $logger->debug("Current seek {$seek}"); $tasks = []; $taskNumber = 0; while ($taskNumber < $limit && !$tasksFileObject->eof()) { $this->getProcessor()->getSignalHandler()->dispatch(); $row = $tasksFileObject->fgetcsv(); if ($row === false) { throw new \RuntimeException("Can't get row from file {$this->getTasksFile()}"); } // пропускаем пустые строки if (!$row || $row === [null]) { continue; } $seek = $tasksFileObject->ftell(); if ($seek === false) { throw new \RuntimeException("Can't get seek from file {$this->getTasksFile()}"); } if (count($row) < 2) { $eventId = $row[0]; $data = null; } else { list($eventId, $data) = $row; } $tasks[] = new Task($eventId, $data); ++$taskNumber; } if ($statusFileObject->fseek(0) !== 0) { throw new \RuntimeException("Can't seek to 0 in status file {$this->getStatusFile()}"); } if (!$statusFileObject->ftruncate(0)) { throw new \RuntimeException("Can't truncate status file {$this->getStatusFile()}"); } if (!$statusFileObject->fwrite($seek)) { throw new \RuntimeException("Can't write new seek {$seek} to status file {$this->getStatusFile()}"); } $logger->debug("New seek {$seek}"); if (!$statusFileObject->flock(LOCK_UN)) { $logger->warn("Can't release lock"); } return $tasks; }
<?php $obj = new SplFileObject(__FILE__); $obj->fseek(1, 2, 3); $obj->fseek();
/** * grow_generic() * * A function that grows an archive file from already calculated contet list * * * @param string $zip Full path & filename of ZIP Archive file to grow * @param string $tempdir Full path of directory for temporary usage * @param array $state * @return bool True if the creation was successful, array for continuation, false otherwise * */ protected function grow_generic($zip, $tempdir, $state) { $result = false; $exitcode = 255; $output = array(); $zippath = ''; $command = ''; $temp_zip = ''; $excluding_additional = false; $exclude_count = 0; $exclusions = array(); $have_zip_errors = false; $zip_errors_count = 0; $zip_errors = array(); $have_zip_warnings = false; $zip_warnings_count = 0; $zip_warnings = array(); $have_zip_additions = false; $zip_additions_count = 0; $zip_additions = array(); $have_zip_debug = false; $zip_debug_count = 0; $zip_debug = array(); $have_zip_other = false; $zip_other_count = 0; $zip_other = array(); $zip_skipped_count = 0; $zip_using_log_file = false; $logfile_name = ''; $contentfile_name = ''; $contentfile_fp = 0; $contentfile_fp_start = 0; $have_more_content = true; $zip_ignoring_symlinks = false; $zm = null; $lister = null; $visitor = null; $logger = null; $total_size = 0; $total_count = 0; $the_list = array(); $zip_error_encountered = false; $zip_period_expired = false; // Ensure no stale file information clearstatcache(); // Create the monitor function here $zm = new pb_backupbuddy_zip_monitor($this); // $zm->set_burst_max_period( self::ZIP_EXEC_DEFAULT_BURST_MAX_PERIOD )->set_burst_threshold_period( 'auto' )->log_parameters(); $zm->set_burst_size_min($this->get_min_burst_content())->set_burst_size_max($this->get_max_burst_content())->set_burst_current_size_threshold($zm->get_burst_size_min())->log_parameters(); // Set some state on it $zm->set_added_dir_count($state['helper']['dc']); $zm->set_added_file_count($state['helper']['fc']); // Note: could enforce trailing directory separator for robustness if (empty($tempdir) || !file_exists($tempdir)) { // This breaks the rule of single point of exit (at end) but it's early enough to not be a problem $this->log('details', __('Zip process reported: Temporary working directory must be available.', 'it-l10n-backupbuddy')); return false; } // Log the temporary working directory so we might be able to spot problems $this->log('details', __('Zip process reported: Temporary working directory available: ', 'it-l10n-backupbuddy') . '`' . $tempdir . '`'); $this->log('message', __('Zip process reported: Using Exec Mode.', 'it-l10n-backupbuddy')); // Tell which zip version is being used $version = $this->get_zip_version(); if (true === is_array($version)) { 2 == $version['major'] && 0 == $version['minor'] ? $version['minor'] = 'X' : true; $this->log('details', sprintf(__('Zip process reported: Using zip version: %1$s.%2$s', 'it-l10n-backupbuddy'), $version['major'], $version['minor'])); } else { $version = array("major" => "X", "minor" => "Y"); $this->log('details', sprintf(__('Zip process reported: Using zip version: %1$s.%2$s', 'it-l10n-backupbuddy'), $version['major'], $version['minor'])); } // Get the command path for the zip command - should return a trimmed string $zippath = $this->get_command_path(self::COMMAND_ZIP_PATH); // Determine if we are using an absolute path if (!empty($zippath)) { $this->log('details', __('Zip process reported: Using absolute zip path: ', 'it-l10n-backupbuddy') . $zippath); } // Add the trailing slash if required $command = $this->slashify($zippath) . 'zip'; // Notify the start of the step $this->log('details', sprintf(__('Zip process reported: Zip archive continuation step started with step period threshold: %1$ss', 'it-l10n-backupbuddy'), $this->get_step_period())); // In case that took a while use the monitor to try and keep the process alive $zm->burst_end(); $this->get_process_monitor()->checkpoint(); // Temporary convenience $result = true; // This is where we previously calculated this when deriving the list $total_size = $state['zipper']['ts']; $total_count = $state['zipper']['tc']; // Only continue if we have a valid list // This isn't ideal at present but will suffice if (true === $result) { // Check if the version of zip in use supports log file (which will help with memory usage for large sites) if (true === $this->get_zip_supports_log_file()) { // Choose to use log file so quieten stdout - we'll set up the log file later $command .= ' -q'; $zip_using_log_file = true; } // Check if we need to turn off compression by settings (faster but larger backup) if (true !== $this->get_compression()) { $command .= ' -0'; $this->log('details', __('Zip process reported: Zip archive creation compression disabled based on settings.', 'it-l10n-backupbuddy')); } else { $this->log('details', __('Zip process reported: Zip archive creation compression enabled based on settings.', 'it-l10n-backupbuddy')); } // Check if ignoring (not following) symlinks if (true === $this->get_ignore_symlinks()) { // Not all OS support this for command line zip but best to handle it late and just // indicate here it is requested but not supported by OS switch ($this->get_os_type()) { case self::OS_TYPE_NIX: // Want to not follow symlinks so set command option and set flag for later use $command .= ' -y'; $zip_ignoring_symlinks = true; $this->log('details', __('Zip process reported: Zip archive creation symbolic links will not be followed based on settings.', 'it-l10n-backupbuddy')); break; case self::OS_TYPE_WIN: $this->log('details', __('Zip process reported: Zip archive creation symbolic links requested to not be followed based on settings but this option is not supported on this operating system.', 'it-l10n-backupbuddy')); break; default: $this->log('details', __('Zip process reported: Zip archive creation symbolic links requested to not be followed based on settings but this option is not supported on this operating system.', 'it-l10n-backupbuddy')); } } else { $this->log('details', __('Zip process reported: Zip archive creation symbolic links will be followed based on settings.', 'it-l10n-backupbuddy')); } // Check if we are ignoring warnings - meaning can still get a backup even // if, e.g., some files cannot be read if (true === $this->get_ignore_warnings()) { // Note: warnings are being ignored but will still be gathered and logged $this->log('details', __('Zip process reported: Zip archive creation actionable warnings will be ignored based on settings.', 'it-l10n-backupbuddy')); } else { $this->log('details', __('Zip process reported: Zip archive creation actionable warnings will not be ignored based on settings.', 'it-l10n-backupbuddy')); } // We want to "grow" a file with each successive "burst" after the first. If the zip // file doesn't exist when -g is given it will be created - but the problem is that // zip also throws a warning and if we are not ignoring warnings we get caught on this. // Currently we'll check if this is the first burst and if it is we'll remove the // -g option from the command so that the archive file is created by default. // TODO: Later we are always going to have created the archive before we start // adding content (so that we can add the archive comment when the file is small) // and then we'll always use -g option for every burts. $command .= ' -g'; // Set up the log file - if $zip_using_log_file is true it means we can log // directly to the log file from the zip utility so we'll set that up. If it // is false it means the version of zip utility in use cnnot log directly to // file so we'll be accumulating the output of each burst into an array and // at burst completion we'll append the log details to the log file. So in // either case we'll end up with a log file that we process from warnings, etc. // This approach gives us a unified process and also makes it easy to handle // the log over multiple steps if required. $logfile_name = $tempdir . self::ZIP_LOG_FILE_NAME; if (true === $zip_using_log_file) { $command .= " -lf '{$logfile_name}' -li -la"; } // Set temporary directory to store ZIP while it's being generated. $command .= " -b '{$tempdir}'"; // Temporary zip file is _always_ located in the temp dir now and we move it // to the final location after completion if it is a good completion $temp_zip = $tempdir . basename($zip); $command .= " '{$temp_zip}' ."; // Now create the inclusions file in the tempdir $ifile = $tempdir . self::ZIP_INCLUSIONS_FILE_NAME; // Now the tricky bit - we have to determine how we are going to give the lisy of files // to zip to use. Preferred way would be as a parameter that tells it to include the // files listed in the file. Unfortunately there is no such option for zip - a list of // files to include in a zip can only be given as discrete file names on the command line // or read from stdin. Giving a long list of names on the command line is not // feasible so we have to use a stdin based method which is either to cat the file and // pipe it in to zip or we can use an stdin file descriptor redirection to fetch the // contents of the file. We can only use these methods safely on *nix systems and when // exec_dir is not in use. // When we cannot use the stdin approach we have to resort to using the -i@file // parameter along with the -r recursion option so that zip will match the "patterns" // we give it in the file as it recurses the directory tree. This is not an ideal solution // because the recursion can be slow on some servers where there is a big directory tree // and much of it is irrelevant and does not belong to the site - but we have no other choice. // We shouldn't have to use this method very much and it should be ok in many cases // where there isn't much that is superfluous in the directory tree. // So let's make up the final command to execute based on the operational environment // and then we can simply "reuse" the command on each burst, with the addition of the // -g option on bursts after the first. if (true === $this->get_exec_dir_flag() || self::OS_TYPE_WIN === $this->get_os_type()) { // We are running on Windows or using exec_dir so have to use -r and -i@file $command .= " -r -i@" . "'{$ifile}'"; } else { // We aer running under a nice *nix environment so we can use a stdin redirection // approach. Let's just use redirection for now as that avoids having to use cat and // piping. // $command .= " -@"; // $command = "cat '{$ifile}' | " . $command; $command .= " -@"; $command .= " <'{$ifile}'"; } // If we can't use a log file but exec_dir isn't in use we can redirect stderr to stdout // If exec_dir is in use we cannot redirect because of command line escaping so cannot log errors/warnings if (false === $zip_using_log_file) { if (false === $this->get_exec_dir_flag()) { $command .= ' 2>&1'; } else { $this->log('details', sprintf(__('Zip process reported: Zip Errors/Warnings cannot not be logged with this version of zip and exec_dir active', 'it-l10n-backupbuddy'), true)); } } else { // Using log file but need to redirect stderr to null because zip // still seems to send some stuff to stderr as well as to log file. // Note: if exec_dir is in use then we cannot redirect so theer may // be some stuff gets sent to stderr and logged but it's not worth // telling that - if we really need that then we can change the below // into an if/then/else and log the condition. $command .= $this->get_exec_dir_flag() ? "" : " 2>" . $this->get_null_device(); } // Remember our "master" command $master_command = $command; // Remember the "master" inclusions list filename $master_ifile = $ifile; // Use this to memorise the worst exit code we had (where we didn't immediately // bail out because it signalled a bad failure) $max_exitcode = $state['zipper']['mec']; // This is where we want to read the contens from $contentfile_name = $tempdir . self::ZIP_CONTENT_FILE_NAME; // Need to setup where we are going to dip into the content file // and remember the start position so we can test for whether we // actually advanced through our content or not. $contentfile_fp = $state['zipper']['fp']; $contentfile_fp_start = $contentfile_fp; // Do this as close to when we actually want to start monitoring usage // Maybe this is redundant as we have already called this in the constructor. // If we want to do this then we have to call with true to reset monitoring to // start now. $this->get_process_monitor()->initialize_monitoring_usage(); // Now we have our command prototype we can start bursting // Simply build a burst list based on content size. Currently no // look-ahead so the size will always exceed the current size threshold // by some amount. May consider using a look-ahead to see if the next // item would exceed the threshold in which case don't add it (unless it // would be the only content in which case have to add it but also log // a warning). // We'll stop either when nothing more to add or we have exceeded our step // period or we have encountered an error. // Note: we might bail out immediately if previous processing has already // caused us to exceed the step period. We need to detect this as a corner // case otherwise we can go into an infinite loop because we have more // content and no error but we never get a chance to advance through the // content. while ($have_more_content && !($zip_period_expired = $this->exceeded_step_period($this->get_process_monitor()->get_elapsed_time())) && !$zip_error_encountered) { // Populate the content file for zip $ilist = array(); // Tell helper that we are preparing a new burst $zm->burst_begin(); $this->log('details', sprintf(__('Zip process reported: Starting burst number: %1$s', 'it-l10n-backupbuddy'), $zm->get_burst_count())); $this->log('details', sprintf(__('Zip process reported: Current burst size threshold: %1$s bytes', 'it-l10n-backupbuddy'), number_format($zm->get_burst_current_size_threshold(), 0, ".", ""))); // Open the content list file and seek to the "current" position. This // will be initially zero and then updated after each burst. For multi-step // it will be zero on the first step and then would be passed back in // as a parameter on subsequent steps based on where in the file the previous // step reached. // TODO: Maybe a sanity check to make sure position seems tenable try { $contentfile = new SplFileObject($contentfile_name, "rb"); $contentfile->fseek($contentfile_fp); // Helper keeps track of what is being added to the burst content and will // tell us when the content is sufficient for this burst based on it's // criteria - this can adapt to how each successive burst goes. while (!$contentfile->eof() && false === $zm->burst_content_complete()) { // Should be at least one item to grab from the list and then move to next // and remember it for if we drop out because burst content complete, in // that case we'll return to that point in the file at the next burst start. // Check for unserialize failure and bail $item = @unserialize($contentfile->current()); if (false === $item) { throw new Exception('Unserialization of content list data failed: `' . $contentfile->current() . '`'); } $contentfile->next(); $file = $item['relative_path'] . $item['filename']; // We shouldn't have any empty items here as we should have removed them // earlier, but just in case... if (!empty($file)) { $ilist[] = $file; // Call the helper event handler as we add each file to the list $zm->burst_content_added($item); } } // Burst list is completed by way of end of content list file or size threshold if (!$contentfile->eof()) { // We haven't exhausted the content list yet so remember where we // are at for next burst (which may be in a following step) $contentfile_fp = $contentfile->ftell(); } else { // Exhausted the content list so make sure we drop out after this burst // if we don't break out of the loop due to a zip error or reached step // duration limit. We must not schedule a new step if this burst has // exhausted the contents file. $have_more_content = false; } // Finished one way or another so close content list file for this burst unset($contentfile); } catch (Exception $e) { // Something fishy - we should have been able to open the content file... // TODO: We need to bail out totally here I think $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: Zip content list file could not be opened/read - error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); $exitcode = 255; $zip_error_encountered = true; break; } // Retain this for reference for now //file_put_contents( ( dirname( $tempdir ) . DIRECTORY_SEPARATOR . self::ZIP_CONTENT_FILE_NAME ), print_r( $ilist, true ) ); // Make sure we expunge any previous version of the inclusions file if (file_exists($ifile)) { @unlink($ifile); } // Slight kludge for now to make sure each burst content file is uniquely named $ifile = str_replace(".txt", "_" . $zm->get_burst_count() . ".txt", $master_ifile); $file_ok = @file_put_contents($ifile, implode(PHP_EOL, $ilist) . PHP_EOL); if (false === $file_ok || 0 === $file_ok) { // The file write failed for some reason, e.g., no disk space? We need to // bail and set exit code so that problem is apparent $this->log('details', sprintf(__('Zip process reported: Unable to write burst content file: `%1$s`', 'it-l10n-backupbuddy'), $ifile)); $exitcode = 255; $zip_error_encountered = true; break; } unset($ilist); // Remember the current directory and change to the directory being added so that "." is valid in command $working_dir = getcwd(); chdir($state['zipper']['root']); // As we are growing the zip we always use -g so no need to filter it out on first burst in this step $command = $master_command; // Make sure we put the correct burst content file name in the command // Slight kludge for now until we build the command line dynamically each burst $command = str_replace($master_ifile, $ifile, $command); $command = self::OS_TYPE_WIN === $this->get_os_type() ? str_replace('\'', '"', $command) : $command; $this->log('details', sprintf(__('Zip process reported: Burst requests %1$s (directories + files) items with %2$s bytes of content to be added to backup zip archive', 'it-l10n-backupbuddy'), $zm->get_burst_content_count(), $zm->get_burst_content_size())); $this->log('details', sprintf(__('Zip process reported: Using burst content file: `%1$s`', 'it-l10n-backupbuddy'), $ifile)); $this->log('details', __('Zip process reported: ') . $this->get_method_tag() . __(' command', 'it-l10n-backupbuddy') . ': ' . $command); // Allow helper to check how the burst goes $zm->burst_start(); // We need the $output array to contain only output for this burst so // always reset it before invoking exec. $output = array(); @exec($command, $output, $exitcode); // And now we can analyse what happened and plan for next burst if any $zm->burst_stop(); // Wrap up the individual burst handling // Note: because we called exec we basically went into a wait condition and so (on Linux) // we didn't consume any max_execution_time so we never really have to bother about // resetting it. However, it is true that time will have elapsed so if this burst _does_ // take longer than our current burst threshold period then max_execution_time would be // reset - but what this doesn't cover is a _cumulative_ effect of bursts and so we might // consider reworking the mechanism to monitor this separately from the individual burst // period (the confusion relates to this having originally applied to the time based // burst handling fro pclzip rather than teh size based for exec). It could also be more // relevant for Windows that doesn't stop the clock when exec is called. $zm->burst_end(); $this->get_process_monitor()->checkpoint(); // Now if we are not logging directly to file we need to append the $output array // to the log file - first invocation will create the file. if (false === $zip_using_log_file) { $this->log('details', sprintf(__('Zip process reported: Appending zip burst log detail to zip log file: %1$s', 'it-l10n-backupbuddy'), $logfile_name)); try { $logfile = new SplFileObject($logfile_name, "ab"); foreach ($output as $line) { $bytes_written = $logfile->fwrite($line . PHP_EOL); // Be very careful to make sure we had a valid write - in paticular // make sure we didn't write 0 bytes since even an empty line from the // array should have the PHP_EOL bytes written if (null === $bytes_written || 0 === $bytes_written) { throw new Exception('Failed to append to zip log file during zip creation - zip log details will be incomplete but zip exit code will still be valid'); } } unset($logfile); unset($output); } catch (Exception $e) { // Something fishy - we should have been able to open and // write to the log file... $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: Zip log file could not be opened/appended-to - error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); // Put the log file away - safe even if we failed to get a logfile unset($logfile); // And throw away the output array as we cannot use it unset($output); } } // Report progress at end of step $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s (directories + files) items to be added to backup zip archive (end of burst)', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count())); // Work out percentage progress on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count()) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s%% of %2$s (directories + files) total items to be added to backup zip archive (end of burst)', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } // Keep a running total of the backup file size (this is temporary code) // Using our stat() function in case file size exceeds 2GB on a 32 bit PHP system $temp_zip_stats = pluginbuddy_stat::stat($temp_zip); // Only log anything if we got some valid file stats if (false !== $temp_zip_stats) { $this->log('details', sprintf(__('Zip process reported: Accumulated zip archive file size: %1$s bytes', 'it-l10n-backupbuddy'), number_format($temp_zip_stats['dsize'], 0, ".", ""))); } $this->log('details', sprintf(__('Zip process reported: Ending burst number: %1$s', 'it-l10n-backupbuddy'), $zm->get_burst_count())); // Set current working directory back to where we were chdir($working_dir); // We have to check the exit code to decide whether to keep going ot bail out (break). // If we get a 0 exit code ot 18 exit code then keep going and remember we got the 18 // so that we can emit that as the final exit code if applicable. If we get any other // exit code then we must break out immediately. if (0 !== $exitcode && 18 !== $exitcode) { // Zip failure of some sort - must bail out with current exit code $zip_error_encountered = true; } else { // Make sure exit code is always the worst we've had so that when // we've done our last burst we drop out with the correct exit code set // This is really to make sure we drop out with exit code 18 if we had // this in _any_ burst as we would keep going and subsequent burst(s) may // return 0. If we had any other non-zero exit code it would be a "fatal" // error and we would have dropped out immediately anyway. $exitcode = $max_exitcode > $exitcode ? $max_exitcode : ($max_exitcode = $exitcode); } // Now inject a little delay until the next burst. This may be required to give the // server time to catch up with finalizing file creation and/or it may be required to // reduce the average load a little so there isn't a sustained "peak" // Theoretically a sleep could be interrupted by a signal and it would return some // non-zero value or false - but if that is the case it probably signals something // more troubling so there is little point in tryng to "handle" such a condition here. if (0 < ($burst_gap = $this->get_burst_gap())) { $this->log('details', sprintf(__('Zip process reported: Starting burst gap delay of: %1$ss', 'it-l10n-backupbuddy'), $burst_gap)); sleep($burst_gap); } } // Exited the loop for some reason so decide what to do now. // // We only want to invoke another step if the zip period has expired, // there is more content and no error was encountered (based on zip // exit code). For any other combination of conditions that we ended // up exiting the loop for we simply want to drop though and process // final success or failure. // // Expiry of zip period is determined before the start of the loop // (given by $zip_period_expired boolean) // More content is determined by not having reached eof on contents // file during latest burst (given by $have_more_content boolean) - note // we should never enter this function if the contents file eof has // been reached in a previous step so we don't test eof at the start // of the loop, only during the loop when constructing content list for // a burst. // Zip error encountered is determined by the zip utility exit code // during the loop from the latest burst (given by $zip_error_encountered // boolean). // // Note: we might consider having the zip helper give us a state to // restore on it when we create one again - but for now we'll not do that if ($zip_period_expired && $have_more_content && !$zip_error_encountered) { // Conditions are good for running a new step but make sure that we // did actually progress through the content with at least one burst // before zip period expiry was detected. We can do this by checking // that the content file pointer (that we would pass to a next step) // is not the same as the value it had at the start of current step. // If we find no progress then there is a server issue and we need // to bail out with a failure which we can do by setting an error // exit code. // Always report progress at end of step $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s (directories + files) items requested to be added to backup zip archive (end of step)', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count())); // Work out percentage progress on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count()) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s%% of %2$s (directories + files) total items to be added to backup zip archive (end of step)', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } if ($contentfile_fp != $contentfile_fp_start) { // We have advanced through content file $this->log('details', sprintf(__('Zip process reported: Zip archive build step terminated after %1$ss, continuation step will be scheduled', 'it-l10n-backupbuddy'), $this->get_process_monitor()->get_elapsed_time())); // Need to set up the state information we'll need to tell the next // loop how to set things up to continue. Next time around if another // step is required then some of these may be changed and others may // stay the same. // Note: the method tag 'mt' is used to tell zipbuddy exactly which // zipper to use, the one that was picked first time through. $new_state = $state; $new_state['zipper']['fp'] = $contentfile_fp; $new_state['zipper']['mec'] = $max_exitcode; $new_state['zipper']['sp'] = $this->get_step_period(); $new_state['helper']['dc'] = $zm->get_added_dir_count(); $new_state['helper']['fc'] = $zm->get_added_file_count(); // Now we can return directly as we have nothing to clear up return $new_state; } else { // It appears the content file pointer didn't change so we // haven't advanced through the content for some reason so // we need to bail out as there is a risk of getting into // an infinite loop $this->log('details', sprintf(__('Zip process reported: Zip archive build step did not progress through content due to unknown server issue', 'it-l10n-backupbuddy'), $this->get_process_monitor()->get_elapsed_time())); $this->log('details', sprintf(__('Zip process reported: Zip archive build step terminated after %1$ss, continuation step will not be scheduled due to abnormal progress indication', 'it-l10n-backupbuddy'), $this->get_process_monitor()->get_elapsed_time())); // Set a generic exit code to force termination of the build // after what we have so far is processed $exitcode = 255; $zip_error_encountered = true; } } // Convenience for handling different scanarios $result = false; // We can report how many dirs/files finally added $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s (directories + files) items requested to be added to backup zip archive (final)', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count())); // Work out percentage progress on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count()) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s%% of %2$s (directories + files) total items to be added to backup zip archive (final)', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } // Always logging to file one way or another // Always scan the output/logfile for warnings, etc. and show warnings even if user has chosen to ignore them try { $logfile = new SplFileObject($logfile_name, "rb"); while (!$logfile->eof()) { $line = $logfile->current(); $id = $logfile->key(); // Use the line number as unique key for later sorting $logfile->next(); if (preg_match('/^\\s*(zip warning:)/i', $line)) { // Looking for specific types of warning - in particular want the warning that // indicates a file couldn't be read as we want to treat that as a "skipped" // warning that indicates that zip flagged this as a potential problem but // created the zip file anyway - but it would have generated the non-zero exit // code of 18 and we key off that later. All other warnings are not considered // reasons to return a non-zero exit code whilst still creating a zip file so // we'll follow the lead on that and not have other warning types halt the backup. // So we'll try and look for a warning output that looks like it is file related... if (preg_match('/^\\s*(zip warning:)\\s*([^:]*:)\\s*(.*)/i', $line, $matches)) { // Matched to what looks like a file related warning so check particular cases switch (strtolower($matches[2])) { case "could not open for reading:": $zip_warnings[self::ZIP_WARNING_SKIPPED][$id] = trim($line); $zip_warnings_count++; break; case "filtered:": $zip_warnings[self::ZIP_WARNING_FILTERED][$id] = trim($line); $zip_warnings_count++; break; case "filename too long:": $zip_warnings[self::ZIP_WARNING_LONGPATH][$id] = trim($line); $zip_warnings_count++; break; case "unknown add status:": $zip_warnings[self::ZIP_WARNING_GENERIC][$id] = trim($line); $zip_warnings_count++; break; case "name not matched:": $zip_other[self::ZIP_OTHER_GENERIC][$id] = trim($line); $zip_other_count++; break; default: $zip_warnings[self::ZIP_WARNING_GENERIC][$id] = trim($line); $zip_warnings_count++; } } else { // Didn't match to what would look like a file related warning so count it regardless $zip_warnings[self::ZIP_WARNING_GENERIC][$id] = trim($line); $zip_warnings_count++; } } elseif (preg_match('/^\\s*(zip info:)/i', $line)) { // An informational may have associated reason and filename so // check for that if (preg_match('/^\\s*(zip info:)\\s*([^:]*:)\\s*(.*)/i', $line, $matches)) { // Matched to what looks like a file related info so check particular cases switch (strtolower($matches[2])) { case "ignored symlink:": $zip_other[self::ZIP_OTHER_IGNORED_SYMLINK][$id] = trim($line); $zip_other_count++; break; default: $zip_other[self::ZIP_OTHER_GENERIC][$id] = trim($line); $zip_other_count++; } } else { // Didn't match to what would look like a file related info so count it regardless $zip_other[self::ZIP_OTHER_GENERIC][$id] = trim($line); $zip_other_count++; } } elseif (preg_match('/^\\s*(zip error:)/i', $line)) { $zip_errors[$id] = trim($line); $zip_errors_count++; } elseif (preg_match('/^\\s*(adding:)/i', $line)) { // Currently not processing additions entried //$zip_additions[] = trim( $line ); //$zip_additions_count++; } elseif (preg_match('/^\\s*(sd:)/i', $line)) { $zip_debug[$id] = trim($line); $zip_debug_count++; } elseif (preg_match('/^.*(skipped:)\\s*(?P<skipped>\\d+)/i', $line, $matches)) { // Each burst may have some skipped files and each will report separately if (isset($matches['skipped'])) { $zip_skipped_count += $matches['skipped']; } } else { // Currently not processing other entries //$zip_other[] = trim( $line ); //$zip_other_count++; } } unset($logfile); @unlink($logfile_name); } catch (Exception $e) { // Something fishy - we should have been able to open the log file... $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: Zip log file could not be opened - error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); } // Set convenience flags $have_zip_warnings = 0 < $zip_warnings_count; $have_zip_errors = 0 < $zip_errors_count; $have_zip_additions = 0 < $zip_additions_count; $have_zip_debug = 0 < $zip_debug_count; $have_zip_other = 0 < $zip_other_count; // Always report the exit code regardless of whether we might ignore it or not $this->log('details', __('Zip process reported: Zip process exit code: ', 'it-l10n-backupbuddy') . $exitcode); // Always report the number of warnings - even just to confirm that we didn't have any $this->log('details', sprintf(__('Zip process reported: Zip process reported: %1$s warning%2$s', 'it-l10n-backupbuddy'), $zip_warnings_count, 1 == $zip_warnings_count ? '' : 's')); // Always report warnings regardless of whether user has selected to ignore them if (true === $have_zip_warnings) { $this->log_zip_reports($zip_warnings, self::$_warning_desc, "WARNING", self::MAX_WARNING_LINES_TO_SHOW, dirname(dirname($tempdir)) . DIRECTORY_SEPARATOR . 'pb_backupbuddy' . DIRECTORY_SEPARATOR . self::ZIP_WARNINGS_FILE_NAME); } // Always report other reports regardless if (true === $have_zip_other) { // Only report number of informationals if we have any as they are not that important $this->log('details', sprintf(__('Zip process reported: %1$s information%2$s', 'it-l10n-backupbuddy'), $zip_other_count, 1 == $zip_other_count ? 'al' : 'als')); $this->log_zip_reports($zip_other, self::$_other_desc, "INFORMATION", self::MAX_OTHER_LINES_TO_SHOW, dirname(dirname($tempdir)) . DIRECTORY_SEPARATOR . 'pb_backupbuddy' . DIRECTORY_SEPARATOR . self::ZIP_OTHERS_FILE_NAME); } // See if we can figure out what happened - note that $exitcode could be non-zero for actionable warning(s) or error // if ( (no zip file) or (fatal exit code) or (not ignoring warnable exit code) ) // TODO: Handle condition testing with function calls based on mapping exit codes to exit type (fatal vs non-fatal) if (!@file_exists($temp_zip) || 0 != $exitcode && 18 != $exitcode || 18 == $exitcode && !$this->get_ignore_warnings()) { // If we have any zip errors reported show them regardless if (true === $have_zip_errors) { $this->log('details', sprintf(__('Zip process reported: %1$s error%2$s', 'it-l10n-backupbuddy'), $zip_errors_count, 1 == $zip_errors_count ? '' : 's')); foreach ($zip_errors as $line) { $this->log('details', __('Zip process reported: ', 'it-l10n-backupbuddy') . $line); } } // Report whether or not the zip file was created (whether that be in the final or temporary location) if (!@file_exists($temp_zip)) { $this->log('details', __('Zip process reported: Zip Archive file not created - check process exit code.', 'it-l10n-backupbuddy')); } else { $this->log('details', __('Zip process reported: Zip Archive file created but with errors/actionable-warnings so will be deleted - check process exit code and warnings.', 'it-l10n-backupbuddy')); } // The operation has failed one way or another. Note that as the user didn't choose to ignore errors the zip file // is always created in a temporary location and then only moved to final location on success without error or warnings. // Therefore if there is a zip file (produced but with warnings) it will not be visible and will be deleted when the // temporary directory is deleted below. $result = false; } else { // NOTE: Probably the two paths below can be reduced to one because even if we are // ignoring warnings we are still building the zip in temporary location and finally // moving it because we are growing it. // Got file with no error or warnings _or_ with warnings that the user has chosen to ignore if (false === $this->get_ignore_warnings()) { // Because not ignoring warnings the zip archive was built in temporary location so we need to move it $this->log('details', __('Zip process reported: Moving Zip Archive file to local archive directory.', 'it-l10n-backupbuddy')); // Make sure no stale file information clearstatcache(); @rename($temp_zip, $zip); if (@file_exists($zip)) { $this->log('details', __('Zip process reported: Zip Archive file moved to local archive directory.', 'it-l10n-backupbuddy')); $this->log('message', __('Zip process reported: Zip Archive file successfully created with no errors or actionable warnings.', 'it-l10n-backupbuddy')); $this->log_archive_file_stats($zip, array('content_size' => $total_size)); // Temporary for now - try and incorporate into stats logging (makes the stats logging function part of the zip helper class?) $this->log('details', sprintf(__('Zip process reported: Zip Archive file size: %1$s of %2$s (directories + files) actually added', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count() - $zip_skipped_count, $total_count)); // Work out percentage on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count() - $zip_skipped_count) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Zip archive file size: %1$s%% of %2$s (directories + files) actually added', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } $result = true; } else { $this->log('details', __('Zip process reported: Zip Archive file could not be moved to local archive directory.', 'it-l10n-backupbuddy')); $result = false; } } else { // With multi-burst we haev to always build the zip in temp location so always have to move it $this->log('details', __('Zip process reported: Moving Zip Archive file to local archive directory.', 'it-l10n-backupbuddy')); // Make sure no stale file information clearstatcache(); @rename($temp_zip, $zip); if (@file_exists($zip)) { $this->log('details', __('Zip process reported: Zip Archive file moved to local archive directory.', 'it-l10n-backupbuddy')); $this->log('message', __('Zip process reported: Zip Archive file successfully created with no errors (any actionable warnings ignored by user settings).', 'it-l10n-backupbuddy')); $this->log_archive_file_stats($zip, array('content_size' => $total_size)); // Temporary for now - try and incorporate into stats logging (makes the stats logging function part of the zip helper class?) $this->log('details', sprintf(__('Zip process reported: Zip Archive file size: %1$s of %2$s (directories + files) actually added', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count() - $zip_skipped_count, $total_count)); // Work out percentage on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count() - $zip_skipped_count) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Zip archive file size: %1$s%% of %2$s (directories + files) actually added', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } $result = true; } else { $this->log('details', __('Zip process reported: Zip Archive file could not be moved to local archive directory.', 'it-l10n-backupbuddy')); $result = false; } } } } // Cleanup the temporary directory that will have all detritus and maybe incomplete zip file $this->log('details', __('Zip process reported: Removing temporary directory.', 'it-l10n-backupbuddy')); if (!$this->delete_directory_recursive($tempdir)) { $this->log('details', __('Zip process reported: Temporary directory could not be deleted: ', 'it-l10n-backupbuddy') . $tempdir); } return $result; }
/** * @covers TextFile\Reader\SimpleReader::getPreviousCharacterContent */ public function testGetPreviousCharacterContent() { $reader = new SimpleReader(new SimpleWalker()); $filePath = $this->createTestFileFromFixtures('complex_multiline_file.txt'); $file = new \SplFileObject($filePath, 'r+'); $file->fseek(2); $this->assertEquals('i', $reader->getPreviousCharacterContent($file)); }
/** * grow() * * A function that grows an archive file from already calculated contet list * Always cleans up after itself * * * @param string $zip Full path & filename of ZIP Archive file to grow * @param string $tempdir Full path of directory for temporary usage * @param array $state * @return bool True if the creation was successful, array for continuation, false otherwise * */ public function grow($zip, $tempdir, $state) { $za = null; $result = false; $exitcode = 255; $output = array(); $temp_zip = ''; $excluding_additional = false; $exclude_count = 0; $exclusions = array(); $temp_file_compression_threshold = 5; $pre_add_func = ''; $have_zip_errors = false; $zip_errors_count = 0; $zip_errors = array(); $have_zip_warnings = false; $zip_warnings_count = 0; $zip_warnings = array(); $have_zip_additions = false; $zip_additions_count = 0; $zip_additions = array(); $have_zip_debug = false; $zip_debug_count = 0; $zip_debug = array(); $have_zip_other = false; $zip_other_count = 0; $zip_other = array(); $zip_skipped_count = 0; $logfile_name = ''; $contentfile_name = ''; $contentfile_fp = 0; $contentfile_fp_start = 0; $have_more_content = true; $zip_ignoring_symlinks = false; $zm = null; $lister = null; $visitor = null; $logger = null; $total_size = 0; $total_count = 0; $the_list = array(); $count_ignored_symdirs = 0; $saved_ignored_symdirs = array(); $zip_error_encountered = false; $zip_period_expired = false; // Ensure no stale file information clearstatcache(); // Create the helper function here so we can use it outside of the post-add // function. Using all defaults so includes multi-burst and server tickling // for now but with options we can modify this. $zm = new pb_backupbuddy_zip_monitor($this); // $zm->set_burst_max_period( self::ZIP_PCLZIP_DEFAULT_BURST_MAX_PERIOD )->set_burst_threshold_period( 'auto' )->log_parameters(); $zm->set_burst_size_min($this->get_min_burst_content())->set_burst_size_max($this->get_max_burst_content())->set_burst_current_size_threshold($zm->get_burst_size_min())->log_parameters(); // Set some state on it $zm->set_added_dir_count($state['helper']['dc']); $zm->set_added_file_count($state['helper']['fc']); // Note: could enforce trailing directory separator for robustness if (empty($tempdir) || !file_exists($tempdir)) { // This breaks the rule of single point of exit (at end) but it's early enough to not be a problem $this->log('details', __('Zip process reported: Temporary working directory not available: ', 'it-l10n-backupbuddy') . '`' . $tempdir . '`'); return false; } // Log the temporary working directory so we might be able to spot problems $this->log('details', __('Zip process reported: Temporary working directory available: ', 'it-l10n-backupbuddy') . '`' . $tempdir . '`'); $this->log('message', __('Zip process reported: Using Compatibility Mode.', 'it-l10n-backupbuddy')); // Notify the start of the step $this->log('details', sprintf(__('Zip process reported: Zip archive continuation step started with step period threshold: %1$ss', 'it-l10n-backupbuddy'), $this->get_step_period())); // In case that took a while use the monitor to try and keep the process alive $zm->burst_end(); $this->get_process_monitor()->checkpoint(); // Temporary convenience $result = true; // This is where we previously calculated this when deriving the list $total_size = $state['zipper']['ts']; $total_count = $state['zipper']['tc']; // Only continue if we have a valid list // This isn't ideal at present but will suffice if (true === $result) { // We need to force the pclzip library to load at this point if it is // not already loaded so that we can use defined constants it creates // but we don't actually want to create a zip archive at this point. // We can also use this as an early test of being able to use the library // as an exception will be raised if the class does not exist. // Note that this is only really required when zip method caching is // in use, if this is disabled then the library would already have been // loaded by the method testing. try { // Select to just load the pclzip library only and tell it the // temporary directory to use if required (this is only possible // if it hasn't already been loaded and the temp dir defined) $za = new pluginbuddy_PclZip("", true, $tempdir); // We have no purpose for this object any longer, the library // will remain loaded unset($za); $result = true; } catch (Exception $e) { // Something fishy - the methods indicated pclzip but we couldn't find the class $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: pclzip indicated as available method but error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); $result = false; } } // Only continue if we have a valid list // This isn't ideal at present but will suffice if (true === $result) { // Basic argument list (will be used for each burst) $arguments = array(); array_push($arguments, PCLZIP_OPT_REMOVE_PATH, $state['zipper']['root']); if (true !== $this->get_compression()) { // Note: don't need to force use of temporary files for compression $this->log('details', __('Zip process reported: Zip archive creation compression disabled based on settings.', 'it-l10n-backupbuddy')); array_push($arguments, PCLZIP_OPT_NO_COMPRESSION); } else { // Note: force the use of temporary files for compression when file size exceeds given value. // This over-rides the "auto-sense" which is based on memory_limit and this _may_ indicate a // memory availability that is higher than reality leading to memory allocation failure if // trying to compress large files. Set the threshold low enough (specify in MB) so that except in // The tightest memory situations we should be ok. Could have option to force use of temporary // files regardless. $this->log('details', __('Zip process reported: Zip archive creation compression enabled based on settings.', 'it-l10n-backupbuddy')); array_push($arguments, PCLZIP_OPT_TEMP_FILE_THRESHOLD, $temp_file_compression_threshold); } // Check if ignoring (not following) symlinks if (true === $this->get_ignore_symlinks()) { // Want to not follow symlinks so set flag for later use $zip_ignoring_symlinks = true; $this->log('details', __('Zip process reported: Zip archive creation symbolic links will be ignored based on settings.', 'it-l10n-backupbuddy')); } else { $this->log('details', __('Zip process reported: Zip archive creation symbolic links will not be ignored based on settings.', 'it-l10n-backupbuddy')); } // Check if we are ignoring warnings - meaning can still get a backup even // if, e.g., some files cannot be read if (true === $this->get_ignore_warnings()) { // Note: warnings are being ignored but will still be gathered and logged $this->log('details', __('Zip process reported: Zip archive creation actionable warnings will be ignored based on settings.', 'it-l10n-backupbuddy')); } else { $this->log('details', __('Zip process reported: Zip archive creation actionable warnings will not be ignored based on settings.', 'it-l10n-backupbuddy')); } // Set up the log file - for each file added we'll append a log entry to the // log file that maps the result of the add to the nearest equivalent command // line zip log entry and this allows us to eventually process and present the // relevant log details in a consistent manner across different methods which // should cut down on confusion a bit. Note that we'll also try and map the // pclzip exit codes to equivalent zip utility codes but we may have to still // maintain our own code space for those that cannot be mapped - just have to // see how it goes. // This approach gives us a unified process and also makes it easy to handle // the log over multiple steps if required. $logfile_name = $tempdir . self::ZIP_LOG_FILE_NAME; // Temporary zip file is _always_ located in the temp dir now and we move it // to the final location after completion if it is a good completion $temp_zip = $tempdir . basename($zip); // Use anonymous function to weed out the unreadable and non-existent files (common reason for failure) // and possibly symlinks based on user settings. // PclZip will record these files as 'skipped' in the file status and we can post-process to determine // if we had any of these and hence either stop the backup or continue dependent on whether the user // has chosen to ignore warnings or not and/or ignore symlinks or not. // Unfortunately we cannot directly tag the file with the reason why it has been skipped so when we // have to process the skipped items we have to try and work out why it was skipped - but shouldn't // be too hard. // TODO: Consider moving this into the PclZip wrapper and have a method to set the various pre/post // functions or select predefined functions (such as this). if (true) { // Note: This could be simplified - it's written to be extensible but may not need to be $args = '$event, &$header'; $code = ''; // $code .= 'static $symlinks = array(); '; $code .= '$result = true; '; // Handle symlinks - keep the two cases of ignoring/not-ignoring separate for now to make logic more // apparent - but could be merged with different conditional handling // For a valid symlink: is_link() -> true; is_file()/is_dir() -> true; file_exists() -> true // For a broken symlink: is_link() -> true; is_file()/is_dir() -> false; file_exists() -> false // Note: pclzip first tests every file using file_exists() before ever trying to add the file so // for a broken symlink it will _always_ error out immediately it discovers a broken symlink so // we never have a chance to filter these out at this stage. // Note: now that we are generating the file list and not following symlinks at that stage we // never have the situation where we need to remember a symdir prefix to filter out dirs/files // under that symdir (once you have passed "through" a dir symlink the dirs/files under that // do not register as symlinks because they themselves are not so previously when pclzip was // generating the list internally we had to make sure we skipped such dirs/files based on // there being a dir symlink as a prefix to the dir/file path). if (true === $zip_ignoring_symlinks) { // If it's a symlink or it's neither a file nor a directory then ignore it. A broken symlink // will never get this far because pclzip will have choked on it $code .= 'if ( ( true === $result ) && !( @is_link( $header[\'filename\'] ) ) ) { '; $code .= ' if ( @is_file( $header[\'filename\'] ) || @is_dir( $header[\'filename\'] ) ) { '; $code .= ' $result = true; '; // $code .= ' foreach ( $symlinks as $prefix ) { '; // $code .= ' if ( !( false === strpos( $header[\'filename\'], $prefix ) ) ) { '; // $code .= ' $result = false; '; // $code .= ' break; '; // $code .= ' } '; // $code .= ' } '; $code .= ' } else { '; // $code .= ' error_log( "Neither a file nor a directory (ignoring): \'" . $header[\'filename\'] . "\'" ); '; $code .= ' $result = false; '; $code .= ' } '; $code .= '} else { '; // $code .= ' error_log( "File is a symlink (ignoring): \'" . $header[\'filename\'] . "\'" ); '; // $code .= ' $symlinks[] = $header[\'filename\']; '; // $code .= ' error_log( "Symlinks Array: \'" . print_r( $symlinks, true ) . "\'" ); '; $code .= ' $result = false; '; $code .= '} '; } else { // If it's neither a file nor directory then ignore it - a valid symlink will register as a file // or directory dependent on what it is pointing at. A broken symlink will never get this far. // because pclzip will have barfed on its file_exists() check before calling the pre-add. We may // choose later to catch this earlier during the list creation I think. $code .= 'if ( ( true === $result ) && ( @is_file( $header[\'filename\'] ) || @is_dir( $header[\'filename\'] ) ) ) { '; $code .= ' $result = true; '; $code .= '} else { '; // $code .= ' error_log( "Neither a file nor a directory (ignoring): \'" . $header[\'filename\'] . "\'" ); '; $code .= ' $result = false; '; $code .= '} '; } // Add the code block for ignoring unreadable files if (true) { $code .= 'if ( ( true === $result ) && ( @is_readable( $header[\'filename\'] ) ) ) { '; $code .= ' $result = true; '; $code .= '} else { '; // $code .= ' error_log( "File not readable: \'" . $header[\'filename\'] . "\'" ); '; $code .= ' $result = false; '; $code .= '} '; } // Return true (to include file) if file passes conditions otherwise false (to skip file) if not $code .= 'return ( ( true === $result ) ? 1 : 0 ); '; $pre_add_func = create_function($args, $code); } // If we had cause to create a pre add function then add it to the argument list here if (!empty($pre_add_func)) { array_push($arguments, PCLZIP_CB_PRE_ADD, $pre_add_func); } // Add a post-add function for progress monitoring, usage data monitoring, // burst handling and server tickling - using the zip helper object // we created earlier $post_add_func = ''; // if (true) { // // $args = '$event, &$header'; // $code = ''; // $code .= '$result = true; '; // $code .= '$zm = pb_backupbuddy_pclzip_helper::get_instance();'; // $code .= '$result = $zm->event_handler( $event, $header );'; // $code .= 'return $result;'; // // $post_add_func = create_function( $args, $code ); // // } // If we had cause to create a pre add function then add it to the argument list here if (!empty($post_add_func)) { array_push($arguments, PCLZIP_CB_POST_ADD, $post_add_func); } // Remember our "master" arguments $master_arguments = $arguments; // Use this to memorise the worst exit code we had (where we didn't immediately // bail out because it signalled a bad failure) $max_exitcode = $state['zipper']['mec']; // This is where we want to read the contens from $contentfile_name = $tempdir . self::ZIP_CONTENT_FILE_NAME; // Need to setup where we are going to dip into the content file // and remember the start position so we can test for whether we // actually advanced through our content or not. $contentfile_fp = $state['zipper']['fp']; $contentfile_fp_start = $contentfile_fp; // Do this as close to when we actually want to start monitoring usage // Maybe this is redundant as we have already called this in the constructor. // If we want to do this then we have to call with true to reset monitoring to // start now. $this->get_process_monitor()->initialize_monitoring_usage(); // Now we have built our common arguments and we have the list defined we can // start on the bursts. Note that each burst will either succeed with an array // output or will fail and no array. When we get an array we will iterate over // it and generate log file entries. For case where we have a non-fatal warning // condition we change the actual pclzip exit code to be the sam eas the zip // utility exit code (18) and this lets us handle the outcome the same way. In // the case of no array but an error code we map that to an equivalent zip utility // exit code (as much as possible) and then we'll drop out with that and a // logged error that the log file processing will pick up. // Now we have our command prototype we can start bursting // Simply build a burst list based on content size. Currently no // look-ahead so the size will always exceed the current size threshold // by some amount. May consider using a look-ahead to see if the next // item would exceed the threshold in which case don't add it (unless it // would be the only content in which case have to add it but also log // a warning). // We'll stop either when noting more to add or we have exceeded our step // period or we have encountered an error. // Note: we might bail out immediately if previous processing has already // caused us to exceed the step period. while ($have_more_content && !($zip_period_expired = $this->exceeded_step_period($this->get_process_monitor()->get_elapsed_time())) && !$zip_error_encountered) { clearstatcache(); // Populate the content array for zip $ilist = array(); // Keep track of any symdirs that are being ignored $saved_ignored_symdirs = array(); // Tell helper that we are preparing a new burst $zm->burst_begin(); $this->log('details', sprintf(__('Zip process reported: Starting burst number: %1$s', 'it-l10n-backupbuddy'), $zm->get_burst_count())); $this->log('details', sprintf(__('Zip process reported: Current burst size threshold: %1$s bytes', 'it-l10n-backupbuddy'), number_format($zm->get_burst_current_size_threshold(), 0, ".", ""))); // Open the content list file and seek to the "current" position. This // will be initially zero and then updated after each burst. For multi-step // it will be zero on the first step and then would be passed back in // as a parameter on subsequent steps based on where in the file the previous // step reached. // TODO: Maybe a sanity check to make sure position seems tenable try { $contentfile = new SplFileObject($contentfile_name, "rb"); $contentfile->fseek($contentfile_fp); // Helper keeps track of what is being added to the burst content and will // tell us when the content is sufficient for this burst based on it's // criteria - this can adapt to how each successive burst goes. while (!$contentfile->eof() && false === $zm->burst_content_complete()) { // Should be at least one item to grab from the list and then move to next // and remember it for if we drop out because burst content complete, in // that case we'll return to that point in the file at the next burst start. // Check for unserialize failure and bail $item = @unserialize($contentfile->current()); if (false === $item) { throw new Exception('Unserialization of content list data failed: `' . $contentfile->current() . '`'); } $contentfile->next(); $file = $item['absolute_path'] . $item['filename']; // Filter out symdirs if we are ignoring symlinks and record them to log // Because of the way the list creation works this condition indicates // a symlink directory only in the case of ignorign symlinks. If we // were not ignoring symlinks then the "vacant" attribute would be set // if the directory were vacant or alternatively this entry would have // already been filtered out if the symlinked directory were not vacant. // So we must filter it out and move on if (true === $item['directory'] && !isset($item['vacant'])) { $saved_ignored_symdirs[] = $file; } else { // We shouldn't have any empty items here as we should have removed them // earlier, but just in case... if (!empty($file)) { $ilist[] = $file; // Call the helper event handler as we add each file to the list $zm->burst_content_added($item); } } } // Burst list is completed by way of end of content list file or size threshold if (!$contentfile->eof()) { // We haven't exhausted the content list yet so remember where we // are at for next burst $contentfile_fp = $contentfile->ftell(); } else { // Exhausted the content list so make sure we drop out after this burst // if we don't break out of the loop due to a zip error or reached step // duration limit $have_more_content = false; } // Finished one way or another so close content list file for this burst unset($contentfile); } catch (Exception $e) { // Something fishy - we should have been able to open the content file... $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: Zip content list file could not be opened/read - error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); $exitcode = 255; $zip_error_encountered = true; break; } // Retain this for reference for now //file_put_contents( ( dirname( $tempdir ) . DIRECTORY_SEPARATOR . self::ZIP_CONTENT_FILE_NAME ), print_r( $ilist, true ) ); // add() method will create archive file if it doesn't aleady exist //$command = 'add'; $command = 'grow'; // Now create our zip handler object for thsi burst // This should give us a new archive object, if not catch it and bail out // Note we previously loaded the library and defined the temporary directory try { $za = new pluginbuddy_PclZip($temp_zip); $result = true; } catch (Exception $e) { // Something fishy - the methods indicated pclzip but we couldn't find the class $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: pclzip indicated as available method but error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); $exitcode = 255; $zip_error_encountered = true; break; } // Allow helper to check how the burst goes $zm->burst_start(); // Create the argument list for this burst $arguments = array(); array_push($arguments, $ilist); $arguments = array_merge($arguments, $master_arguments); // Showing the "master" arguments // First implode any embedded array in the argument list and truncate the result if too long // Assume no arrays embedded in arrays - currently no reason for that // Make sure that there are no non-printable characters (such as in pre- or post-add function // names created by create_function()) by replacing with "*" using preg_replace() // TODO: Make the summary length configurable so that can see more if required // TODO: Consider mapping pclzip argument identifiers to string representations for clarity $args = '$item'; $code = 'if ( is_array( $item ) ) { $string_item = implode( ",", $item); return ( ( strlen( $string_item ) <= 50 ) ? preg_replace( "/[^[:print:]]/", "*", $string_item ) : "List: " . preg_replace( "/[^[:print:]]/", "*", substr( $string_item, 0, 50 ) ) . "..." ); } else { return preg_replace( "/[^[:print:]]/", "*", $item ); }; '; $imploder_func = create_function($args, $code); $imploded_arguments = array_map($imploder_func, $arguments); $this->log('details', sprintf(__('Zip process reported: Burst requests %1$s (directories + files) items with %2$s bytes of content to be added to backup zip archive', 'it-l10n-backupbuddy'), $zm->get_burst_content_count(), $zm->get_burst_content_size())); $this->log('details', __('Zip process reported: ') . $this->get_method_tag() . __(' command arguments', 'it-l10n-backupbuddy') . ': ' . implode(';', $imploded_arguments)); $zip_output = call_user_func_array(array(&$za, $command), $arguments); // And now we can analyse what happened and plan for next burst if any $zm->burst_stop(); // Wrap up the individual burst handling // Note: because we called exec we basically went into a wait condition and so (on Linux) // we didn't consume any max_execution_time so we never really have to bother about // resetting it. However, it is true that time will have elapsed so if this burst _does_ // take longer than our current burst threshold period then max_execution_time would be // reset - but what this doesn't cover is a _cumulative_ effect of bursts and so we might // consider reworking the mechanism to monitor this separately from the individual burst // period (the confusion relates to this having originally applied to the time based // burst handling fro pclzip rather than teh size based for exec). It could also be more // relevant for Windows that doesn't stop the clock when exec is called. $zm->burst_end(); $this->get_process_monitor()->checkpoint(); // If the output is an array then we need to do a quick iteration over the output // in order to determine whetehr we need to change the exit code from 0 to any other // value (essentially to 18). The alternative is some messy stuff with iterating // around and doing stuff based on whether the log file is available or not. By // doing the preprocessing we can simply bail out at any point if the file cannot be // opened or if a write fails. if (is_array($zip_output)) { // Something reasonable happened // For now we'll assume everything rosy but if we find unreadable // files we'll modify the exit code $exitcode = 0; foreach ($zip_output as $file) { switch ($file['status']) { case "ok": break; case "skipped": // For skipped files need to determine why it was skipped if (true === $zip_ignoring_symlinks && @is_link($file['filename'])) { // Skipped because we are ignoring symlinks and this is a symlink. // This just handles files as we have previously filtered out symdirs } else { // Skipped because probably unreadable or non-existent (catch-all for now) // Change the exit code as this is a warning we want to catch later $exitcode = 18; } break; case "filtered": // Log it and change exit code as this is a warning we want to catch later $exitcode = 18; break; case "filename_too_long": // Log it and change exit code as this is a warning we want to catch later $exitcode = 18; break; default: // Unknown status that we'll not consider for changing exit code } } } else { // Something really failed $exitcode = $za->errorCode(); } // This method never directly produces a log file so we need to append the $zip_output array // to the log file - first invocation will create the file. // We now have our exit code so this iteration is simply to log output if we can. // If we fail to open the log file or there is a falure writing we can just bail out $this->log('details', sprintf(__('Zip process reported: Appending zip burst log detail to zip log file: %1$s', 'it-l10n-backupbuddy'), $logfile_name)); try { $logfile = new SplFileObject($logfile_name, "ab"); // Now handle whether the outcome of the addition if (is_array($zip_output)) { // Something reasonable happened // Note if we have skipped any files $skipped_count = 0; // Now we need to put the log information to file // Need to process each status to determine how to log the outcome // for the item - in particular how to log skipped items as the item // status didn't allow us to give any particular reason for an item // being skipped, so we have to try and deduce that from information // about the item. // Our logs are mapped to format like zip utility uses so we can use // a common log processor subsequently. foreach ($zip_output as $file) { // Use this to amass what we want to write to log file $line = ''; switch ($file['status']) { case "ok": // Item was added ok $line = 'adding: ' . $file['filename']; break; case "skipped": // For skipped files need to determine why it was skipped if (true === $zip_ignoring_symlinks && @is_link($file['filename'])) { // Skipped because we are ignoring symlinks and this is a symlink. // This just handles files as we have previously filtered out symdirs // Just treat as an informational $line = 'zip info: ignored symlink: ' . $file['filename']; } else { // Skipped because probably unreadable or non-existent (catch-all for now) $line = 'zip warning: could not open for reading: ' . $file['filename']; } $skipped_count++; break; case "filtered": // Log that it was filtered for some reason $line = 'zip warning: filtered: ' . $file['filename']; // This counts as a skip because we didn't add it $skipped_count++; break; case "filename_too_long": // Log that the given name was too long $line = 'zip warning: filename too long: ' . $file['filename']; // This counts as a skip because we didn't add it $skipped_count++; break; default: // Hmm, have to assume something was not right so we'll log it as // a warning to be on the safe side $line = 'zip warning: unknown add status: ' . $file['status'] . ': ' . $file['filename']; } // Now try and commit the log line to file $bytes_written = $logfile->fwrite($line . PHP_EOL); // Be very careful to make sure we had a valid write - in paticular // make sure we didn't write 0 bytes since even an empty line from the // array should have the PHP_EOL bytes written if (null === $bytes_written || 0 === $bytes_written) { throw new Exception('Failed to append to zip log file during zip creation - zip log details will be incomplete but zip exit code will still be valid'); } } // Now assemble some optional lines $lines = array(); // Now also add in INFORMATIONALs for any ignored symdirs because these would not have // been included in the build list. They were not included because pclzip would have attempted // to follow them and then we would have had to "filter" them and all entries that pclzip // would have created under them which is just a wster of time - best to not include at all // at tell the user now that we didnt include them foreach ($saved_ignored_symdirs as $ignored_symdir) { $lines[] = 'zip info: ignored symlink: ' . $ignored_symdir . self::NORM_DIRECTORY_SEPARATOR; } // Now add log entry related to skiped files if we did skip any // Make this look like zip utility output to some extent if (0 != $skipped_count) { $lines[] = 'zip warning: Not all files were readable'; $lines[] = ' skipped: ' . $skipped_count; } foreach ($lines as $line) { $bytes_written = $logfile->fwrite($line . PHP_EOL); // Be very careful to make sure we had a valid write - in paticular // make sure we didn't write 0 bytes since even an empty line from the // array should have the PHP_EOL bytes written if (null === $bytes_written || 0 === $bytes_written) { throw new Exception('Failed to append to zip log file during zip creation - zip log details will be incomplete but zip exit code will still be valid'); } } } else { // Have to map exit code and warn that not all warnings/etc may be logged // Something really failed $bytes_written = $logfile->fwrite('zip error: ' . $za->errorInfo(true) . PHP_EOL); // Be very careful to make sure we had a valid write - in paticular // make sure we didn't write 0 bytes since even an empty line from the // array should have the PHP_EOL bytes written if (null === $bytes_written || 0 === $bytes_written) { throw new Exception('Failed to append to zip log file during zip creation - zip log details will be incomplete but zip exit code will still be valid'); } } // Put the log file away - safe even if we failed to get a logfile unset($logfile); // And throw away the output result as we have no further use for it unset($zip_output); } catch (Exception $e) { // Something fishy - we should have been able to open and // write to the log file... $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: Zip log file could not be opened/appended-to - error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); // Put the log file away - safe even if we failed to get a logfile unset($logfile); // And throw away the output result as we cannot use it unset($zip_output); } // Put the zip archive away unset($za); // Report progress at end of step $this->log('details', sprintf(__('Zip process reported: Accumulated burst requested %1$s (directories + files) items requested to be added to backup zip archive (end of burst)', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count())); // Work out percentage progress on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count()) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s%% of %2$s (directories + files) total items to be added to backup zip archive (end of burst)', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } clearstatcache(); // Keep a running total of the backup file size (this is temporary code) // Using our stat() function in case file size exceeds 2GB on a 32 bit PHP system $temp_zip_stats = pluginbuddy_stat::stat($temp_zip); // Only log anything if we got some valid file stats if (false !== $temp_zip_stats) { $this->log('details', sprintf(__('Zip process reported: Accumulated zip archive file size: %1$s bytes', 'it-l10n-backupbuddy'), number_format($temp_zip_stats['dsize'], 0, ".", ""))); } $this->log('details', sprintf(__('Zip process reported: Ending burst number: %1$s', 'it-l10n-backupbuddy'), $zm->get_burst_count())); // Now work out the result of that burst and what to do // If it is an array then append to the cumulative array and continue // otherwise we have an error and we must bail out. So we don't need // the complexity of exec to handle non-fatal errors (as warnings) // Note: in the multi-burst case we will still have the results array // accumulated from previous bursts so we _could_ chose to handle that // but for now we'll just throw that away. At some point we can thnk about // handling the output array. // We have to check the exit code to decide whether to keep going ot bail out (break). // If we get a 0 exit code ot 18 exit code then keep going and remember we got the 18 // so that we can emit that as the final exit code if applicable. If we get any other // exit code then we must break out immediately. if (0 !== $exitcode && 18 !== $exitcode) { // Zip failure of some sort - must bail out with current exit code $zip_error_encountered = true; } else { // Make sure exit code is always the worst we've had so that when // we've done our last burst we drop out with the correct exit code set // This is really to make sure we drop out with exit code 18 if we had // this in _any_ burst as we would keep going and subsequent burst(s) may // return 0. If we had any other non-zero exit code it would be a "fatal" // error and we would have dropped out immediately anyway. $exitcode = $max_exitcode > $exitcode ? $max_exitcode : ($max_exitcode = $exitcode); } // Report progress at end of step $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s (directories + files) items requested to be added to backup zip archive (end of burst)', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count())); // Now inject a little delay until the next burst. This may be required to give the // server time to catch up with finalizing file creation and/or it may be required to // reduce the average load a little so there isn't a sustained "peak" // Theoretically a sleep could be interrupted by a signal and it would return some // non-zero value or false - but if that is the case it probably signals something // more troubling so there is little point in tryng to "handle" such a condition here. if (0 < ($burst_gap = $this->get_burst_gap())) { $this->log('details', sprintf(__('Zip process reported: Starting burst gap delay of: %1$ss', 'it-l10n-backupbuddy'), $burst_gap)); sleep($burst_gap); } } // Exited the loop for some reason so decide what to do now. // If we didn't exit because of exceeding the step period then it's a // normal exit and we'll process accordingly and end up returning true // or false. If we exited because of exceeding step period then we need // to return the current state array to enable next iteration to pick up // where we left off. // Note: we might consider having the zip helper give us a state to // restore on it when we create one again - but for now we'll not do that if ($zip_period_expired && $have_more_content && !$zip_error_encountered) { // Report progress at end of step $this->log('details', sprintf(__('Zip process reported: Accumulated burst requested %1$s (directories + files) items requested to be added to backup zip archive (end of step)', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count())); // Work out percentage progress on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count()) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s%% of %2$s (directories + files) total items to be added to backup zip archive (end of step)', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } if ($contentfile_fp != $contentfile_fp_start) { // We have advanced through content file $this->log('details', sprintf(__('Zip process reported: Zip archive build step terminated after %1$ss, continuation step will be scheduled', 'it-l10n-backupbuddy'), $this->get_process_monitor()->get_elapsed_time())); // Need to set up the state information we'll need to tell the next // loop how to set things up to continue. Next time around if another // step is required then some of these may be changed and others may // stay the same. // Note: the method tag 'mt' is used to tell zipbuddy exactly which // zipper to use, the one that was picked first time through. $new_state = $state; $new_state['zipper']['fp'] = $contentfile_fp; $new_state['zipper']['mec'] = $max_exitcode; $new_state['zipper']['sp'] = $this->get_step_period(); $new_state['helper']['dc'] = $zm->get_added_dir_count(); $new_state['helper']['fc'] = $zm->get_added_file_count(); // Now we can return directly as we have nothing to clear up return $new_state; } else { // It appears the content file pointer didn't change so we // haven't advanced through the content for some reason so // we need to bail out as there is a risk of getting into // an infinite loop $this->log('details', sprintf(__('Zip process reported: Zip archive build step did not progress through content due to unknown server issue', 'it-l10n-backupbuddy'), $this->get_process_monitor()->get_elapsed_time())); $this->log('details', sprintf(__('Zip process reported: Zip archive build step terminated after %1$ss, continuation step will not be scheduled due to abnormal progress indication', 'it-l10n-backupbuddy'), $this->get_process_monitor()->get_elapsed_time())); // Set a generic exit code to force termination of the build // after what we have so far is processed $exitcode = 255; $zip_error_encountered = true; } } // Convenience for handling different scanarios $result = false; // We can report how many dirs/files added $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s (directories + files) items requested to be added to backup zip archive (final)', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count())); // Work out percentage progress on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count()) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Accumulated bursts requested %1$s%% of %2$s (directories + files) total items to be added to backup zip archive (final)', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } // Always logging to file one way or another // Always scan the output/logfile for warnings, etc. and show warnings even if user has chosen to ignore them try { $logfile = new SplFileObject($logfile_name, "rb"); while (!$logfile->eof()) { $line = $logfile->current(); $id = $logfile->key(); // Use the line number as unique key for later sorting $logfile->next(); if (preg_match('/^\\s*(zip warning:)/i', $line)) { // Looking for specific types of warning - in particular want the warning that // indicates a file couldn't be read as we want to treat that as a "skipped" // warning that indicates that zip flagged this as a potential problem but // created the zip file anyway - but it would have generated the non-zero exit // code of 18 and we key off that later. All other warnings are not considered // reasons to return a non-zero exit code whilst still creating a zip file so // we'll follow the lead on that and not have other warning types halt the backup. // So we'll try and look for a warning output that looks like it is file related... if (preg_match('/^\\s*(zip warning:)\\s*([^:]*:)\\s*(.*)/i', $line, $matches)) { // Matched to what looks like a file related warning so check particular cases switch (strtolower($matches[2])) { case "could not open for reading:": $zip_warnings[self::ZIP_WARNING_SKIPPED][$id] = trim($line); $zip_warnings_count++; break; case "filtered:": $zip_warnings[self::ZIP_WARNING_FILTERED][$id] = trim($line); $zip_warnings_count++; break; case "filename too long:": $zip_warnings[self::ZIP_WARNING_LONGPATH][$id] = trim($line); $zip_warnings_count++; break; case "unknown add status:": $zip_warnings[self::ZIP_WARNING_GENERIC][$id] = trim($line); $zip_warnings_count++; break; case "name not matched:": $zip_other[self::ZIP_OTHER_GENERIC][$id] = trim($line); $zip_other_count++; break; default: $zip_warnings[self::ZIP_WARNING_GENERIC][$id] = trim($line); $zip_warnings_count++; } } else { // Didn't match to what would look like a file related warning so count it regardless $zip_warnings[self::ZIP_WARNING_GENERIC][$id] = trim($line); $zip_warnings_count++; } } elseif (preg_match('/^\\s*(zip info:)/i', $line)) { // An informational may have associated reason and filename so // check for that if (preg_match('/^\\s*(zip info:)\\s*([^:]*:)\\s*(.*)/i', $line, $matches)) { // Matched to what looks like a file related info so check particular cases switch (strtolower($matches[2])) { case "ignored symlink:": $zip_other[self::ZIP_OTHER_IGNORED_SYMLINK][$id] = trim($line); $zip_other_count++; break; default: $zip_other[self::ZIP_OTHER_GENERIC][$id] = trim($line); $zip_other_count++; } } else { // Didn't match to what would look like a file related info so count it regardless $zip_other[self::ZIP_OTHER_GENERIC][$id] = trim($line); $zip_other_count++; } } elseif (preg_match('/^\\s*(zip error:)/i', $line)) { $zip_errors[$id] = trim($line); $zip_errors_count++; } elseif (preg_match('/^\\s*(adding:)/i', $line)) { // Currently not processing additions entried //$zip_additions[] = trim( $line ); //$zip_additions_count++; } elseif (preg_match('/^\\s*(sd:)/i', $line)) { $zip_debug[$id] = trim($line); $zip_debug_count++; } elseif (preg_match('/^.*(skipped:)\\s*(?P<skipped>\\d+)/i', $line, $matches)) { // Each burst may have some skipped files and each will report separately if (isset($matches['skipped'])) { $zip_skipped_count += $matches['skipped']; } } else { // Currently not processing other entries //$zip_other[] = trim( $line ); //$zip_other_count++; } } unset($logfile); @unlink($logfile_name); } catch (Exception $e) { // Something fishy - we should have been able to open the log file... $error_string = $e->getMessage(); $this->log('details', sprintf(__('Zip process reported: Zip log file could not be opened - error reported: %1$s', 'it-l10n-backupbuddy'), $error_string)); } // Set convenience flags $have_zip_warnings = 0 < $zip_warnings_count; $have_zip_errors = 0 < $zip_errors_count; $have_zip_additions = 0 < $zip_additions_count; $have_zip_debug = 0 < $zip_debug_count; $have_zip_other = 0 < $zip_other_count; // Always report the exit code regardless of whether we might ignore it or not $this->log('details', __('Zip process reported: Zip process exit code: ', 'it-l10n-backupbuddy') . $exitcode); // Always report the number of warnings - even just to confirm that we didn't have any $this->log('details', sprintf(__('Zip process reported: %1$s warning%2$s', 'it-l10n-backupbuddy'), $zip_warnings_count, 1 == $zip_warnings_count ? '' : 's')); // Always report warnings regardless of whether user has selected to ignore them if (true === $have_zip_warnings) { $this->log_zip_reports($zip_warnings, self::$_warning_desc, "WARNING", self::MAX_WARNING_LINES_TO_SHOW, dirname(dirname($tempdir)) . DIRECTORY_SEPARATOR . 'pb_backupbuddy' . DIRECTORY_SEPARATOR . self::ZIP_WARNINGS_FILE_NAME); } // Always report other reports regardless if (true === $have_zip_other) { // Only report number of informationals if we have any as they are not that important $this->log('details', sprintf(__('Zip process reported: %1$s information%2$s', 'it-l10n-backupbuddy'), $zip_other_count, 1 == $zip_other_count ? 'al' : 'als')); $this->log_zip_reports($zip_other, self::$_other_desc, "INFORMATION", self::MAX_OTHER_LINES_TO_SHOW, dirname(dirname($tempdir)) . DIRECTORY_SEPARATOR . 'pb_backupbuddy' . DIRECTORY_SEPARATOR . self::ZIP_OTHERS_FILE_NAME); } // See if we can figure out what happened - note that $exitcode could be non-zero for actionable warning(s) or error // if ( (no zip file) or (fatal exit code) or (not ignoring warnable exit code) ) // TODO: Handle condition testing with function calls based on mapping exit codes to exit type (fatal vs non-fatal) if (!@file_exists($temp_zip) || 0 != $exitcode && 18 != $exitcode || 18 == $exitcode && !$this->get_ignore_warnings()) { // If we have any zip errors reported show them regardless if (true == $have_zip_errors) { $this->log('details', sprintf(__('Zip process reported: %1$s error%2$s', 'it-l10n-backupbuddy'), $zip_errors_count, 1 == $zip_errors_count ? '' : 's')); foreach ($zip_errors as $line) { $this->log('details', __('Zip process reported: ', 'it-l10n-backupbuddy') . $line); } } // Report whether or not the zip file was created (this will always be in the temporary location) if (!@file_exists($temp_zip)) { $this->log('details', __('Zip process reported: Zip Archive file not created - check process exit code.', 'it-l10n-backupbuddy')); } else { $this->log('details', __('Zip process reported: Zip Archive file created but with errors/actionable-warnings so will be deleted - check process exit code and warnings.', 'it-l10n-backupbuddy')); } // The operation has failed one way or another. Note that for pclzip the zip file is always created in the temporary // location regardless of whether the user selected to ignore errors or not (we can never guarantee to create a valid // zip file because the script might be terminated by the server so we must wait to produce a valid file and then // move it to the final location if it is valid). // Therefore if there is a zip file (produced but with warnings) it will not be visible and will be deleted when the // temporary directory is deleted below. $result = false; } else { // Got file with no error or warnings _or_ with warnings that the user has chosen to ignore // File always built in temporary location so always need to move it $this->log('details', __('Zip process reported: Moving Zip Archive file to local archive directory.', 'it-l10n-backupbuddy')); // Make sure no stale file information clearstatcache(); // Relocate the temporary zip file to final location @rename($temp_zip, $zip); // Check that we moved the file ok if (@file_exists($zip)) { $this->log('details', __('Zip process reported: Zip Archive file moved to local archive directory.', 'it-l10n-backupbuddy')); $this->log('message', __('Zip process reported: Zip Archive file successfully created with no errors (any actionable warnings ignored by user settings).', 'it-l10n-backupbuddy')); $this->log_archive_file_stats($zip, array('content_size' => $total_size)); // Temporary for now - try and incorporate into stats logging (makes the stats logging function part of the zip helper class?) $this->log('details', sprintf(__('Zip process reported: Zip Archive file size: %1$s of %2$s (directories + files) actually added', 'it-l10n-backupbuddy'), $zm->get_added_dir_count() + $zm->get_added_file_count() - $zip_skipped_count, $total_count)); // Work out percentage on items if (0 < $total_count) { $percentage_complete = (int) (($zm->get_added_dir_count() + $zm->get_added_file_count() - $zip_skipped_count) / $total_count * 100); $this->log('details', sprintf(__('Zip process reported: Zip archive file size: %1$s%% of %2$s (directories + files) actually added', 'it-l10n-backupbuddy'), $percentage_complete, $total_count)); } $result = true; } else { $this->log('details', __('Zip process reported: Zip Archive file could not be moved to local archive directory.', 'it-l10n-backupbuddy')); $result = false; } } } // Cleanup the temporary directory that will have all detritus and maybe incomplete zip file $this->log('details', __('Zip process reported: Removing temporary directory.', 'it-l10n-backupbuddy')); if (!$this->delete_directory_recursive($tempdir)) { $this->log('details', __('Zip process reported: Temporary directory could not be deleted: ', 'it-l10n-backupbuddy') . $tempdir); } // if ( null != $za ) { unset( $za ); } return $result; }
public function getSize() { $this->store->fseek(0, SEEK_END); return $this->store->ftell(); }
/** * Generates a uuid. * * @return string */ private function getJobFilename($queueName) { $path = $this->baseDirectory . '/bernard.meta'; if (!is_file($path)) { touch($path); } $file = new \SplFileObject($path, 'r+'); $file->flock(LOCK_EX); $meta = unserialize($file->fgets()); $id = isset($meta[$queueName]) ? $meta[$queueName] : 0; $id++; $filename = sprintf('%d.job', $id); $meta[$queueName] = $id; $content = serialize($meta); $file->fseek(0); $file->fwrite($content, strlen($content)); $file->flock(LOCK_UN); return $filename; }
/** * Get the next character from the input stream, without gettng it. * * @return string The next character from the specified input stream, without advancing the position * in the underlying file. * @see $in * @see get() */ function peek() { if ($this->isString) { if ($this->inPos < $this->inLength) { $c = $this->in[$this->inPos]; } else { return EOF; } } else { // Get next input character $c = $this->in->fgetc(); // Regress position in file $this->in->fseek(-1, SEEK_CUR); // Return character obtained } return $c; }
if ($error_log === null) { $error_log = ini_get('error_log'); } if (empty($error_log)) { die('No error log was defined or could be determined from the ini settings.'); } try { $log = new SplFileObject($error_log); $log->setFlags(SplFileObject::DROP_NEW_LINE); } catch (RuntimeException $e) { die("The file '{$error_log}' cannot be opened for reading.\n"); } if ($cache !== null && file_exists($cache)) { $cacheData = unserialize(file_get_contents($cache)); extract($cacheData); $log->fseek($seek); } $prevError = new stdClass(); while (!$log->eof()) { if (preg_match('/stack trace:$/i', $log->current())) { $stackTrace = $parts = []; $log->next(); while (preg_match('!^\\[(?P<time>[^\\]]*)\\] PHP\\s+(?P<msg>\\d+\\. .*)$!', $log->current(), $parts) || preg_match('!^(?P<msg>#\\d+ .*)$!', $log->current(), $parts) && !$log->eof()) { $stackTrace[] = $parts['msg']; $log->next(); } if (substr($stackTrace[0], 0, 2) == '#0') { $stackTrace[] = $log->current(); $log->next(); } $prevError->trace = join("\n", $stackTrace);