protected function _refreshLocal($myrole, $drivers) { $job = $this->_out->jobStart("traversing file system starting at {$this->_baseDir}"); /* @var $compare Compare_Sqlite */ $compare = $drivers[Core_Engine::ROLE_COMPARE]; $stat = $this->_fileStat; $it = new RecursiveDirectoryIterator($this->_baseDir); $it->setFlags(RecursiveDirectoryIterator::SKIP_DOTS); //$it = new DirnameFilter($it, "/informa/i"); if (isset($this->_options['filter'])) { $this->_out->logNotice(">>>applying filter " . $this->_options['filterType']); $filterConf = $this->_options['filter'][$this->_options['filterType']]; /** @var $filter Filter_Interface */ $filter = new $filterConf['class']($filterConf); $filter->setIterator($it); $it = $filter->applyFilters(); } $itemCount = 0; $this->_trimFromBegin = strlen($this->_baseDir); $compare->updateFromLocalStart(); foreach (new RecursiveIteratorIterator($it, RecursiveIteratorIterator::SELF_FIRST) as $file) { /** @var $file SplFileObject */ //we don't need make entry for not empty folder if ($file->isDir() && !Storage_Filesystem_FileStat::isEmptyDir((string) $file)) { continue; } $file = (string) $file; $obj = Core_FsObject::factoryFromStat($this->_makePath($file), $stat->getStat($file)); $compare->updateFromLocal($obj); $itemCount++; } $compare->updateFromLocalEnd(); $this->_out->jobEnd($job, "updated info about {$itemCount} files"); }
/** * Initialize iterator of tasks to be executed on storage * * @param int $storageType type of storage (Core_Engine::ROLE_REMOTE or Core_Engine::ROLE_LOCAL) * @return bool */ public function initChangesOn($storageType) { switch ($storageType) { case Core_Engine::ROLE_LOCAL: // we don't care at this moment break; case Core_Engine::ROLE_REMOTE: // TODO on this place there should be SQL using union which will be executed against SQL // and its results will be returned on each Iterator::next() // the $storageType defines if we want to receive changes for local or remote storage $cmdDelete = self::CMD_DELETE; $cmdMkdir = self::CMD_MKDIR; $cmdTs = self::CMD_TS; $cmdPut = self::CMD_PUT; // count actions $sql = <<<SQL SELECT '{$cmdDelete}' as action, count(*) FROM {$this->_prefix} WHERE remote and local=0 UNION SELECT '{$cmdMkdir}' as action , count(*)FROM {$this->_prefix} WHERE isdir=1 and local and remote=0 UNION SELECT '{$cmdPut}' as action, count(*) FROM {$this->_prefix} WHERE isdir<>1 and local and (remote=0 or rsize<>lsize or rmd5!=lmd5) UNION SELECT '{$cmdTs}' as action, count(*) FROM {$this->_prefix} WHERE isdir<>1 and local and remote and rmd5=lmd5 and (rts is null or rts<>ltime) SQL; $result = $this->_db->query($sql); // $this->_countInfo = array(); while ($res = $result->fetchArray(SQLITE3_NUM)) { $this->_out->logNotice("will execute {$res['1']} time command {$res['0']}"); // $this->_countInfo[$res[0]] = $res[1]; } // prepare result iteration $sql = <<<SQL SELECT path, ltime, '{$cmdDelete}' as action FROM {$this->_prefix} WHERE remote and local=0 UNION SELECT path, ltime, '{$cmdMkdir}' as action FROM {$this->_prefix} WHERE isdir=1 and local and remote=0 UNION SELECT path, ltime, '{$cmdPut}' as action FROM {$this->_prefix} WHERE isdir<>1 and local and (remote=0 or rsize<>lsize or rmd5!=lmd5) UNION SELECT path, ltime, '{$cmdTs}' as action FROM {$this->_prefix} WHERE isdir<>1 and local and remote and rmd5=lmd5 and (rts is null or rts<>ltime) SQL; $this->_jobSqlResult = $this->_db->query($sql); break; default: throw new Exception("wrong {$storageType} parameter"); } }
public function updateRemote($myrole, $drivers) { if ($this->_options['update'] == 'simulate') { $simulate = true; $this->_out->logWarning("only SIMULATION mode"); } else { if ($this->_options['update'] === false || (int) $this->_options['update'] === 0) { $this->_out->logNotice("skipped, not requested and not needed"); return; } $simulate = false; } /** @var $compare Compare_Interface */ $compare = $drivers['compare']; /** @var $local Storage_Interface */ $local = $drivers['local']; if (!$compare->initChangesOn("remote")) { // TODO not sure, but maybe we will need it } $job = $this->_out->jobStart("updating remote storage"); $this->_out->jobSetProgressStep($job, 1000); foreach ($compare as $task) { $repeat = 3; do { $msg = ""; try { $path = $this->_getPathWithBasedir($task->path, self::ADD_BASE_DIR); switch ($task->action) { case Compare_Interface::CMD_MKDIR: $msg = "mkdir " . $path . " into s3 bucket"; $this->_out->logDebug($msg); if (!$simulate) { // create folders $this->_s3->create_object($this->getBucket(), $path, array('body' => '', 'storage' => $this->_defaultRedundancyStorage)); } break; case Compare_Interface::CMD_PUT: $msg = "put " . $path . " into s3 bucket"; $this->_out->logDebug($msg); $uploadPath = $local->getBaseDir() . $task->path; //fix for windows encoding issue $uploadPath = $local->convertEncodingPath($uploadPath); if (!file_exists($uploadPath)) { $this->_out->logError("file {$uploadPath} does not exists anymore locally"); continue; } if (!$simulate) { //empty directory if (ord(substr($path, -1)) === 47) { //for empty folders we need little different options $this->_out->logWarning("TODO putting empty folder {$path} ... is it possible ?"); $this->_s3->create_object($this->getBucket(), $path, array('body' => '', 'storage' => $this->_defaultRedundancyStorage)); } else { $options = array('fileUpload' => $uploadPath, 'storage' => $this->_defaultRedundancyStorage); // TODO it should be possible to speedup upload of small upload but using S3 batch if ($this->_options['multipart']['big-files']) { // multipart upload for big files if ($this->_options['multipart']['part-size']) { $options['partSize'] = $this->_options['multipart']['part-size']; } $this->_s3->create_mpu_object($this->getBucket(), $path, $options); } else { // normal upload $this->_s3->create_object($this->getBucket(), $path, $options); } } } break; case Compare_Interface::CMD_DELETE: $msg = "deleting " . $path . " from s3 bucket"; $this->_out->logDebug($msg); if (!$simulate) { $this->_s3->delete_object($this->getBucket(), $path); } break; case Compare_Interface::CMD_TS: // storing this information as metadata is too slow to be used // $this->_out->logDebug("remember local timestamp for " . $path . " into s3 bucket"); // if (!$simulate) { // $this->_s3->update_object( // $this->getBucket(), $path, // array( // 'meta' => array('localts' => $task->ltime), // ) // ); // } break; default: $this->_out->logError("ignored command {$task->action}"); } $repeat = 0; } catch (Exception $e) { $repeat--; if ($repeat) { $this->_out->logError("need to repeat: {$msg}"); } else { if ($msg) { $this->_out->logError($msg); } throw new Exception($e->getMessage(), $e->getCode()); } } } while ($repeat); if (!$simulate) { $compare->remoteHasDone($task); } $this->_out->jobStep($job); } $this->_out->jobEnd($job, "remote storage updated"); }