protected function _refreshLocal($myrole, $drivers)
{
$job = $this->_out->jobStart("traversing file system starting at {$this->_baseDir}");
/* @var $compare Compare_Sqlite */
$compare = $drivers[Core_Engine::ROLE_COMPARE];
$stat = $this->_fileStat;
$it = new RecursiveDirectoryIterator($this->_baseDir);
$it->setFlags(RecursiveDirectoryIterator::SKIP_DOTS);
//$it = new DirnameFilter($it, "/informa/i");
if (isset($this->_options['filter'])) {
$this->_out->logNotice(">>>applying filter " . $this->_options['filterType']);
$filterConf = $this->_options['filter'][$this->_options['filterType']];
/** @var $filter Filter_Interface */
$filter = new $filterConf['class']($filterConf);
$filter->setIterator($it);
$it = $filter->applyFilters();
}
$itemCount = 0;
$this->_trimFromBegin = strlen($this->_baseDir);
$compare->updateFromLocalStart();
foreach (new RecursiveIteratorIterator($it, RecursiveIteratorIterator::SELF_FIRST) as $file) {
/** @var $file SplFileObject */
//we don't need make entry for not empty folder
if ($file->isDir() && !Storage_Filesystem_FileStat::isEmptyDir((string) $file)) {
continue;
}
$file = (string) $file;
$obj = Core_FsObject::factoryFromStat($this->_makePath($file), $stat->getStat($file));
$compare->updateFromLocal($obj);
$itemCount++;
}
$compare->updateFromLocalEnd();
$this->_out->jobEnd($job, "updated info about {$itemCount} files");
}
public function updateRemote($myrole, $drivers)
{
if ($this->_options['update'] == 'simulate') {
$simulate = true;
$this->_out->logWarning("only SIMULATION mode");
} else {
if ($this->_options['update'] === false || (int) $this->_options['update'] === 0) {
$this->_out->logNotice("skipped, not requested and not needed");
return;
}
$simulate = false;
}
/** @var $compare Compare_Interface */
$compare = $drivers['compare'];
/** @var $local Storage_Interface */
$local = $drivers['local'];
if (!$compare->initChangesOn("remote")) {
// TODO not sure, but maybe we will need it
}
$job = $this->_out->jobStart("updating remote storage");
$this->_out->jobSetProgressStep($job, 1000);
foreach ($compare as $task) {
$repeat = 3;
do {
$msg = "";
try {
$path = $this->_getPathWithBasedir($task->path, self::ADD_BASE_DIR);
switch ($task->action) {
case Compare_Interface::CMD_MKDIR:
$msg = "mkdir " . $path . " into s3 bucket";
$this->_out->logDebug($msg);
if (!$simulate) {
// create folders
$this->_s3->create_object($this->getBucket(), $path, array('body' => '', 'storage' => $this->_defaultRedundancyStorage));
}
break;
case Compare_Interface::CMD_PUT:
$msg = "put " . $path . " into s3 bucket";
$this->_out->logDebug($msg);
$uploadPath = $local->getBaseDir() . $task->path;
//fix for windows encoding issue
$uploadPath = $local->convertEncodingPath($uploadPath);
if (!file_exists($uploadPath)) {
$this->_out->logError("file {$uploadPath} does not exists anymore locally");
continue;
}
if (!$simulate) {
//empty directory
if (ord(substr($path, -1)) === 47) {
//for empty folders we need little different options
$this->_out->logWarning("TODO putting empty folder {$path} ... is it possible ?");
$this->_s3->create_object($this->getBucket(), $path, array('body' => '', 'storage' => $this->_defaultRedundancyStorage));
} else {
$options = array('fileUpload' => $uploadPath, 'storage' => $this->_defaultRedundancyStorage);
// TODO it should be possible to speedup upload of small upload but using S3 batch
if ($this->_options['multipart']['big-files']) {
// multipart upload for big files
if ($this->_options['multipart']['part-size']) {
$options['partSize'] = $this->_options['multipart']['part-size'];
}
$this->_s3->create_mpu_object($this->getBucket(), $path, $options);
} else {
// normal upload
$this->_s3->create_object($this->getBucket(), $path, $options);
}
}
}
break;
case Compare_Interface::CMD_DELETE:
$msg = "deleting " . $path . " from s3 bucket";
$this->_out->logDebug($msg);
if (!$simulate) {
$this->_s3->delete_object($this->getBucket(), $path);
}
break;
case Compare_Interface::CMD_TS:
// storing this information as metadata is too slow to be used
// $this->_out->logDebug("remember local timestamp for " . $path . " into s3 bucket");
// if (!$simulate) {
// $this->_s3->update_object(
// $this->getBucket(), $path,
// array(
// 'meta' => array('localts' => $task->ltime),
// )
// );
// }
break;
default:
$this->_out->logError("ignored command {$task->action}");
}
$repeat = 0;
} catch (Exception $e) {
$repeat--;
if ($repeat) {
$this->_out->logError("need to repeat: {$msg}");
} else {
if ($msg) {
$this->_out->logError($msg);
}
throw new Exception($e->getMessage(), $e->getCode());
}
}
} while ($repeat);
if (!$simulate) {
$compare->remoteHasDone($task);
}
$this->_out->jobStep($job);
}
$this->_out->jobEnd($job, "remote storage updated");
}
public function compare($myrole, $drivers)
{
$this->_out->logNotice(">>>compare with Sqlite compare driver");
// creating indexes
$this->_dbIndexOn(true);
// delete invalid rows
$this->_exec("DELETE FROM {$this->_prefix} WHERE local=0 and remote=0");
// update missing md5 for remote files where needed
$this->_out->logNotice(">>>starting update md5 of remote files ...");
$resCount = $this->_db->querySingle($this->_sql(<<<SQL
SELECT count(*) FROM {$this->_prefix} WHERE isdir=0 and local and remote and rmd5 is null and (rts<>ltime or rts is null)
SQL
));
$stmt = $this->_db->query($this->_sql(<<<SQL
SELECT path FROM {$this->_prefix} WHERE isdir=0 and local and remote and rmd5 is null and (rts<>ltime or rts is null)
SQL
));
/** @var $driver Storage_Interface */
$driver = $drivers['remote'];
$counter = 0;
$job = $this->_out->jobStart("we need to calculate md5 for {$resCount} remote files");
$this->_out->jobSetProgressStep($job, 50);
while ($row = $stmt->fetchArray()) {
$fullPath = $driver->getBaseDir() . $row['path'];
$this->_out->logDebug($fullPath);
if ($md5 = $driver->getMd5($fullPath)) {
$this->_prepFromRemoteMd5->bindValue(":md5", $md5);
$this->_prepFromRemoteMd5->bindValue(":path", $row['path']);
$this->_prepFromRemoteMd5->execute();
$this->_out->jobStep($job);
$counter++;
}
}
$this->_out->jobEnd($job, "md5 calculated for {$counter} files");
// update missing md5 for local files where needed
$this->_out->logNotice(">>>starting update md5 of local files ...");
$resCount = $this->_db->querySingle($this->_sql(<<<SQL
SELECT count(*) FROM {$this->_prefix} WHERE isdir=0 and local and remote and (rts<>ltime or rts is null)
SQL
));
$stmt = $this->_db->query($this->_sql(<<<SQL
SELECT path FROM {$this->_prefix} WHERE isdir=0 and local and remote and (rts<>ltime or rts is null)
SQL
));
$driver = $drivers['local'];
$counter = 0;
$job = $this->_out->jobStart("we need to calculate md5 for {$resCount} local files");
$this->_out->jobSetProgressStep($job, 50);
while ($row = $stmt->fetchArray()) {
$fullPath = $driver->getBaseDir() . $row['path'];
$this->_out->logDebug($fullPath);
if ($md5 = $driver->getMd5($fullPath)) {
$this->_prepFromLocalMd5->bindValue(":md5", $md5);
$this->_prepFromLocalMd5->bindValue(":path", $row['path']);
$this->_prepFromLocalMd5->execute();
$this->_out->jobStep($job);
$counter++;
}
}
$this->_out->jobEnd($job, "md5 calculated for {$counter} files");
}
