protected function _refreshLocal($myrole, $drivers)
{
$job = $this->_out->jobStart("traversing file system starting at {$this->_baseDir}");
/* @var $compare Compare_Sqlite */
$compare = $drivers[Core_Engine::ROLE_COMPARE];
$stat = $this->_fileStat;
$it = new RecursiveDirectoryIterator($this->_baseDir);
$it->setFlags(RecursiveDirectoryIterator::SKIP_DOTS);
//$it = new DirnameFilter($it, "/informa/i");
if (isset($this->_options['filter'])) {
$this->_out->logNotice(">>>applying filter " . $this->_options['filterType']);
$filterConf = $this->_options['filter'][$this->_options['filterType']];
/** @var $filter Filter_Interface */
$filter = new $filterConf['class']($filterConf);
$filter->setIterator($it);
$it = $filter->applyFilters();
}
$itemCount = 0;
$this->_trimFromBegin = strlen($this->_baseDir);
$compare->updateFromLocalStart();
foreach (new RecursiveIteratorIterator($it, RecursiveIteratorIterator::SELF_FIRST) as $file) {
/** @var $file SplFileObject */
//we don't need make entry for not empty folder
if ($file->isDir() && !Storage_Filesystem_FileStat::isEmptyDir((string) $file)) {
continue;
}
$file = (string) $file;
$obj = Core_FsObject::factoryFromStat($this->_makePath($file), $stat->getStat($file));
$compare->updateFromLocal($obj);
$itemCount++;
}
$compare->updateFromLocalEnd();
$this->_out->jobEnd($job, "updated info about {$itemCount} files");
}
/**
* Initialize iterator of tasks to be executed on storage
*
* @param int $storageType type of storage (Core_Engine::ROLE_REMOTE or Core_Engine::ROLE_LOCAL)
* @return bool
*/
public function initChangesOn($storageType)
{
switch ($storageType) {
case Core_Engine::ROLE_LOCAL:
// we don't care at this moment
break;
case Core_Engine::ROLE_REMOTE:
// TODO on this place there should be SQL using union which will be executed against SQL
// and its results will be returned on each Iterator::next()
// the $storageType defines if we want to receive changes for local or remote storage
$cmdDelete = self::CMD_DELETE;
$cmdMkdir = self::CMD_MKDIR;
$cmdTs = self::CMD_TS;
$cmdPut = self::CMD_PUT;
// count actions
$sql = <<<SQL
SELECT '{$cmdDelete}' as action, count(*) FROM {$this->_prefix} WHERE remote and local=0
UNION
SELECT '{$cmdMkdir}' as action , count(*)FROM {$this->_prefix} WHERE isdir=1 and local and remote=0
UNION
SELECT '{$cmdPut}' as action, count(*) FROM {$this->_prefix} WHERE isdir<>1 and local and (remote=0 or rsize<>lsize or rmd5!=lmd5)
UNION
SELECT '{$cmdTs}' as action, count(*) FROM {$this->_prefix} WHERE isdir<>1 and local and remote and rmd5=lmd5 and (rts is null or rts<>ltime)
SQL;
$result = $this->_db->query($sql);
// $this->_countInfo = array();
while ($res = $result->fetchArray(SQLITE3_NUM)) {
$this->_out->logNotice("will execute {$res['1']} time command {$res['0']}");
// $this->_countInfo[$res[0]] = $res[1];
}
// prepare result iteration
$sql = <<<SQL
SELECT path, ltime, '{$cmdDelete}' as action FROM {$this->_prefix} WHERE remote and local=0
UNION
SELECT path, ltime, '{$cmdMkdir}' as action FROM {$this->_prefix} WHERE isdir=1 and local and remote=0
UNION
SELECT path, ltime, '{$cmdPut}' as action FROM {$this->_prefix} WHERE isdir<>1 and local and (remote=0 or rsize<>lsize or rmd5!=lmd5)
UNION
SELECT path, ltime, '{$cmdTs}' as action FROM {$this->_prefix} WHERE isdir<>1 and local and remote and rmd5=lmd5 and (rts is null or rts<>ltime)
SQL;
$this->_jobSqlResult = $this->_db->query($sql);
break;
default:
throw new Exception("wrong {$storageType} parameter");
}
}
public function updateRemote($myrole, $drivers)
{
if ($this->_options['update'] == 'simulate') {
$simulate = true;
$this->_out->logWarning("only SIMULATION mode");
} else {
if ($this->_options['update'] === false || (int) $this->_options['update'] === 0) {
$this->_out->logNotice("skipped, not requested and not needed");
return;
}
$simulate = false;
}
/** @var $compare Compare_Interface */
$compare = $drivers['compare'];
/** @var $local Storage_Interface */
$local = $drivers['local'];
if (!$compare->initChangesOn("remote")) {
// TODO not sure, but maybe we will need it
}
$job = $this->_out->jobStart("updating remote storage");
$this->_out->jobSetProgressStep($job, 1000);
foreach ($compare as $task) {
$repeat = 3;
do {
$msg = "";
try {
$path = $this->_getPathWithBasedir($task->path, self::ADD_BASE_DIR);
switch ($task->action) {
case Compare_Interface::CMD_MKDIR:
$msg = "mkdir " . $path . " into s3 bucket";
$this->_out->logDebug($msg);
if (!$simulate) {
// create folders
$this->_s3->create_object($this->getBucket(), $path, array('body' => '', 'storage' => $this->_defaultRedundancyStorage));
}
break;
case Compare_Interface::CMD_PUT:
$msg = "put " . $path . " into s3 bucket";
$this->_out->logDebug($msg);
$uploadPath = $local->getBaseDir() . $task->path;
//fix for windows encoding issue
$uploadPath = $local->convertEncodingPath($uploadPath);
if (!file_exists($uploadPath)) {
$this->_out->logError("file {$uploadPath} does not exists anymore locally");
continue;
}
if (!$simulate) {
//empty directory
if (ord(substr($path, -1)) === 47) {
//for empty folders we need little different options
$this->_out->logWarning("TODO putting empty folder {$path} ... is it possible ?");
$this->_s3->create_object($this->getBucket(), $path, array('body' => '', 'storage' => $this->_defaultRedundancyStorage));
} else {
$options = array('fileUpload' => $uploadPath, 'storage' => $this->_defaultRedundancyStorage);
// TODO it should be possible to speedup upload of small upload but using S3 batch
if ($this->_options['multipart']['big-files']) {
// multipart upload for big files
if ($this->_options['multipart']['part-size']) {
$options['partSize'] = $this->_options['multipart']['part-size'];
}
$this->_s3->create_mpu_object($this->getBucket(), $path, $options);
} else {
// normal upload
$this->_s3->create_object($this->getBucket(), $path, $options);
}
}
}
break;
case Compare_Interface::CMD_DELETE:
$msg = "deleting " . $path . " from s3 bucket";
$this->_out->logDebug($msg);
if (!$simulate) {
$this->_s3->delete_object($this->getBucket(), $path);
}
break;
case Compare_Interface::CMD_TS:
// storing this information as metadata is too slow to be used
// $this->_out->logDebug("remember local timestamp for " . $path . " into s3 bucket");
// if (!$simulate) {
// $this->_s3->update_object(
// $this->getBucket(), $path,
// array(
// 'meta' => array('localts' => $task->ltime),
// )
// );
// }
break;
default:
$this->_out->logError("ignored command {$task->action}");
}
$repeat = 0;
} catch (Exception $e) {
$repeat--;
if ($repeat) {
$this->_out->logError("need to repeat: {$msg}");
} else {
if ($msg) {
$this->_out->logError($msg);
}
throw new Exception($e->getMessage(), $e->getCode());
}
}
} while ($repeat);
if (!$simulate) {
$compare->remoteHasDone($task);
}
$this->_out->jobStep($job);
}
$this->_out->jobEnd($job, "remote storage updated");
}
