/**
* scan a single file and store it in the cache
*
* @param string $file
* @param int $reuseExisting
* @return array an array of metadata of the scanned file
*/
public function scanFile($file, $reuseExisting = 0)
{
if (!self::isPartialFile($file) and !Filesystem::isFileBlacklisted($file)) {
$this->emit('\\OC\\Files\\Cache\\Scanner', 'scanFile', array($file, $this->storageId));
\OC_Hook::emit('\\OC\\Files\\Cache\\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId));
$data = $this->getData($file);
if ($data) {
$parent = dirname($file);
if ($parent === '.' or $parent === '/') {
$parent = '';
}
$parentId = $this->cache->getId($parent);
// scan the parent if it's not in the cache (id -1) and the current file is not the root folder
if ($file and $parentId === -1) {
$parentData = $this->scanFile($parent);
$parentId = $parentData['fileid'];
}
if ($parent) {
$data['parent'] = $parentId;
}
$cacheData = $this->cache->get($file);
if ($cacheData and $reuseExisting) {
// prevent empty etag
if (empty($cacheData['etag'])) {
$etag = $data['etag'];
} else {
$etag = $cacheData['etag'];
}
// only reuse data if the file hasn't explicitly changed
if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) {
$data['mtime'] = $cacheData['mtime'];
if ($reuseExisting & self::REUSE_SIZE && $data['size'] === -1) {
$data['size'] = $cacheData['size'];
}
if ($reuseExisting & self::REUSE_ETAG) {
$data['etag'] = $etag;
}
}
// Only update metadata that has changed
$newData = array_diff_assoc($data, $cacheData);
if (isset($newData['etag'])) {
$cacheDataString = print_r($cacheData, true);
$dataString = print_r($data, true);
\OCP\Util::writeLog('OC\\Files\\Cache\\Scanner', "!!! No reuse of etag for '{$file}' !!! \ncache: {$cacheDataString} \ndata: {$dataString}", \OCP\Util::DEBUG);
}
} else {
$newData = $data;
}
if (!empty($newData)) {
$data['fileid'] = $this->addToCache($file, $newData);
$this->emit('\\OC\\Files\\Cache\\Scanner', 'postScanFile', array($file, $this->storageId));
\OC_Hook::emit('\\OC\\Files\\Cache\\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId));
}
} else {
$this->removeFromCache($file);
}
return $data;
}
return null;
}
private function updateStorageMTimeOnly($internalPath)
{
$fileId = $this->cache->getId($internalPath);
if ($fileId !== -1) {
$this->cache->update($fileId, ['mtime' => null, 'storage_mtime' => $this->storage->filemtime($internalPath)]);
}
}
/**
* scan all the files and folders in a folder
*
* @param string $path
* @param bool $recursive
* @param int $reuse
* @param int $folderId id for the folder to be scanned
* @param bool $lock set to false to disable getting an additional read lock during scanning
* @return int the size of the scanned folder or -1 if the size is unknown at this stage
*/
protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true)
{
if ($reuse === -1) {
$reuse = $recursive === self::SCAN_SHALLOW ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
}
$this->emit('\\OC\\Files\\Cache\\Scanner', 'scanFolder', array($path, $this->storageId));
$size = 0;
if (!is_null($folderId)) {
$folderId = $this->cache->getId($path);
}
$childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size);
foreach ($childQueue as $child => $childId) {
$childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock);
if ($childSize === -1) {
$size = -1;
} else {
if ($size !== -1) {
$size += $childSize;
}
}
}
if ($this->cacheActive) {
$this->cache->update($folderId, array('size' => $size));
}
$this->emit('\\OC\\Files\\Cache\\Scanner', 'postScanFolder', array($path, $this->storageId));
return $size;
}
public function testSimple() {
$file1 = 'foo';
$file2 = 'foo/bar';
$data1 = array('size' => 100, 'mtime' => 50, 'mimetype' => 'foo/folder');
$data2 = array('size' => 1000, 'mtime' => 20, 'mimetype' => 'foo/file');
$this->assertFalse($this->cache->inCache($file1));
$this->assertEquals($this->cache->get($file1), null);
$id1 = $this->cache->put($file1, $data1);
$this->assertTrue($this->cache->inCache($file1));
$cacheData1 = $this->cache->get($file1);
foreach ($data1 as $key => $value) {
$this->assertEquals($value, $cacheData1[$key]);
}
$this->assertEquals($cacheData1['mimepart'], 'foo');
$this->assertEquals($cacheData1['fileid'], $id1);
$this->assertEquals($id1, $this->cache->getId($file1));
$this->assertFalse($this->cache->inCache($file2));
$id2 = $this->cache->put($file2, $data2);
$this->assertTrue($this->cache->inCache($file2));
$cacheData2 = $this->cache->get($file2);
foreach ($data2 as $key => $value) {
$this->assertEquals($value, $cacheData2[$key]);
}
$this->assertEquals($cacheData1['fileid'], $cacheData2['parent']);
$this->assertEquals($cacheData2['fileid'], $id2);
$this->assertEquals($id2, $this->cache->getId($file2));
$this->assertEquals($id1, $this->cache->getParentId($file2));
$newSize = 1050;
$newId2 = $this->cache->put($file2, array('size' => $newSize));
$cacheData2 = $this->cache->get($file2);
$this->assertEquals($newId2, $id2);
$this->assertEquals($cacheData2['size'], $newSize);
$this->assertEquals($cacheData1, $this->cache->get($file1));
$this->cache->remove($file2);
$this->assertFalse($this->cache->inCache($file2));
$this->assertEquals($this->cache->get($file2), null);
$this->assertTrue($this->cache->inCache($file1));
$this->assertEquals($cacheData1, $this->cache->get($id1));
}
public function testRepairParentShallow()
{
$this->fillTestFolders();
$this->scanner->scan('');
$this->assertTrue($this->cache->inCache('folder/bar.txt'));
$oldFolderId = $this->cache->getId('folder');
// delete the folder without removing the childs
$sql = 'DELETE FROM `*PREFIX*filecache` WHERE `fileid` = ?';
\OC_DB::executeAudited($sql, array($oldFolderId));
$cachedData = $this->cache->get('folder/bar.txt');
$this->assertEquals($oldFolderId, $cachedData['parent']);
$this->assertFalse($this->cache->inCache('folder'));
$this->scanner->scan('folder', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->assertTrue($this->cache->inCache('folder'));
$newFolderId = $this->cache->getId('folder');
$this->assertNotEquals($oldFolderId, $newFolderId);
$cachedData = $this->cache->get('folder/bar.txt');
$this->assertEquals($newFolderId, $cachedData['parent']);
}
/**
* scan all the files and folders in a folder
*
* @param string $path
* @param bool $recursive
* @param int $reuse
* @param array $folderData existing cache data for the folder to be scanned
* @param bool $lock set to false to disable getting an additional read lock during scanning
* @return int the size of the scanned folder or -1 if the size is unknown at this stage
*/
protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderData = null, $lock = true)
{
if ($reuse === -1) {
$reuse = $recursive === self::SCAN_SHALLOW ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
}
$this->emit('\\OC\\Files\\Cache\\Scanner', 'scanFolder', array($path, $this->storageId));
$size = 0;
$childQueue = array();
if (is_array($folderData) and isset($folderData['fileid'])) {
$folderId = $folderData['fileid'];
} else {
$folderId = $this->cache->getId($path);
}
$existingChildren = $this->getExistingChildren($folderId);
$newChildren = $this->getNewChildren($path);
if ($this->useTransactions) {
\OC::$server->getDatabaseConnection()->beginTransaction();
}
$exceptionOccurred = false;
foreach ($newChildren as $file) {
$child = $path ? $path . '/' . $file : $file;
try {
$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : null;
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock);
if ($data) {
if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
$childQueue[$child] = $data;
} else {
if ($data['size'] === -1) {
$size = -1;
} else {
if ($size !== -1) {
$size += $data['size'];
}
}
}
}
} catch (\Doctrine\DBAL\DBALException $ex) {
// might happen if inserting duplicate while a scanning
// process is running in parallel
// log and ignore
\OCP\Util::writeLog('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OCP\Util::DEBUG);
$exceptionOccurred = true;
} catch (\OCP\Lock\LockedException $e) {
if ($this->useTransactions) {
\OC::$server->getDatabaseConnection()->rollback();
}
throw $e;
}
}
$removedChildren = \array_diff(array_keys($existingChildren), $newChildren);
foreach ($removedChildren as $childName) {
$child = $path ? $path . '/' . $childName : $childName;
$this->removeFromCache($child);
}
if ($this->useTransactions) {
\OC::$server->getDatabaseConnection()->commit();
}
if ($exceptionOccurred) {
// It might happen that the parallel scan process has already
// inserted mimetypes but those weren't available yet inside the transaction
// To make sure to have the updated mime types in such cases,
// we reload them here
\OC::$server->getMimeTypeLoader()->reset();
}
foreach ($childQueue as $child => $childData) {
$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse, $childData, $lock);
if ($childSize === -1) {
$size = -1;
} else {
if ($size !== -1) {
$size += $childSize;
}
}
}
if (!is_array($folderData) or !isset($folderData['size']) or $folderData['size'] !== $size) {
$this->updateCache($path, array('size' => $size), $folderId);
}
$this->emit('\\OC\\Files\\Cache\\Scanner', 'postScanFolder', array($path, $this->storageId));
return $size;
}
/**
* get the file id for a file
*
* @param string $file
* @return int
*/
public function getId($file)
{
return $this->cache->getId($file);
}