public function testStoreDataFolderExisting()
{
$path = '/tmp/test-path';
$index = 'my-index';
$type = 'my-type';
$docs = [['_id' => 'F3jhso8', '_source' => []], ['_id' => 'KI3jhso8mb4n3asd23ku', '_source' => []]];
$folderPath = $path . DIRECTORY_SEPARATOR . FilesystemRepositoryInterface::DIR_DATA . DIRECTORY_SEPARATOR . $index . DIRECTORY_SEPARATOR . $type;
$filepathDoc1 = $folderPath . DIRECTORY_SEPARATOR . substr($docs[0]['_id'], 0, 2) . DIRECTORY_SEPARATOR . $docs[0]['_id'] . FilesystemRepositoryInterface::FILE_EXTENSION;
$filepathDoc2 = $folderPath . DIRECTORY_SEPARATOR . substr($docs[1]['_id'], 0, 2) . DIRECTORY_SEPARATOR . $docs[1]['_id'] . FilesystemRepositoryInterface::FILE_EXTENSION;
$this->filesystem->expects($this->once())->method('exists')->with($folderPath)->willReturn(true);
$this->filesystem->expects($this->never())->method('mkdir');
$this->filesystem->expects($this->exactly(2))->method('dumpFile')->withConsecutive(array($filepathDoc1, json_encode($docs[0])), array($filepathDoc2, json_encode($docs[1])));
$docsCreated = $this->filesystemRepository->storeData($path, $index, $type, $docs);
$this->assertSame(2, $docsCreated);
}
/**
* Stores data into the filesystem and returns stored stats
*
* @param BackupJob $job
* @param JobStats $jobStats
* @param OutputInterface $output
* @return array
* @author Daniel Wendlandt
*/
private function storeData(BackupJob $job, JobStats $jobStats, OutputInterface $output)
{
$memoryAtSection = memory_get_usage();
$timeStartSection = microtime(true);
$output->writeln('<info>*** Starting with data storing ***</info>' . PHP_EOL);
$docCount = $this->elastic->getDocCountByIndexType($job->getHost(), $job->getPort());
$storedStats = array();
/** @var Index $index */
foreach ($job->getMappings()->getIndices() as $index) {
if (0 === $docCount->getDocCount($index->getName())) {
continue;
}
/** @var Type $type */
foreach ($index->getTypes() as $type) {
$docsInType = $docCount->getDocCount($index->getName(), $type->getName());
if (0 === $docsInType) {
continue;
}
$scrollId = $this->elastic->createScrollSearch($index->getName(), $type->getName(), $job->getHost(), $job->getPort());
$storedStats[$index->getName()][$type->getName()]['aggregatedNumberOfDocs'] = $docsInType;
$storedStats[$index->getName()][$type->getName()]['storedNumberOfDocs'] = 0;
$output->writeln('<comment>Store Data for: ' . $index->getName() . '/' . $type->getName() . '</comment>');
$progress = new ProgressBar($output, $docsInType);
$progress->setFormat('debug');
$progress->display();
$progress->start();
while (true) {
$data = $this->elastic->getScrollSearchData($scrollId, $job->getHost(), $job->getPort());
if (0 === count($data['hits'])) {
break;
}
$scrollId = $data['scrollId'];
$storedDocs = $this->filesystem->storeData($job->getPath(), $index->getName(), $type->getName(), $data['hits']);
$storedStats[$index->getName()][$type->getName()]['storedNumberOfDocs'] += $storedDocs;
$progress->advance($storedDocs);
}
$progress->finish();
$output->writeln(PHP_EOL);
}
}
$jobStats->setStoreData(microtime(true) - $timeStartSection, memory_get_usage(), memory_get_usage() - $memoryAtSection, array('stats' => $storedStats));
return $storedStats;
}