private function restoreData(RestoreJob $job, JobStats $jobStats, OutputInterface $output)
{
$memoryAtSection = memory_get_usage();
$timeStartSection = microtime(true);
$output->writeln('<info>*** Starting with data restoring ***</info>' . PHP_EOL);
$storedStats = array();
/** @var Index $index */
foreach ($job->getMappings()->getIndices() as $index) {
$toIndex = null;
/** @var Type $type */
foreach ($index->getTypes() as $type) {
/** @var Finder $finder */
$finder = $this->filesystem->loadDataFiles($job->getPath(), $index->getName(), $type->getName());
if (null === $finder) {
continue;
}
if (!isset($storedStats[$index->getName()])) {
$storedStats[$index->getName()] = array();
}
if (!isset($storedStats[$index->getName()][$type->getName()])) {
$storedStats[$index->getName()][$type->getName()] = array('filesRead' => 0, 'dataInType' => 0);
}
$mappingAction = $job->getStrategy()->getMapping($index->getName(), $type->getName());
if (null === $mappingAction) {
throw new \Exception('Mapping action missing for "' . $index->getName() . '/' . $type->getName() . '"');
}
$docCount = $finder->count();
$output->writeln('<comment>Store Data for:</comment>');
$output->writeln('<comment> [from]' . $index->getName() . '/' . $type->getName() . '</comment>');
$output->writeln('<comment> [to]' . $mappingAction->getTargetIndex() . '/' . $mappingAction->getTargetType() . '</comment>' . PHP_EOL);
$progress = new ProgressBar($output, $docCount);
$progress->setFormat('debug');
$progress->display();
$progress->start();
/** @var SplFileInfo $file */
foreach ($finder as $file) {
$data = json_decode($file->getContents(), true);
$id = $data['_id'];
$this->elastic->createDocument($mappingAction->getTargetIndex(), $mappingAction->getTargetType(), $id, $data['_source'], $job->getHost(), $job->getPort());
$progress->advance();
$storedStats[$index->getName()][$type->getName()]['filesRead']++;
}
$progress->finish();
$output->writeln(PHP_EOL);
$toIndex = $mappingAction->getTargetIndex();
}
$this->elastic->refreshIndex($toIndex, $job->getHost(), $job->getPort());
}
//add aggregated to storedStats for comparing later
$docStats = $this->elastic->getDocCountByIndexType($job->getHost(), $job->getPort());
foreach ($storedStats as $indexName => $types) {
foreach ($types as $typeName => $typeStats) {
$storedStats[$indexName][$typeName]['dataInType'] = $docStats->getDocCount($indexName, $typeName);
}
}
$jobStats->setRestoreData(microtime(true) - $timeStartSection, memory_get_usage(), memory_get_usage() - $memoryAtSection, array('storedStats' => $storedStats));
return $storedStats;
}
/**
* Stores data into the filesystem and returns stored stats
*
* @param BackupJob $job
* @param JobStats $jobStats
* @param OutputInterface $output
* @return array
* @author Daniel Wendlandt
*/
private function storeData(BackupJob $job, JobStats $jobStats, OutputInterface $output)
{
$memoryAtSection = memory_get_usage();
$timeStartSection = microtime(true);
$output->writeln('<info>*** Starting with data storing ***</info>' . PHP_EOL);
$docCount = $this->elastic->getDocCountByIndexType($job->getHost(), $job->getPort());
$storedStats = array();
/** @var Index $index */
foreach ($job->getMappings()->getIndices() as $index) {
if (0 === $docCount->getDocCount($index->getName())) {
continue;
}
/** @var Type $type */
foreach ($index->getTypes() as $type) {
$docsInType = $docCount->getDocCount($index->getName(), $type->getName());
if (0 === $docsInType) {
continue;
}
$scrollId = $this->elastic->createScrollSearch($index->getName(), $type->getName(), $job->getHost(), $job->getPort());
$storedStats[$index->getName()][$type->getName()]['aggregatedNumberOfDocs'] = $docsInType;
$storedStats[$index->getName()][$type->getName()]['storedNumberOfDocs'] = 0;
$output->writeln('<comment>Store Data for: ' . $index->getName() . '/' . $type->getName() . '</comment>');
$progress = new ProgressBar($output, $docsInType);
$progress->setFormat('debug');
$progress->display();
$progress->start();
while (true) {
$data = $this->elastic->getScrollSearchData($scrollId, $job->getHost(), $job->getPort());
if (0 === count($data['hits'])) {
break;
}
$scrollId = $data['scrollId'];
$storedDocs = $this->filesystem->storeData($job->getPath(), $index->getName(), $type->getName(), $data['hits']);
$storedStats[$index->getName()][$type->getName()]['storedNumberOfDocs'] += $storedDocs;
$progress->advance($storedDocs);
}
$progress->finish();
$output->writeln(PHP_EOL);
}
}
$jobStats->setStoreData(microtime(true) - $timeStartSection, memory_get_usage(), memory_get_usage() - $memoryAtSection, array('stats' => $storedStats));
return $storedStats;
}
public function testGetDocCountByIndexType()
{
$version = '1.6.0';
$serverInfoData = $this->getServerInfoData($version);
$docCount = 88;
$aggsData['aggregations']['count_docs_in_index']['buckets'] = [['key' => 'my-index', 'doc_count' => $docCount, 'count_docs_in_types' => ['buckets' => [['key' => 'my-type', 'doc_count' => 44]]]]];
$query = new DocsInIndexTypeQuery();
$this->serverInfoResponse->expects($this->exactly(3))->method('getData')->willReturn($serverInfoData);
$this->response->expects($this->once())->method('getData')->willReturn($aggsData);
$this->request->expects($this->once())->method('setBody')->with($this->equalTo($query->getBody(array('size' => 10000))))->willReturn(json_encode($aggsData));
$this->requestFactory->expects($this->once())->method('create')->with('SearchRequest', $version, null, null, $this->serializer)->willReturn($this->request);
$this->client->expects($this->exactly(2))->method('send')->withConsecutive($this->isInstanceOf('Elastification\\Client\\Request\\V1x\\NodeInfoRequest'), $this->isInstanceOf('Elastification\\Client\\Request\\V1x\\SearchRequest'))->willReturnOnConsecutiveCalls($this->serverInfoResponse, $this->response);
$result = $this->repository->getDocCountByIndexType(self::HOST, self::PORT);
$this->assertInstanceOf('Elastification\\BackupRestore\\Entity\\IndexTypeStats', $result);
$this->assertSame($docCount, $result->getDocCount('my-index'));
$this->assertCount(1, $result->getIndices());
$index = $result->getIndices()['my-index'];
$this->assertSame('my-index', $index->getName());
$this->assertCount(1, $index->getTypes());
$this->assertSame($docCount, $index->getDocsInIndex());
$type = $index->getTypes()['my-type'];
$this->assertSame('my-type', $type->getName());
$this->assertSame(44, $type->getDocsInType());
}
