public function runAction($parameters)
{
$path = $parameters["path"];
$skipped = 0;
$processed = 0;
foreach ($parameters['tables'] as $table) {
$sourceType = !empty($table['tableId']) ? 'table' : 'file';
if ($sourceType == 'table') {
$logPrefix = sprintf('Table %s - ', $table['tableId']);
} else {
$logPrefix = sprintf('File %s - ', $table['file']);
}
if (empty($table['export'])) {
$this->logger->info($logPrefix . 'Skipped');
$skipped++;
continue;
}
$this->logger->info($logPrefix . 'Export start');
// load options
$options = new Options\LoadOptions();
$options->setIndex($table['index'])->setType($table['type']);
if (!empty($config['elastic']['bulkSize'])) {
$options->setBulkSize($config['elastic']['bulkSize']);
}
$idColumn = !empty($table['id']) ? $table['id'] : null;
// source file
if (!empty($table['tableId'])) {
$file = new CsvFile(sprintf('%s/%s.csv', $path, $table['tableId']));
} else {
$file = new CsvFile(sprintf('%s/%s', $path, $table['file']));
if (mb_strtolower($file->getExtension()) !== 'csv') {
throw new Exception\UserException($logPrefix . 'Export failed. Only csv files are supported');
}
}
if (!$file->isFile()) {
throw new Exception\UserException($logPrefix . 'Export failed. Missing csv file');
}
$result = $this->writer->loadFile($file, $options, $idColumn);
if (!$result) {
throw new Exception\UserException($logPrefix . 'Export failed');
} else {
$this->logger->info($logPrefix . 'Export finished', array());
}
$processed++;
}
$this->logger->info(sprintf("Exported %d tables. %d was skipped", $processed, $skipped));
}
/**
* @param CsvFile $file
* @param LoadOptions $options
* @param $primaryIndex
* @return bool
*/
public function loadFile(CsvFile $file, LoadOptions $options, $primaryIndex = null)
{
$csvHeader = $file->getHeader();
$params = ['body' => []];
$iBulk = 1;
foreach ($file as $i => $line) {
// skip header
if (!$i) {
continue;
}
$lineData = array_combine($csvHeader, $line);
if ($primaryIndex) {
if (!array_key_exists($primaryIndex, $lineData)) {
$this->logger->error(sprintf("CSV error: Missing id column %s on line %s", $primaryIndex, $i + 1));
return false;
}
$params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType(), '_id' => $lineData[$primaryIndex]]];
} else {
$params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType()]];
}
$params['body'][] = $lineData;
if ($i % $options->getBulkSize() == 0) {
$this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
$responses = $this->client->bulk($params);
$this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
$params = ['body' => []];
if ($responses['errors'] !== false) {
if (!empty($responses['items'])) {
foreach ($responses['items'] as $itemResult) {
if (!empty($itemResult['index']['error'])) {
if (is_array($itemResult['index']['error'])) {
$this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
} else {
$this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
}
return false;
}
}
}
return false;
}
$iBulk++;
unset($responses);
}
}
if (!empty($params['body'])) {
$this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
$responses = $this->client->bulk($params);
$this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
if ($responses['errors'] !== false) {
if (!empty($responses['items'])) {
foreach ($responses['items'] as $itemResult) {
if (!empty($itemResult['index']['error'])) {
if (is_array($itemResult['index']['error'])) {
$this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
} else {
$this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
}
return false;
}
}
}
return false;
}
unset($responses);
}
return true;
}
/**
* Test bulk load
*/
public function testWriterTwice()
{
$writer = $this->writer;
$options = new LoadOptions();
$options->setIndex($this->index)->setType('language')->setBulkSize(LoadOptions::DEFAULT_BULK_SIZE);
$csv1 = new CsvFile(__DIR__ . '/../../data/csv/' . $options->getType() . '.csv');
$result = $writer->loadFile($csv1, $options, null);
$this->assertTrue($result);
$result = $writer->loadFile($csv1, $options, null);
$this->assertTrue($result);
// test if index exists
$params = ['index' => $options->getIndex()];
$settings = $writer->getClient()->indices()->getSettings($params);
$this->assertCount(1, $settings);
$this->assertArrayHasKey($options->getIndex(), $settings);
$this->assertArrayHasKey('settings', $settings[$options->getIndex()]);
$this->assertArrayHasKey('index', $settings[$options->getIndex()]['settings']);
$writer->getClient()->indices()->refresh(['index' => $options->getIndex()]);
$params = ['index' => $options->getIndex(), 'type' => $options->getType()];
$count = $writer->getClient()->count($params);
$this->assertArrayHasKey('count', $count);
$this->assertEquals($this->countTable($csv1) * 2, $count['count']);
}