/**
* @param CsvFile $file
* @param LoadOptions $options
* @param $primaryIndex
* @return bool
*/
public function loadFile(CsvFile $file, LoadOptions $options, $primaryIndex = null)
{
$csvHeader = $file->getHeader();
$params = ['body' => []];
$iBulk = 1;
foreach ($file as $i => $line) {
// skip header
if (!$i) {
continue;
}
$lineData = array_combine($csvHeader, $line);
if ($primaryIndex) {
if (!array_key_exists($primaryIndex, $lineData)) {
$this->logger->error(sprintf("CSV error: Missing id column %s on line %s", $primaryIndex, $i + 1));
return false;
}
$params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType(), '_id' => $lineData[$primaryIndex]]];
} else {
$params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType()]];
}
$params['body'][] = $lineData;
if ($i % $options->getBulkSize() == 0) {
$this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
$responses = $this->client->bulk($params);
$this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
$params = ['body' => []];
if ($responses['errors'] !== false) {
if (!empty($responses['items'])) {
foreach ($responses['items'] as $itemResult) {
if (!empty($itemResult['index']['error'])) {
if (is_array($itemResult['index']['error'])) {
$this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
} else {
$this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
}
return false;
}
}
}
return false;
}
$iBulk++;
unset($responses);
}
}
if (!empty($params['body'])) {
$this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
$responses = $this->client->bulk($params);
$this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
if ($responses['errors'] !== false) {
if (!empty($responses['items'])) {
foreach ($responses['items'] as $itemResult) {
if (!empty($itemResult['index']['error'])) {
if (is_array($itemResult['index']['error'])) {
$this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
} else {
$this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
}
return false;
}
}
}
return false;
}
unset($responses);
}
return true;
}
/**
* Test bulk load
*/
public function testWriterTwice()
{
$writer = $this->writer;
$options = new LoadOptions();
$options->setIndex($this->index)->setType('language')->setBulkSize(LoadOptions::DEFAULT_BULK_SIZE);
$csv1 = new CsvFile(__DIR__ . '/../../data/csv/' . $options->getType() . '.csv');
$result = $writer->loadFile($csv1, $options, null);
$this->assertTrue($result);
$result = $writer->loadFile($csv1, $options, null);
$this->assertTrue($result);
// test if index exists
$params = ['index' => $options->getIndex()];
$settings = $writer->getClient()->indices()->getSettings($params);
$this->assertCount(1, $settings);
$this->assertArrayHasKey($options->getIndex(), $settings);
$this->assertArrayHasKey('settings', $settings[$options->getIndex()]);
$this->assertArrayHasKey('index', $settings[$options->getIndex()]['settings']);
$writer->getClient()->indices()->refresh(['index' => $options->getIndex()]);
$params = ['index' => $options->getIndex(), 'type' => $options->getType()];
$count = $writer->getClient()->count($params);
$this->assertArrayHasKey('count', $count);
$this->assertEquals($this->countTable($csv1) * 2, $count['count']);
}
