public function runAction($parameters) { $path = $parameters["path"]; $skipped = 0; $processed = 0; foreach ($parameters['tables'] as $table) { $sourceType = !empty($table['tableId']) ? 'table' : 'file'; if ($sourceType == 'table') { $logPrefix = sprintf('Table %s - ', $table['tableId']); } else { $logPrefix = sprintf('File %s - ', $table['file']); } if (empty($table['export'])) { $this->logger->info($logPrefix . 'Skipped'); $skipped++; continue; } $this->logger->info($logPrefix . 'Export start'); // load options $options = new Options\LoadOptions(); $options->setIndex($table['index'])->setType($table['type']); if (!empty($config['elastic']['bulkSize'])) { $options->setBulkSize($config['elastic']['bulkSize']); } $idColumn = !empty($table['id']) ? $table['id'] : null; // source file if (!empty($table['tableId'])) { $file = new CsvFile(sprintf('%s/%s.csv', $path, $table['tableId'])); } else { $file = new CsvFile(sprintf('%s/%s', $path, $table['file'])); if (mb_strtolower($file->getExtension()) !== 'csv') { throw new Exception\UserException($logPrefix . 'Export failed. Only csv files are supported'); } } if (!$file->isFile()) { throw new Exception\UserException($logPrefix . 'Export failed. Missing csv file'); } $result = $this->writer->loadFile($file, $options, $idColumn); if (!$result) { throw new Exception\UserException($logPrefix . 'Export failed'); } else { $this->logger->info($logPrefix . 'Export finished', array()); } $processed++; } $this->logger->info(sprintf("Exported %d tables. %d was skipped", $processed, $skipped)); }
/** * @param CsvFile $file * @param LoadOptions $options * @param $primaryIndex * @return bool */ public function loadFile(CsvFile $file, LoadOptions $options, $primaryIndex = null) { $csvHeader = $file->getHeader(); $params = ['body' => []]; $iBulk = 1; foreach ($file as $i => $line) { // skip header if (!$i) { continue; } $lineData = array_combine($csvHeader, $line); if ($primaryIndex) { if (!array_key_exists($primaryIndex, $lineData)) { $this->logger->error(sprintf("CSV error: Missing id column %s on line %s", $primaryIndex, $i + 1)); return false; } $params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType(), '_id' => $lineData[$primaryIndex]]]; } else { $params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType()]]; } $params['body'][] = $lineData; if ($i % $options->getBulkSize() == 0) { $this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex())); $responses = $this->client->bulk($params); $this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took'])); $params = ['body' => []]; if ($responses['errors'] !== false) { if (!empty($responses['items'])) { foreach ($responses['items'] as $itemResult) { if (!empty($itemResult['index']['error'])) { if (is_array($itemResult['index']['error'])) { $this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error']))); } else { $this->logger->error(sprintf("ES error: %s", $itemResult['index']['error'])); } return false; } } } return false; } $iBulk++; unset($responses); } } if (!empty($params['body'])) { $this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex())); $responses = $this->client->bulk($params); $this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took'])); if ($responses['errors'] !== false) { if (!empty($responses['items'])) { foreach ($responses['items'] as $itemResult) { if (!empty($itemResult['index']['error'])) { if (is_array($itemResult['index']['error'])) { $this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error']))); } else { $this->logger->error(sprintf("ES error: %s", $itemResult['index']['error'])); } return false; } } } return false; } unset($responses); } return true; }
/** * Test bulk load */ public function testWriterTwice() { $writer = $this->writer; $options = new LoadOptions(); $options->setIndex($this->index)->setType('language')->setBulkSize(LoadOptions::DEFAULT_BULK_SIZE); $csv1 = new CsvFile(__DIR__ . '/../../data/csv/' . $options->getType() . '.csv'); $result = $writer->loadFile($csv1, $options, null); $this->assertTrue($result); $result = $writer->loadFile($csv1, $options, null); $this->assertTrue($result); // test if index exists $params = ['index' => $options->getIndex()]; $settings = $writer->getClient()->indices()->getSettings($params); $this->assertCount(1, $settings); $this->assertArrayHasKey($options->getIndex(), $settings); $this->assertArrayHasKey('settings', $settings[$options->getIndex()]); $this->assertArrayHasKey('index', $settings[$options->getIndex()]['settings']); $writer->getClient()->indices()->refresh(['index' => $options->getIndex()]); $params = ['index' => $options->getIndex(), 'type' => $options->getType()]; $count = $writer->getClient()->count($params); $this->assertArrayHasKey('count', $count); $this->assertEquals($this->countTable($csv1) * 2, $count['count']); }