private function validateColumns($tableName, $columnsToImport)
{
if (count($columnsToImport) == 0) {
throw new Exception('No columns found in CSV file.', Exception::NO_COLUMNS, null, 'csvImport.noColumns');
}
$tableColumns = $this->connection->getTableColumns($this->schemaName, $tableName);
$moreColumns = array_diff($columnsToImport, $tableColumns);
if (!empty($moreColumns)) {
throw new Exception('Columns doest not match. Non existing columns: ' . implode(', ', $moreColumns), Exception::COLUMNS_COUNT_NOT_MATCH);
}
}
/**
* @dataProvider incrementalImportData
* @param \Keboola\Csv\CsvFile $initialImportFile
* @param \Keboola\Csv\CsvFile $incrementFile
* @param $columns
* @param $expected
* @param $tableName
* @param $rowsShouldBeUpdated
* @param $importOptions
*/
public function testIncrementalImport(\Keboola\Csv\CsvFile $initialImportFile, \Keboola\Csv\CsvFile $incrementFile, $columns, $expected, $tableName, $rowsShouldBeUpdated, $importOptions = ['useTimestamp' => true])
{
$diffColumn = $importOptions['useTimestamp'] ? '_timestamp' : (isset($importOptions['diffColumn']) ? $importOptions['diffColumn'] : null);
if (is_null($diffColumn)) {
throw new Exception("If you don't want to use timestamp, please specify an alternate column to use");
}
// initial import
$import = $this->getImport();
$import->setIgnoreLines(1)->setIncremental(false)->import($tableName, $columns, [$initialImportFile], $importOptions);
$rowsByIdsAfterFullLoad = [];
foreach ($this->fetchAll($this->destSchemaName, $tableName, ['id', $diffColumn]) as $row) {
$rowsByIdsAfterFullLoad[$row[0]] = $row[1];
}
sleep(2);
$import->setIncremental(true)->import($tableName, $columns, [$incrementFile], $importOptions);
$tableColumns = $this->connection->getTableColumns($this->destSchemaName, $tableName);
if ($importOptions['useTimestamp']) {
$this->assertContains('_timestamp', $tableColumns);
} else {
$this->assertNotContains('_timestamp', $tableColumns);
}
$tableColumns = array_filter($tableColumns, function ($column) {
return $column !== '_timestamp';
});
$rowsByIdsAfterIncrement = [];
foreach ($this->fetchAll($this->destSchemaName, $tableName, ['id', $diffColumn]) as $row) {
$rowsByIdsAfterIncrement[$row[0]] = $row[1];
}
$changedRows = array_diff($rowsByIdsAfterIncrement, $rowsByIdsAfterFullLoad);
$updatedRows = array_keys($changedRows);
sort($updatedRows);
sort($rowsShouldBeUpdated);
$this->assertEquals($rowsShouldBeUpdated, $updatedRows);
$importedData = $this->fetchAll($this->destSchemaName, $tableName, $tableColumns);
$this->assertArrayEqualsSorted($expected, $importedData, 0);
}
