private function validateColumns($tableName, $columnsToImport) { if (count($columnsToImport) == 0) { throw new Exception('No columns found in CSV file.', Exception::NO_COLUMNS, null, 'csvImport.noColumns'); } $tableColumns = $this->connection->getTableColumns($this->schemaName, $tableName); $moreColumns = array_diff($columnsToImport, $tableColumns); if (!empty($moreColumns)) { throw new Exception('Columns doest not match. Non existing columns: ' . implode(', ', $moreColumns), Exception::COLUMNS_COUNT_NOT_MATCH); } }
/** * @dataProvider incrementalImportData * @param \Keboola\Csv\CsvFile $initialImportFile * @param \Keboola\Csv\CsvFile $incrementFile * @param $columns * @param $expected * @param $tableName * @param $rowsShouldBeUpdated * @param $importOptions */ public function testIncrementalImport(\Keboola\Csv\CsvFile $initialImportFile, \Keboola\Csv\CsvFile $incrementFile, $columns, $expected, $tableName, $rowsShouldBeUpdated, $importOptions = ['useTimestamp' => true]) { $diffColumn = $importOptions['useTimestamp'] ? '_timestamp' : (isset($importOptions['diffColumn']) ? $importOptions['diffColumn'] : null); if (is_null($diffColumn)) { throw new Exception("If you don't want to use timestamp, please specify an alternate column to use"); } // initial import $import = $this->getImport(); $import->setIgnoreLines(1)->setIncremental(false)->import($tableName, $columns, [$initialImportFile], $importOptions); $rowsByIdsAfterFullLoad = []; foreach ($this->fetchAll($this->destSchemaName, $tableName, ['id', $diffColumn]) as $row) { $rowsByIdsAfterFullLoad[$row[0]] = $row[1]; } sleep(2); $import->setIncremental(true)->import($tableName, $columns, [$incrementFile], $importOptions); $tableColumns = $this->connection->getTableColumns($this->destSchemaName, $tableName); if ($importOptions['useTimestamp']) { $this->assertContains('_timestamp', $tableColumns); } else { $this->assertNotContains('_timestamp', $tableColumns); } $tableColumns = array_filter($tableColumns, function ($column) { return $column !== '_timestamp'; }); $rowsByIdsAfterIncrement = []; foreach ($this->fetchAll($this->destSchemaName, $tableName, ['id', $diffColumn]) as $row) { $rowsByIdsAfterIncrement[$row[0]] = $row[1]; } $changedRows = array_diff($rowsByIdsAfterIncrement, $rowsByIdsAfterFullLoad); $updatedRows = array_keys($changedRows); sort($updatedRows); sort($rowsShouldBeUpdated); $this->assertEquals($rowsShouldBeUpdated, $updatedRows); $importedData = $this->fetchAll($this->destSchemaName, $tableName, $tableColumns); $this->assertArrayEqualsSorted($expected, $importedData, 0); }