/** * upload all data to recommeder * */ public function upload(DataApi\Client $client) { if (!is_null($this->filePath)) { $batchSize = 25000; $csvFile = new CsvFile($this->filePath); $csvFile->rewind(); if (!$csvFile->valid()) { break; } $header = $csvFile->current(); $csvFile->next(); $interactionsBatch = new DataApi\Batch\InteractionsBatch(); $batchRowsCount = 0; while ($csvFile->valid()) { $attributes = array_combine($header, $csvFile->current()); $csvFile->next(); // process row try { if (is_numeric($attributes['timestamp'])) { $date = new DateTime('@' . $attributes['timestamp']); } else { $date = new DateTime($attributes['timestamp']); } } catch (Exception $e) { throw new Exception('Invalid date format in "' . $this->getName() . '" table'); } $interactionsBatch->addInteraction($attributes['user_id'], $attributes['item_id'], $attributes['interaction_id'], $date); $batchRowsCount += 1; if ($batchRowsCount == $batchSize || !$csvFile->valid()) { $client->insertInteractions($interactionsBatch); $interactionsBatch = new DataApi\Batch\InteractionsBatch(); $batchRowsCount = 0; } } } return $this; }
/** * upload all data to recommeder * */ public function upload(DataApi\Client $client) { if (!is_null($this->filePath)) { $batchSize = 5000; $csvFile = new CsvFile($this->filePath); $csvFile->rewind(); if (!$csvFile->valid()) { break; } $header = $csvFile->current(); $csvFile->next(); $itemsBatch = new DataApi\Batch\EntitiesBatch(); $batchRowsCount = 0; while ($csvFile->valid()) { $attributes = array_combine($header, $csvFile->current()); $csvFile->next(); // process row - remove id column etc. $pk = $this->manifest->getPrimaryKey()[0]; $id = $attributes[$pk]; unset($attributes[$pk]); $itemsBatch->addEntity($id, $attributes); $batchRowsCount += 1; if ($batchRowsCount == $batchSize || !$csvFile->valid()) { $client->insertOrUpdateItems($itemsBatch); $itemsBatch = new DataApi\Batch\EntitiesBatch(); $batchRowsCount = 0; } } } return $this; }
protected function countLines(CsvFile $csvFile) { $cnt = 0; foreach ($csvFile as $row) { $cnt++; } $csvFile->rewind(); return $cnt; }
public function testIterator() { $csvFile = new CsvFile(__DIR__ . '/_data/test-input.csv'); $expected = array("id", "idAccount", "date", "totalFollowers", "followers", "totalStatuses", "statuses", "kloutScore", "timestamp"); // header line $csvFile->rewind(); $this->assertEquals($expected, $csvFile->current()); // first line $csvFile->next(); $this->assertTrue($csvFile->valid()); // second line $csvFile->next(); $this->assertTrue($csvFile->valid()); // file end $csvFile->next(); $this->assertFalse($csvFile->valid()); }