/**
* upload all data to recommeder
*
*/
public function upload(DataApi\Client $client)
{
if (!is_null($this->filePath)) {
$batchSize = 25000;
$csvFile = new CsvFile($this->filePath);
$csvFile->rewind();
if (!$csvFile->valid()) {
break;
}
$header = $csvFile->current();
$csvFile->next();
$interactionsBatch = new DataApi\Batch\InteractionsBatch();
$batchRowsCount = 0;
while ($csvFile->valid()) {
$attributes = array_combine($header, $csvFile->current());
$csvFile->next();
// process row
try {
if (is_numeric($attributes['timestamp'])) {
$date = new DateTime('@' . $attributes['timestamp']);
} else {
$date = new DateTime($attributes['timestamp']);
}
} catch (Exception $e) {
throw new Exception('Invalid date format in "' . $this->getName() . '" table');
}
$interactionsBatch->addInteraction($attributes['user_id'], $attributes['item_id'], $attributes['interaction_id'], $date);
$batchRowsCount += 1;
if ($batchRowsCount == $batchSize || !$csvFile->valid()) {
$client->insertInteractions($interactionsBatch);
$interactionsBatch = new DataApi\Batch\InteractionsBatch();
$batchRowsCount = 0;
}
}
}
return $this;
}
/**
* upload all data to recommeder
*
*/
public function upload(DataApi\Client $client)
{
if (!is_null($this->filePath)) {
$batchSize = 5000;
$csvFile = new CsvFile($this->filePath);
$csvFile->rewind();
if (!$csvFile->valid()) {
break;
}
$header = $csvFile->current();
$csvFile->next();
$itemsBatch = new DataApi\Batch\EntitiesBatch();
$batchRowsCount = 0;
while ($csvFile->valid()) {
$attributes = array_combine($header, $csvFile->current());
$csvFile->next();
// process row - remove id column etc.
$pk = $this->manifest->getPrimaryKey()[0];
$id = $attributes[$pk];
unset($attributes[$pk]);
$itemsBatch->addEntity($id, $attributes);
$batchRowsCount += 1;
if ($batchRowsCount == $batchSize || !$csvFile->valid()) {
$client->insertOrUpdateItems($itemsBatch);
$itemsBatch = new DataApi\Batch\EntitiesBatch();
$batchRowsCount = 0;
}
}
}
return $this;
}
protected function countLines(CsvFile $csvFile)
{
$cnt = 0;
foreach ($csvFile as $row) {
$cnt++;
}
$csvFile->rewind();
return $cnt;
}
public function testIterator()
{
$csvFile = new CsvFile(__DIR__ . '/_data/test-input.csv');
$expected = array("id", "idAccount", "date", "totalFollowers", "followers", "totalStatuses", "statuses", "kloutScore", "timestamp");
// header line
$csvFile->rewind();
$this->assertEquals($expected, $csvFile->current());
// first line
$csvFile->next();
$this->assertTrue($csvFile->valid());
// second line
$csvFile->next();
$this->assertTrue($csvFile->valid());
// file end
$csvFile->next();
$this->assertFalse($csvFile->valid());
}
