/**
*
*/
public function import($data, Event $event)
{
$csv = Reader::createFromString(trim($data));
$csv->setDelimiter(';');
// get startgroups at first
$groups = new Map();
foreach ($csv->fetchColumn(0) as $name) {
if (!$groups->has($name)) {
$startgroup = $this->getStartgroup($name);
$startgroup->setEvent($event);
$groups->set($name, $startgroup);
}
}
$id = 0;
foreach ($csv as $row) {
$id++;
$routine = $this->getRoutine($id);
$group = $groups->get($row[0]);
$judges = (count($row) - 1) / 3;
for ($j = 1; $j <= $judges; $j++) {
$score = new PerformanceScore();
$score->setRoutine($routine);
$score->setJudge($group->getPerformanceJudge($j));
$score->setExecution($row[($j - 1) * 3 + 1]);
$score->setChoreography($row[($j - 1) * 3 + 2]);
$score->setMusicAndTiming($row[($j - 1) * 3 + 3]);
$score->setTotal($row[($j - 1) * 3 + 1] + $row[($j - 1) * 3 + 2] + $row[($j - 1) * 3 + 3]);
$routine->addPerformanceScore($score);
}
$group->addRoutine($routine);
$group->save();
}
$event->save();
}
protected function readCSV()
{
$csvObj = Reader::createFromString($this->readFile(static::IMPORT_FILE));
$csvObj->setDelimiter(',');
$results = $csvObj->fetchAssoc();
return $results;
}
/**
* Run the actual import
*
* @return Collection
*/
public function createImportEntries() : Collection
{
$config = $this->job->configuration;
$content = $this->job->uploadFileContents();
// create CSV reader.
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$start = $config['has-headers'] ? 1 : 0;
$results = $reader->fetch();
Log::notice('Building importable objects from CSV file.');
foreach ($results as $index => $row) {
if ($index >= $start) {
$line = $index + 1;
Log::debug('----- import entry build start --');
Log::debug(sprintf('Now going to import row %d.', $index));
$importEntry = $this->importSingleRow($index, $row);
$this->collection->put($line, $importEntry);
/**
* 1. Build import entry.
* 2. Validate import entry.
* 3. Store journal.
* 4. Run rules.
*/
$this->job->addTotalSteps(4);
$this->job->addStepsDone(1);
}
}
Log::debug(sprintf('Import collection contains %d entries', $this->collection->count()));
Log::notice(sprintf('Built %d importable object(s) from your CSV file.', $this->collection->count()));
return $this->collection;
}
public function fetch(Connector $connector, EncapsulatedOptions $options = null)
{
$csv = $connector->fetch('http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv');
$reader = Reader::createFromString($csv);
foreach ($reader->fetchAssoc() as $row) {
(yield $row);
}
}
public function __construct($data)
{
if (is_string($data)) {
$this->csv = Reader::createFromString($data);
} else {
throw new InvalidArgumentException('CsvParser only accepts (string) [csv] for $data.');
}
}
/**
* @param string $data
* @return Reader
*/
protected function getReader($data)
{
$csv = Reader::createFromString($data);
$delimiters = $csv->fetchDelimitersOccurrence([' ', '|', ',', ';'], 10);
if (sizeof($delimiters) > 0) {
$csv->setDelimiter(array_keys($delimiters)[0]);
}
return $csv;
}
public function spreadsheetAsCsv()
{
$url = "https://docs.google.com/spreadsheets/d/{$this->fileId}/export?format=csv&gid={$this->gId}";
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
$reader = Reader::createFromString(curl_exec($ch));
curl_close($ch);
return $reader->fetchAssoc(0);
}
public function parse($content)
{
$reader = Reader::createFromString($content);
$reader->setDelimiter($this->delimiter);
$reader->setNewline($this->newline);
$reader->setEnclosure($this->enclosure);
$reader->setEscape($this->escapeChar);
$reader->setFlags($this->flags);
if ($this->hasHeader) {
return array_slice($reader->fetchAssoc($reader->fetchOne()), 1);
}
return $reader->fetchAll();
}
private function send()
{
try {
$response = $this->client->get(null, ['body' => $this->request]);
} catch (ClientException $e) {
throw new FirstDataException($e->getResponse()->getBody());
}
$inputCsv = Reader::createFromString($response->getBody());
$inputCsv->setDelimiter(',');
$inputCsv->setEncodingFrom('ISO-8859-15');
$inputCsv->setFlags(SplFileObject::DROP_NEW_LINE | SplFileObject::READ_AHEAD | SplFileObject::SKIP_EMPTY);
return $inputCsv->fetchAssoc();
}
/**
* Extract user rows from CSV string
*
* @param $csv
* @return \Iterator
*/
private function getRows($csv)
{
$reader = Reader::createFromString($csv);
$reader->setDelimiter($this->delimiter);
// Find column positions
$columns = $this->findColumns($reader->fetchOne());
return $reader->setOffset(1)->fetch(function ($row) use($columns) {
$user = [];
foreach ($columns as $key => $name) {
$user[$name] = $row[$key];
}
return $user;
});
}
public function postUpload()
{
$files = $this->flyLocal()->listContents();
foreach ($files as $file) {
if (preg_match('/.csv$/', $file['basename'])) {
$contents = $this->flyLocal()->readAndDelete($file['basename']);
$csv = Reader::createFromString($contents);
$rows = $csv->setOffset(1)->fetchAll();
foreach ($rows as $row) {
$item = new Item();
$item->date = strtotime($row[0]);
$item->category = $row[1];
$item->title = $row[2];
$item->location = $row[3];
$item->condition = $row[4];
$item->amount = $row[5];
$item->tax_name = $row[6];
$item->tax_amount = $row[7];
$item->save();
}
}
}
return redirect('admin');
}
public function processWorkerunitData($csvresult, $preview = false)
{
set_time_limit(6000);
$csvLines = explode("\n", $csvresult['content']);
$csvLines = array_filter($csvLines, 'strlen');
// $csvLines = array_slice($csvLines, 0, 2);
$csv = Reader::createFromString(implode("\n", $csvLines));
$delimiter = $csv->detectDelimiter();
$csv->setDelimiter($delimiter);
$headers = $csv->fetchOne();
// dd($headers);
foreach ($headers as $columnIndex => $columnValue) {
if ($columnValue == "") {
$headers[$columnIndex] = "empty_" . $columnIndex;
}
}
$rowsMappedWithUnits = array();
$batch = array();
// dd($headers);
// dd($csv->fetchAll());
$workerunitCSVArray = $csv->fetchAssoc($headers);
unset($workerunitCSVArray[0]);
if (strpos($csvresult->title, 'relexcorrected') !== false || strpos($csvresult->title, 'reldir') !== false) {
$result = $this->processRel($csvresult, $workerunitCSVArray, $preview);
return $result;
}
// dd($workerunitCSVArray);
$inputCSVFiles = \MongoDB\Entity::where('documentType', 'csvresult')->where('title', 'like', '%input%')->get();
$processedInputCSVFiles = array();
foreach ($inputCSVFiles as $inputCSVFile) {
array_push($processedInputCSVFiles, $this->processInputData($inputCSVFile->toArray(), true));
}
$aggregatedProcessedInputCSVFiles = array();
foreach ($processedInputCSVFiles as $processedInputCSVFile) {
foreach ($processedInputCSVFile as $processedInputCSVFileMapped) {
array_push($aggregatedProcessedInputCSVFiles, $processedInputCSVFileMapped);
}
}
$mappedWorkerunitsWithUnits = array();
$batchOfUnits = array();
foreach ($workerunitCSVArray as $workerunitRow) {
foreach ($aggregatedProcessedInputCSVFiles as $mappedInputRow) {
if (strpos($csvresult->title, 'workerunit_output_cf_factorspan_merged_relex') !== false) {
if (stripos($workerunitRow['Sent_id'], '-')) {
$workerunitRowIndex = explode('-', $workerunitRow['Sent_id'])[0];
}
} else {
$workerunitRowIndex = $workerunitRow['index'];
}
if ($workerunitRowIndex == $mappedInputRow['index']) {
$mappedWorkerunitWithUnit = $workerunitRow;
$mappedWorkerunitWithUnit['unit'] = $mappedInputRow['unit'];
array_push($batchOfUnits, $mappedInputRow['unit']['_id']);
array_push($mappedWorkerunitsWithUnits, $mappedWorkerunitWithUnit);
}
}
}
$batchOfUnits = array_values(array_unique($batchOfUnits));
natsort($batchOfUnits);
$batchOfUnits = array_values($batchOfUnits);
if ($preview == true) {
return $mappedWorkerunitsWithUnits;
}
if (stripos($csvresult->title, "factorspan") && stripos($csvresult->title, "380140")) {
$status = $this->factorspan_308140($csvresult, $batchOfUnits, $mappedWorkerunitsWithUnits);
} elseif (stripos($csvresult->title, "factorspan") && stripos($csvresult->title, "380640")) {
$status = $this->factorspan_380640($csvresult, $batchOfUnits, $mappedWorkerunitsWithUnits);
} elseif (stripos($csvresult->title, "factorspan") && stripos($csvresult->title, "382004")) {
$status = $this->factorspan_382004($csvresult, $batchOfUnits, $mappedWorkerunitsWithUnits);
} elseif (strpos($csvresult->title, 'annotation_output_cf_factorspan_merged_relex') !== false) {
$status = $this->workerunit_output_cf_factorspan_merged_relex($mappedWorkerunitsWithUnits);
} elseif (stripos($csvresult->title, "relexu") && stripos($csvresult->title, "f387177")) {
$status = $this->relexu_f387177($csvresult, $batchOfUnits, $mappedWorkerunitsWithUnits);
} elseif (stripos($csvresult->title, "relexu") && stripos($csvresult->title, "f387719")) {
$status = $this->relexu_f387719($csvresult, $batchOfUnits, $mappedWorkerunitsWithUnits);
} elseif (stripos($csvresult->title, "relexu") && stripos($csvresult->title, "f388572")) {
$status = $this->relexu_f388572($csvresult, $batchOfUnits, $mappedWorkerunitsWithUnits);
}
return $status;
}
public function processRows()
{
$csv = Reader::createFromString($this->data);
$csv->setOffset(1);
// Add filter to skip empty rows
$csv->addFilter(function ($row) {
return strlen(trim(implode('', $row))) > 0;
});
$columnMap = $this->getModelInstance()->getColumnMap();
$columns = array_keys($columnMap);
$csv->each(function ($row, $index) use($columns, $columnMap) {
$this->refresh();
if ($this->abort) {
return false;
}
try {
$row = array_combine($columns, $row);
if (!$this->processRow($row, $columnMap, $index)) {
$this->error_count++;
}
} catch (\yii\base\Exception $e) {
Yii::error($e->getMessage());
$this->status_id = Import::STATUS_ERROR;
return false;
}
$this->processed_rows++;
$this->progress = $this->getPercent();
$this->update(false, ['processed_rows', 'progress', 'status_id', 'error_count']);
return true;
});
}
/**
* Load content of a document (CSV) as a data matrix.
*
* @param $documentContent Raw content of CSV file.
* @param $delimiter CSV field delimiter.
* @param $separator CSV column separator.
* @param $ignoreHeader Boolean flag for ignoring headers.
* @param $nLines Maximum number of lines to read from file (-1 to read all)
* @return Data matrix representing the file contents.
*/
private function getDocumentData($documentContent, $delimiter, $separator, $ignoreHeader, $nLines)
{
$reader = Reader::createFromString($documentContent);
$reader->setDelimiter($separator);
$reader->setEnclosure($delimiter);
if ($ignoreHeader) {
$reader->setLimit($nLines);
} else {
$reader->setOffset(1);
$reader->setLimit($nLines + 1);
}
$dataTable = $reader->fetchAll();
return $dataTable;
}
/**
* Get table columns
* @returns array
*/
public function getColumns($table)
{
$reader = \League\Csv\Reader::createFromString(implode("\n", $this->_getCSVArray($table, true)));
return $reader->fetchOne(0);
}
/**
*
* @return Reader
*/
public function getReader()
{
if (strlen($this->csvFileContent) === 0) {
$this->loadCsvFile();
}
if (is_null($this->reader)) {
$this->reader = Reader::createFromString($this->getCsvFileContent());
$this->reader->setDelimiter($this->delimiter);
}
return $this->reader;
}
public function doPreview(ProductImportOptions $pio)
{
$data = [];
$path = $pio->getUuid() . '/' . $pio->getFileName();
$fileContents = \Storage::disk('imports')->get($path);
$csv = Reader::createFromString($fileContents);
$csv->setDelimiter(',');
if ($pio->isIncludeHeaders()) {
$headers = $csv->fetchOne();
$data['headers'] = $headers;
$csv->setOffset(1);
}
$previewRows = $csv->setLimit(25)->fetchAll();
$data['rows'] = $previewRows;
return $data;
}
/**
* @param $rawCsv
*
* @dataProvider getIso8859Csv
*/
public function testJsonInterface($rawCsv)
{
$this->csv->setFlags(SplFileObject::READ_AHEAD | SplFileObject::SKIP_EMPTY);
$this->assertSame(json_encode($this->expected), json_encode($this->csv));
$csv = Reader::createFromString($rawCsv);
$csv->setEncodingFrom('iso-8859-15');
json_encode($csv);
$this->assertEquals(JSON_ERROR_NONE, json_last_error());
}
$writer = Writer::createFromString($rawCsv);
//because we raw string delimiter is ";"
//the string delimiter MUST also be ";"
$writer->setDelimiter(';');
//we are creating a CSV from a raw string
$rawCsv2Merge = <<<EOF
Ben,7,M,2007
Benjamin,78,M,2007
Benoît,17,M,2007
Berenice,19,F,2007
Bertille,9,F,2007
Bianca,18,F,2007
Bilal,26,M,2007
Bilel,7,M,2007
EOF;
$csv2merge = Reader::createFromString($rawCsv2Merge);
//because we raw string delimiter is ";"
//the string delimiter MUST also be ","
$csv2merge->setDelimiter(',');
/*
When merging multiples CSV documents don't forget to set the main CSV object
as a `League\Csv\Writer` object with the $open_mode = 'a+' to preserve its content.
This setting is of course not required when your main CSV object is created from String
*/
?>
<!doctype html>
<html lang="fr">
<head>
<meta charset="utf-8">
<title>Merging 2 CSV documents</title>
<link rel="stylesheet" href="example.css">
public function testSortBy2()
{
$string = 'john,doe,john.doe@example.com' . PHP_EOL . 'john,doe,john.doe@example.com';
$csv = Reader::createFromString($string);
$func = function ($rowA, $rowB) {
return strcmp($rowA[0], $rowB[0]);
};
$csv->addSortBy($func);
$this->assertSame([['john', 'doe', '*****@*****.**'], ['john', 'doe', '*****@*****.**']], $csv->fetchAll());
}
/**
* @return array
*/
private function getDataForColumnRoles() : array
{
$config = $this->job->configuration;
$data = ['columns' => [], 'columnCount' => 0];
// show user column role configuration.
$content = $this->job->uploadFileContents();
// create CSV reader.
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$start = $config['has-headers'] ? 1 : 0;
$end = $start + config('csv.example_rows');
// collect example data in $data['columns']
while ($start < $end) {
$row = $reader->fetchOne($start);
// run specifics here:
// and this is the point where the specifix go to work.
foreach ($config['specifics'] as $name => $enabled) {
/** @var SpecificInterface $specific */
$specific = app('FireflyIII\\Import\\Specifics\\' . $name);
// it returns the row, possibly modified:
$row = $specific->run($row);
}
foreach ($row as $index => $value) {
$value = trim($value);
if (strlen($value) > 0) {
$data['columns'][$index][] = $value;
}
}
$start++;
$data['columnCount'] = count($row) > $data['columnCount'] ? count($row) : $data['columnCount'];
}
// make unique example data
foreach ($data['columns'] as $index => $values) {
$data['columns'][$index] = array_unique($values);
}
$data['set_roles'] = [];
// collect possible column roles:
$data['available_roles'] = [];
foreach (array_keys(config('csv.import_roles')) as $role) {
$data['available_roles'][$role] = trans('csv.column_' . $role);
}
$config['column-count'] = $data['columnCount'];
$this->job->configuration = $config;
$this->job->save();
return $data;
}
$geocodeURL = 'https://maps.googleapis.com/maps/api/geocode/json?key=AIzaSyCEM3edAHanZa360b3hPAsLbuXs-s8KAqc&address=';
$csv = $content = file_get_contents($link);
/*
Timestamp
Incident description
Type of Incident
Incident date
Incident city
Victim's gender
Victim's country of origin
LINK TO ARTICLE OR SOURCE OF INFORMATION
Your contact information (Optional)
Incident country
Who is reporting?
*/
$reader = Reader::createFromString($csv, "\n");
$data = $reader->fetchAssoc();
$incidents = [];
foreach ($data as $index => $row) {
$incident = new Incident();
$incident->description = $row['Incident description'];
$incident->date = $row['Incident date'];
$incident->created_at = $row['Timestamp'];
$incident->city = $row['Incident city'];
$incident->country = $row['Incident country'];
$incident->type = $row['Type of Incident'];
$incident->link = $row['LINK TO ARTICLE OR SOURCE OF INFORMATION'];
$incident->victimCountry = $row['Victim\'s country of origin'];
$incident->victimGender = $row['Victim\'s gender'];
$incident->reporterType = $row['Who is reporting?'];
$incident->reporterContact = $row['Your contact information (Optional)'];
private function buildCsvReader()
{
if (is_null($this->options['content']) == true) {
$reader = Reader::createFromPath($this->options['file']);
} else {
$reader = Reader::createFromString($this->options['content']);
}
$reader->setDelimiter($this->options['delimiter_character']);
$reader->setEnclosure($this->options['enclosure_character']);
$reader->setEscape($this->options['escape_character']);
return $reader;
}
