public function csvImport($csvFile, $entity, $user, $fileRecord)
{
// Create and configure the reader
$headers = new ParagonArray();
$file = new \SplFileObject($csvFile);
$csvReader = new CsvReader($file);
$csvReader->setHeaderRowNumber(0);
$csvReader->setStrict(false);
$csvReader->setColumnHeaders($headers->csvReaderArray());
$workflow = new Workflow($csvReader);
$doctrineWriter = new DoctrineWriter($this->em, $entity);
$doctrineWriter->disableTruncate();
$workflow->addWriter($doctrineWriter);
$dateConverter = new DateConventer($fileRecord->getPlanDate());
$weekNumberConverter = new WeekNumberConventer($fileRecord->getPlanDate());
$planNameConverter = new PlanNameConventer($fileRecord->getName());
$routeNameConventer = new RouteNameConventer($fileRecord->getPlanDate());
$dateTimeNow = new DateTimeNow();
$dateToMicroConventer = new TimeToMicroConventer();
$timeConverter = new DateTimeValueConverter('H:i');
$uploadedBy = new UploadedBy($user);
$planDateConvert = new PlanDateConvert($fileRecord->getPlanDate());
//\Doctrine\Common\Util\Debug::dump($date->format('Y-m-d'));
//die;
$date = $fileRecord->getPlanDate();
$workflow->setSkipItemOnFailure(true)->addValueConverter('routeNo', $routeNameConventer)->addValueConverter('startTime', $dateConverter)->addValueConverter('sourceDepotDepartureTime', $dateConverter)->addValueConverter('arrivalTime', $dateConverter)->addValueConverter('departTime', $dateConverter)->addValueConverter('endDepotArrivalTime', $dateConverter)->addValueConverter('tripStart', $dateConverter)->addValueConverter('destDepotArrivalTime', $dateConverter)->addValueConverter('destDepotDepartureTime', $dateConverter)->addValueConverter('sourceDepotArrivalTime2', $dateConverter)->addValueConverter('sourceDepotDepartureTime2', $dateConverter)->addValueConverter('startDepotDepartureTime', $dateConverter)->addValueConverter('endTime', $dateConverter)->addValueConverter('callDuration', $dateToMicroConventer)->addValueConverter('dutyTime', $dateToMicroConventer)->addValueConverter('driveTime', $dateToMicroConventer)->addValueConverter('emptyTime', $dateToMicroConventer)->addValueConverter('timeWindowStart', $dateConverter)->addValueConverter('timeWindowEnd', $dateConverter)->addValueConverter('uploadDate', $dateTimeNow)->addValueConverter('uploadedBy', $uploadedBy)->addValueConverter('planDate', $planDateConvert)->addValueConverter('planName', $planNameConverter)->addValueConverter('weekNumber', $weekNumberConverter);
// \Doctrine\Common\Util\Debug::dump($doctrineWriter);
// die;
$result = $workflow->process();
//update query to generate route and trip combination
$query = $this->em->createQuery("UPDATE NTPBundle:ParagonData p SET p.routeNo = CONCAT(p.routeNo,'0', p.tripNo) " . "WHERE p.planDate=:date")->setParameter('date', $date);
$query->execute();
return $result;
}
/**
* Parse attachments
*
* @return array Returns array with failed or success data
* (See collector-common/src/Collector.php) for more info.
*/
public function parse()
{
if (empty($this->key) || strlen($this->key) < 10) {
return $this->failed("Invalid SNDS key: {$this->key}");
}
if (!filter_var($this->url, FILTER_VALIDATE_URL) === true) {
return $this->failed("Invalid URL configured: {$this->url}");
}
if (!$this->createWorkingDir()) {
return $this->failed("Unable to create working directory");
}
$tempFile = "{$this->tempPath}/snds.csv";
$client = new GuzzleHttp\Client();
$res = $client->request('GET', "{$this->url}?key={$this->key}", ['http_errors' => false, 'save_to' => $tempFile]);
if ($res->getStatusCode() !== 200) {
return $this->failed("URL collection from {$this->url} resulted in a {$res->getStatusCode()}");
}
$csvReports = new Reader\CsvReader(new SplFileObject($tempFile));
$csvReports->setColumnHeaders(['first_ip', 'last_ip', 'blocked', 'feed']);
foreach ($csvReports as $report) {
$this->feedName = 'unknown';
// If report type is an alias, get the real type
foreach (config("{$this->configBase}.collector.aliasses") as $alias => $real) {
if ($report['feed'] == $alias) {
$this->feedName = $real;
}
}
if ($this->isKnownFeed() && $this->isEnabledFeed()) {
$firstIP = inetPtoi($report['first_ip']);
$lastIP = inetPtoi($report['last_ip']);
if (!empty($firstIP) && !empty($lastIP) && $firstIP <= $lastIP) {
for ($x = $firstIP; $x <= $lastIP; $x++) {
$report['ip'] = inetItop($x);
if ($this->hasRequiredFields($report) === true) {
$report = $this->applyFilters($report);
$incident = new Incident();
$incident->source = config("{$this->configBase}.collector.name");
$incident->source_id = false;
$incident->ip = $report['ip'];
$incident->domain = false;
$incident->class = config("{$this->configBase}.feeds.{$this->feedName}.class");
$incident->type = config("{$this->configBase}.feeds.{$this->feedName}.type");
/*
* This prevents multiple incidents on the same day. So info
* blob has a scan time and this a report time
*/
$incident->timestamp = strtotime('0:00');
$incident->information = json_encode($report);
$this->incidents[] = $incident;
}
}
}
}
}
return $this->success();
}
public function load()
{
$reader = new CsvReader($this->getFile(), $this->getOption('delimiter'), $this->getOption('enclosure'), $this->getOption('escape'));
$headers = $this->getOption('headers');
$headerRow = $this->getOption('header_row');
if (false !== $headerRow) {
$duplicateHeaders = $headers == 'merge' ? CsvReader::DUPLICATE_HEADERS_MERGE : CsvReader::DUPLICATE_HEADERS_INCREMENT;
$reader->setHeaderRowNumber($headerRow, $duplicateHeaders);
}
if (is_array($headers)) {
$reader->setColumnHeaders($headers);
} elseif ('number' == $headers) {
$headers = [];
for ($i = 1; $i <= count($reader->getColumnHeaders()); ++$i) {
$headers[] = 'field' . $i;
}
$reader->setColumnHeaders($headers);
}
$reader->setStrict($this->getOption('strict'));
return $this->setReader($reader);
}
public function testMaximumNesting()
{
if (!function_exists('xdebug_is_enabled')) {
$this->markTestSkipped('xDebug is not installed');
}
$xdebug_start = !xdebug_is_enabled();
if ($xdebug_start) {
xdebug_enable();
}
ini_set('xdebug.max_nesting_level', 200);
$file = new \SplTempFileObject();
for ($i = 0; $i < 500; $i++) {
$file->fwrite("1,2,3\n");
}
$reader = new CsvReader($file);
$reader->rewind();
$reader->setStrict(true);
$reader->setColumnHeaders(array('one', 'two'));
$current = $reader->current();
$this->assertEquals(null, $current);
if ($xdebug_start) {
xdebug_disable();
}
}
private function formatToReader(Format $format, \SplFileInfo $file)
{
if ($format instanceof CsvFormat) {
$reader = new CsvReader($file->openFile(), $format->getDelimiter(), $format->getEnclosure(), $format->getEscape());
$reader->setStrict(false);
if ($format->isHeaderInFirstRow()) {
$reader->setHeaderRowNumber(0, CsvReader::DUPLICATE_HEADERS_MERGE);
$reader->setColumnHeaders(array_map('trim', $reader->getColumnHeaders()));
//TODO some header-collaborator?
}
} elseif ($format instanceof ExcelFormat) {
$headerRowNumber = $format->isHeaderInFirstRow() ? 0 : null;
$reader = new ExcelReader($file->openFile(), $headerRowNumber, $format->getActivesheet());
if ($format->isHeaderInFirstRow()) {
$reader->setColumnHeaders(array_map('trim', $reader->getColumnHeaders()));
//TODO some header-collaborator?
}
} elseif ($format instanceof JsonFormat) {
$array = json_decode(file_get_contents($file), true);
$reader = new ArrayReader($array);
} elseif ($format instanceof XmlFormat) {
$reader = new XmlReader($file->openFile(), $format->getXpath());
} elseif ($format instanceof CompressedFormat && $format->getSubFormat()) {
$reader = $this->formatToReader($format->getSubFormat(), $format->getInsideStream($file));
} else {
throw new InvalidConfigurationException('Cannot build reader. Unknown format: ' . $format);
}
return $reader;
}
