public function csvImport($csvFile, $entity, $user, $fileRecord) { // Create and configure the reader $headers = new ParagonArray(); $file = new \SplFileObject($csvFile); $csvReader = new CsvReader($file); $csvReader->setHeaderRowNumber(0); $csvReader->setStrict(false); $csvReader->setColumnHeaders($headers->csvReaderArray()); $workflow = new Workflow($csvReader); $doctrineWriter = new DoctrineWriter($this->em, $entity); $doctrineWriter->disableTruncate(); $workflow->addWriter($doctrineWriter); $dateConverter = new DateConventer($fileRecord->getPlanDate()); $weekNumberConverter = new WeekNumberConventer($fileRecord->getPlanDate()); $planNameConverter = new PlanNameConventer($fileRecord->getName()); $routeNameConventer = new RouteNameConventer($fileRecord->getPlanDate()); $dateTimeNow = new DateTimeNow(); $dateToMicroConventer = new TimeToMicroConventer(); $timeConverter = new DateTimeValueConverter('H:i'); $uploadedBy = new UploadedBy($user); $planDateConvert = new PlanDateConvert($fileRecord->getPlanDate()); //\Doctrine\Common\Util\Debug::dump($date->format('Y-m-d')); //die; $date = $fileRecord->getPlanDate(); $workflow->setSkipItemOnFailure(true)->addValueConverter('routeNo', $routeNameConventer)->addValueConverter('startTime', $dateConverter)->addValueConverter('sourceDepotDepartureTime', $dateConverter)->addValueConverter('arrivalTime', $dateConverter)->addValueConverter('departTime', $dateConverter)->addValueConverter('endDepotArrivalTime', $dateConverter)->addValueConverter('tripStart', $dateConverter)->addValueConverter('destDepotArrivalTime', $dateConverter)->addValueConverter('destDepotDepartureTime', $dateConverter)->addValueConverter('sourceDepotArrivalTime2', $dateConverter)->addValueConverter('sourceDepotDepartureTime2', $dateConverter)->addValueConverter('startDepotDepartureTime', $dateConverter)->addValueConverter('endTime', $dateConverter)->addValueConverter('callDuration', $dateToMicroConventer)->addValueConverter('dutyTime', $dateToMicroConventer)->addValueConverter('driveTime', $dateToMicroConventer)->addValueConverter('emptyTime', $dateToMicroConventer)->addValueConverter('timeWindowStart', $dateConverter)->addValueConverter('timeWindowEnd', $dateConverter)->addValueConverter('uploadDate', $dateTimeNow)->addValueConverter('uploadedBy', $uploadedBy)->addValueConverter('planDate', $planDateConvert)->addValueConverter('planName', $planNameConverter)->addValueConverter('weekNumber', $weekNumberConverter); // \Doctrine\Common\Util\Debug::dump($doctrineWriter); // die; $result = $workflow->process(); //update query to generate route and trip combination $query = $this->em->createQuery("UPDATE NTPBundle:ParagonData p SET p.routeNo = CONCAT(p.routeNo,'0', p.tripNo) " . "WHERE p.planDate=:date")->setParameter('date', $date); $query->execute(); return $result; }
/** * Parse attachments * * @return array Returns array with failed or success data * (See collector-common/src/Collector.php) for more info. */ public function parse() { if (empty($this->key) || strlen($this->key) < 10) { return $this->failed("Invalid SNDS key: {$this->key}"); } if (!filter_var($this->url, FILTER_VALIDATE_URL) === true) { return $this->failed("Invalid URL configured: {$this->url}"); } if (!$this->createWorkingDir()) { return $this->failed("Unable to create working directory"); } $tempFile = "{$this->tempPath}/snds.csv"; $client = new GuzzleHttp\Client(); $res = $client->request('GET', "{$this->url}?key={$this->key}", ['http_errors' => false, 'save_to' => $tempFile]); if ($res->getStatusCode() !== 200) { return $this->failed("URL collection from {$this->url} resulted in a {$res->getStatusCode()}"); } $csvReports = new Reader\CsvReader(new SplFileObject($tempFile)); $csvReports->setColumnHeaders(['first_ip', 'last_ip', 'blocked', 'feed']); foreach ($csvReports as $report) { $this->feedName = 'unknown'; // If report type is an alias, get the real type foreach (config("{$this->configBase}.collector.aliasses") as $alias => $real) { if ($report['feed'] == $alias) { $this->feedName = $real; } } if ($this->isKnownFeed() && $this->isEnabledFeed()) { $firstIP = inetPtoi($report['first_ip']); $lastIP = inetPtoi($report['last_ip']); if (!empty($firstIP) && !empty($lastIP) && $firstIP <= $lastIP) { for ($x = $firstIP; $x <= $lastIP; $x++) { $report['ip'] = inetItop($x); if ($this->hasRequiredFields($report) === true) { $report = $this->applyFilters($report); $incident = new Incident(); $incident->source = config("{$this->configBase}.collector.name"); $incident->source_id = false; $incident->ip = $report['ip']; $incident->domain = false; $incident->class = config("{$this->configBase}.feeds.{$this->feedName}.class"); $incident->type = config("{$this->configBase}.feeds.{$this->feedName}.type"); /* * This prevents multiple incidents on the same day. So info * blob has a scan time and this a report time */ $incident->timestamp = strtotime('0:00'); $incident->information = json_encode($report); $this->incidents[] = $incident; } } } } } return $this->success(); }
public function load() { $reader = new CsvReader($this->getFile(), $this->getOption('delimiter'), $this->getOption('enclosure'), $this->getOption('escape')); $headers = $this->getOption('headers'); $headerRow = $this->getOption('header_row'); if (false !== $headerRow) { $duplicateHeaders = $headers == 'merge' ? CsvReader::DUPLICATE_HEADERS_MERGE : CsvReader::DUPLICATE_HEADERS_INCREMENT; $reader->setHeaderRowNumber($headerRow, $duplicateHeaders); } if (is_array($headers)) { $reader->setColumnHeaders($headers); } elseif ('number' == $headers) { $headers = []; for ($i = 1; $i <= count($reader->getColumnHeaders()); ++$i) { $headers[] = 'field' . $i; } $reader->setColumnHeaders($headers); } $reader->setStrict($this->getOption('strict')); return $this->setReader($reader); }
public function testMaximumNesting() { if (!function_exists('xdebug_is_enabled')) { $this->markTestSkipped('xDebug is not installed'); } $xdebug_start = !xdebug_is_enabled(); if ($xdebug_start) { xdebug_enable(); } ini_set('xdebug.max_nesting_level', 200); $file = new \SplTempFileObject(); for ($i = 0; $i < 500; $i++) { $file->fwrite("1,2,3\n"); } $reader = new CsvReader($file); $reader->rewind(); $reader->setStrict(true); $reader->setColumnHeaders(array('one', 'two')); $current = $reader->current(); $this->assertEquals(null, $current); if ($xdebug_start) { xdebug_disable(); } }
private function formatToReader(Format $format, \SplFileInfo $file) { if ($format instanceof CsvFormat) { $reader = new CsvReader($file->openFile(), $format->getDelimiter(), $format->getEnclosure(), $format->getEscape()); $reader->setStrict(false); if ($format->isHeaderInFirstRow()) { $reader->setHeaderRowNumber(0, CsvReader::DUPLICATE_HEADERS_MERGE); $reader->setColumnHeaders(array_map('trim', $reader->getColumnHeaders())); //TODO some header-collaborator? } } elseif ($format instanceof ExcelFormat) { $headerRowNumber = $format->isHeaderInFirstRow() ? 0 : null; $reader = new ExcelReader($file->openFile(), $headerRowNumber, $format->getActivesheet()); if ($format->isHeaderInFirstRow()) { $reader->setColumnHeaders(array_map('trim', $reader->getColumnHeaders())); //TODO some header-collaborator? } } elseif ($format instanceof JsonFormat) { $array = json_decode(file_get_contents($file), true); $reader = new ArrayReader($array); } elseif ($format instanceof XmlFormat) { $reader = new XmlReader($file->openFile(), $format->getXpath()); } elseif ($format instanceof CompressedFormat && $format->getSubFormat()) { $reader = $this->formatToReader($format->getSubFormat(), $format->getInsideStream($file)); } else { throw new InvalidConfigurationException('Cannot build reader. Unknown format: ' . $format); } return $reader; }