public function openResource() { LogHelper::log_notice(t('Parsing data from a buffer (size: @bufferSize) ...', array('@bufferSize' => $this->bufferSize))); $result = parent::openResource(); if ($result) { $this->index = 0; } return $result; }
public function openResource() { LogHelper::log_notice(t('Parsing @filename ...', array('@filename' => $this->filename))); $result = parent::openResource(); if ($result) { ini_set('auto_detect_line_endings', TRUE); $this->handle = fopen($this->filename, 'r'); $result = $this->handle !== FALSE; } return $result; }
public function parse(AbstractDataProvider $dataProvider, array $dataSubmitters = NULL) { $skippedRecordCount = 0; $loadedRecordCount = 0; $timeStart = microtime(TRUE); if ($dataProvider->openResource()) { LogHelper::log_notice(t( 'Parsing @limitRecordCount records. Skipping first @skipRecordCount records (memory usage: @memoryUsed) ...', array( '@skipRecordCount' => $this->skipRecordCount, '@limitRecordCount' => (isset($this->limitRecordCount) ? $this->limitRecordCount : t('all')), '@memoryUsed' => memory_get_usage()))); try { if ($this->initializeProcessing($dataSubmitters)) { // preparing list of columns $this->prepareMetaData($dataProvider, $dataSubmitters); $metadataColumnCount = $this->metadata->getColumnCount(FALSE, TRUE); if ((!isset($this->limitRecordCount) || ($this->limitRecordCount > 0)) && $this->executeBeforeProcessingRecords($dataSubmitters, $dataProvider)) { // processing records $fileProcessedCompletely = FALSE; while (!isset($this->limitRecordCount) || ($loadedRecordCount < $this->limitRecordCount)) { $dataProvider->startReading(); $record = $this->parseNextRecord($dataProvider, $dataSubmitters); // number of loaded columns should match number of columns in meta data if (isset($record)) { $attempt = 1; while (TRUE) { $recordColumnCount = count($record); if ($recordColumnCount == $metadataColumnCount) { break; } else { if ($attempt > self::$MAX_ATTEMPTS_TO_RESOLVE_PARSING_ISSUES) { $dataProvider->endReading(); LogHelper::log_debug($this->metadata); LogHelper::log_debug($record); throw new DataParserException(t( 'Expected to load values for %metadataColumnCount columns. Loaded %loadedColumnCount [line: %lineNumber]', array('%metadataColumnCount' => $metadataColumnCount, '%loadedColumnCount' => $recordColumnCount, '%lineNumber' => $dataProvider->getCurrentLineNumber()))); } $dataProvider->rollbackReading(); $dataProvider->startReading(); $record = $this->parseNextRecord($dataProvider, $dataSubmitters, $attempt); $attempt++; } } } $dataProvider->endReading(); // checking if we reached the end if (!isset($record)) { $fileProcessedCompletely = TRUE; break; } // skipping required number of records if ($skippedRecordCount < $this->skipRecordCount) { $skippedRecordCount++; continue; } $this->postProcessColumnValues($record); // checking if we need to skip processing the record $recordNumber = $dataProvider->getCurrentLineNumber(); if ($this->executeBeforeRecordSubmitted($dataSubmitters, $recordNumber, $record)) { $this->submitRecord($dataSubmitters, $recordNumber, $record); $this->executeAfterRecordSubmitted($dataSubmitters, $recordNumber, $record); $loadedRecordCount++; if (($loadedRecordCount % 1000) == 0) { LogHelper::log_info(t( 'Processed @recordCount records so far (memory usage: @memoryUsed)', array('@recordCount' => $loadedRecordCount, '@memoryUsed' => memory_get_usage()))); } } } $this->executeAfterProcessingRecords($dataSubmitters, $fileProcessedCompletely); } $this->finishProcessing($dataSubmitters); } } catch (DataParserException $e) { LogHelper::log_warn(t('Place of original exception @file:@line', array('@file' => $e->getFile(), '@line' => $e->getLine()))); try { $this->abortProcessing($dataSubmitters); } catch (Exception $ne) { // we do not need to rethrow this exception. We need to preserve and rethrow original exception LogHelper::log_error($ne); } try { $dataProvider->closeResource(); } catch (Exception $ne) { // we do not need to rethrow this exception. We need to preserve and rethrow original exception LogHelper::log_error($ne); } throw new IllegalStateException($e->getMessage()); } catch (Exception $e) { LogHelper::log_warn(t('Place of original exception @file:@line', array('@file' => $e->getFile(), '@line' => $e->getLine()))); try { $this->abortProcessing($dataSubmitters); } catch (Exception $ne) { // we do not need to rethrow this exception. We need to preserve and rethrow original exception LogHelper::log_error($ne); } $ise = new IllegalStateException( ExceptionHelper::getExceptionMessage($e) . t(' [%lineNumber line(s) parsed so far]', array('%lineNumber' => $dataProvider->getCurrentLineNumber())), 0, $e); try { $dataProvider->closeResource(); } catch (Exception $ne) { // we do not need to rethrow this exception. We need to preserve and rethrow original exception LogHelper::log_error($ne); } throw $ise; } $dataProvider->closeResource(); } LogHelper::log_notice(t( 'Processing @recordCount record(s) took !executionTime', array('@recordCount' => $loadedRecordCount, '!executionTime' => LogHelper::formatExecutionTime($timeStart)))); return $loadedRecordCount; }