public function log($level, &$message)
{
if ($message instanceof Exception) {
$exception = $message;
$backtrace = $exception->getTrace();
// Add the line throwing the exception to the backtrace.
array_unshift($backtrace, array('message' => ExceptionHelper::getExceptionMessage($exception), 'file' => $exception->getFile(), 'line' => $exception->getLine()));
// resolving an issue that exception printing could consume NN MB of log space
foreach ($backtrace as &$trace) {
if (!isset($trace['args'])) {
continue;
}
$updatedArgs = NULL;
foreach ($trace['args'] as $key => $value) {
$isObject = is_object($value);
if (is_array($value) || $isObject) {
if ($isObject) {
$value = (array) $value;
}
$count = count($value);
$max = min(self::$ARGUMENT_ARRAY_INDEXED__VISIBLE_ELEMENT_MAXIMUM, $count);
$convertedValue = '';
$index = 0;
foreach ($value as $k => $v) {
if ($index >= $max) {
break;
}
if (strlen($convertedValue) > 0) {
$convertedValue .= ', ';
}
if (is_int($k) && $k == $index) {
$convertedValue .= $this->logElementValue($v);
} else {
$convertedValue .= $k . ': ' . $this->logElementValue($v);
}
$index++;
}
// checking if we skip some of the elements
if ($count > $max) {
$convertedValue .= ', ... ' . ($count - $max) . ' more ' . ($isObject ? 'property(-ies)' : 'element(s)');
}
$value = ($isObject ? '{' : '[') . $convertedValue . ($isObject ? '}' : ']');
}
$updatedArgs[$key] = $value;
}
$trace['args'] = $updatedArgs;
}
unset($trace);
$message = $backtrace;
}
}
public function doBeforeRecordSubmitted(RecordMetaData $recordMetaData, $recordNumber, array &$record) {
$result = parent::doBeforeRecordSubmitted($recordMetaData, $recordNumber, $record);
if ($result) {
$datatypeFactory = DataTypeFactory::getInstance();
foreach ($recordMetaData->getColumns() as $column) {
if (!isset($record[$column->columnIndex])) {
continue;
}
// FIXME convert data to data type of corresponding lookup dataset column
if ($column->type->getReferencedDatasetName() != NULL) {
continue;
}
$handler = $datatypeFactory->getHandler($column->type->applicationType);
try {
$record[$column->columnIndex] = $handler->castValue($record[$column->columnIndex]);
}
catch (Exception $e) {
$this->exceptionPool[$recordNumber][$column->publicName] = array(
'file' => $e->getFile(),
'line' => $e->getLine(),
'message' => ExceptionHelper::getExceptionMessage($e));
$this->exceptionCount++;
if ($this->exceptionCount >= $this->exceptionPoolSize) {
$this->publishExceptions(TRUE);
}
}
}
$result = !isset($this->exceptionPool);
}
return $result;
}
public function log($level, &$message) {
if ($message instanceof Exception) {
$exception = $message;
$backtrace = $exception->getTrace();
// Add the line throwing the exception to the backtrace.
array_unshift(
$backtrace,
array(
'message' => ExceptionHelper::getExceptionMessage($exception),
'file' => $exception->getFile(),
'line' => $exception->getLine()));
// resolving an issue that exception printing could consume NN MB of log space
foreach ($backtrace as &$trace) {
if (!isset($trace['args'])) {
continue;
}
$updatedArgs = NULL;
foreach ($trace['args'] as $argKey => $argValue) {
$isObject = is_object($argValue);
if (is_array($argValue) || $isObject) {
$value = $isObject ? get_object_vars($argValue) : $argValue;
$count = count($value);
$max = min(self::$ARGUMENT_ARRAY_INDEXED__VISIBLE_ELEMENT_MAXIMUM, $count);
$convertedValue = '';
$index = 0;
foreach ($value as $k => $v) {
if ($index >= $max) {
break;
}
if ($convertedValue != '') {
$convertedValue .= ', ';
}
if (is_int($k) && ($k == $index)) {
$convertedValue .= $this->logElementValue($v);
}
else {
$convertedValue .= $k . self::$FORMATTER_OPTION__COMPOSITE_ELEMENT_KEY_VALUE_DELIMITER . $this->logElementValue($v);
}
$index++;
}
// checking if we skip some of the elements
if ($count > $max) {
$convertedValue .= ', ... ' . ($count - $max) . ' more ' . ($isObject ? 'property(-ies)' : 'element(s)');
}
if ($isObject) {
$s = get_class($argValue) . '{';
if ($convertedValue == '') {
$s .= '...';
}
$s .= '}';
$argValue = $s;
}
else {
$argValue = '[' . $convertedValue . ']';
}
}
$updatedArgs[$argKey] = $argValue;
}
$trace['args'] = $updatedArgs;
}
unset($trace);
$message = $backtrace;
}
}
public function parse(AbstractDataProvider $dataProvider, array $dataSubmitters = NULL) {
$skippedRecordCount = 0;
$loadedRecordCount = 0;
$timeStart = microtime(TRUE);
if ($dataProvider->openResource()) {
LogHelper::log_notice(t(
'Parsing @limitRecordCount records. Skipping first @skipRecordCount records (memory usage: @memoryUsed) ...',
array(
'@skipRecordCount' => $this->skipRecordCount,
'@limitRecordCount' => (isset($this->limitRecordCount) ? $this->limitRecordCount : t('all')),
'@memoryUsed' => memory_get_usage())));
try {
if ($this->initializeProcessing($dataSubmitters)) {
// preparing list of columns
$this->prepareMetaData($dataProvider, $dataSubmitters);
$metadataColumnCount = $this->metadata->getColumnCount(FALSE, TRUE);
if ((!isset($this->limitRecordCount) || ($this->limitRecordCount > 0))
&& $this->executeBeforeProcessingRecords($dataSubmitters, $dataProvider)) {
// processing records
$fileProcessedCompletely = FALSE;
while (!isset($this->limitRecordCount) || ($loadedRecordCount < $this->limitRecordCount)) {
$dataProvider->startReading();
$record = $this->parseNextRecord($dataProvider, $dataSubmitters);
// number of loaded columns should match number of columns in meta data
if (isset($record)) {
$attempt = 1;
while (TRUE) {
$recordColumnCount = count($record);
if ($recordColumnCount == $metadataColumnCount) {
break;
}
else {
if ($attempt > self::$MAX_ATTEMPTS_TO_RESOLVE_PARSING_ISSUES) {
$dataProvider->endReading();
LogHelper::log_debug($this->metadata);
LogHelper::log_debug($record);
throw new DataParserException(t(
'Expected to load values for %metadataColumnCount columns. Loaded %loadedColumnCount [line: %lineNumber]',
array('%metadataColumnCount' => $metadataColumnCount, '%loadedColumnCount' => $recordColumnCount, '%lineNumber' => $dataProvider->getCurrentLineNumber())));
}
$dataProvider->rollbackReading();
$dataProvider->startReading();
$record = $this->parseNextRecord($dataProvider, $dataSubmitters, $attempt);
$attempt++;
}
}
}
$dataProvider->endReading();
// checking if we reached the end
if (!isset($record)) {
$fileProcessedCompletely = TRUE;
break;
}
// skipping required number of records
if ($skippedRecordCount < $this->skipRecordCount) {
$skippedRecordCount++;
continue;
}
$this->postProcessColumnValues($record);
// checking if we need to skip processing the record
$recordNumber = $dataProvider->getCurrentLineNumber();
if ($this->executeBeforeRecordSubmitted($dataSubmitters, $recordNumber, $record)) {
$this->submitRecord($dataSubmitters, $recordNumber, $record);
$this->executeAfterRecordSubmitted($dataSubmitters, $recordNumber, $record);
$loadedRecordCount++;
if (($loadedRecordCount % 1000) == 0) {
LogHelper::log_info(t(
'Processed @recordCount records so far (memory usage: @memoryUsed)',
array('@recordCount' => $loadedRecordCount, '@memoryUsed' => memory_get_usage())));
}
}
}
$this->executeAfterProcessingRecords($dataSubmitters, $fileProcessedCompletely);
}
$this->finishProcessing($dataSubmitters);
}
}
catch (DataParserException $e) {
LogHelper::log_warn(t('Place of original exception @file:@line', array('@file' => $e->getFile(), '@line' => $e->getLine())));
try {
$this->abortProcessing($dataSubmitters);
}
catch (Exception $ne) {
// we do not need to rethrow this exception. We need to preserve and rethrow original exception
LogHelper::log_error($ne);
}
try {
$dataProvider->closeResource();
}
catch (Exception $ne) {
// we do not need to rethrow this exception. We need to preserve and rethrow original exception
LogHelper::log_error($ne);
}
throw new IllegalStateException($e->getMessage());
}
catch (Exception $e) {
LogHelper::log_warn(t('Place of original exception @file:@line', array('@file' => $e->getFile(), '@line' => $e->getLine())));
try {
$this->abortProcessing($dataSubmitters);
}
catch (Exception $ne) {
// we do not need to rethrow this exception. We need to preserve and rethrow original exception
LogHelper::log_error($ne);
}
$ise = new IllegalStateException(
ExceptionHelper::getExceptionMessage($e) . t(' [%lineNumber line(s) parsed so far]', array('%lineNumber' => $dataProvider->getCurrentLineNumber())),
0, $e);
try {
$dataProvider->closeResource();
}
catch (Exception $ne) {
// we do not need to rethrow this exception. We need to preserve and rethrow original exception
LogHelper::log_error($ne);
}
throw $ise;
}
$dataProvider->closeResource();
}
LogHelper::log_notice(t(
'Processing @recordCount record(s) took !executionTime',
array('@recordCount' => $loadedRecordCount, '!executionTime' => LogHelper::formatExecutionTime($timeStart))));
return $loadedRecordCount;
}
