/**
* @return \Ddeboer\DataImport\Result
* @throws \Ddeboer\DataImport\Exception\ExceptionInterface
* @throws \Exception
*/
public function processData()
{
$workflow = new Workflow($this->reader);
$output = new ConsoleOutput();
// Don’t import the non-metadata
$filter = new Filter\CallbackFilter(function ($row) {
if (is_numeric($row['reg_id'])) {
return true;
}
if ( ! trim($row['reg_id'])) {
foreach ($row as $item) {
if ( ! is_array($item)) {
if (trim($item)) {
return true;
}
} else {
foreach ($item as $foo) {
if (trim($foo)) {
return true;
}
}
}
}
}
return false;
});
$trimConverter = new CallbackItemConverter(function ($row) {
foreach ($row as $key => $value) {
$row[$key] = trim($row[$key]);
}
return $row;
});
$lexicalConverter = new CallbackItemConverter(function ($row) {
foreach ($row as $key => $value) {
if (isset( $this->prolog['columns'][$key]['id'] ) and $this->prolog['columns'][$key]['id'] == 27) {
$lang = $this->prolog['columns'][$key]['lang'];
//check for existing language suffix
if ($value and ! preg_match("/\\." . $lang . "$/u", $value)) {
$row[$key] = $value . "." . $lang;
}
}
}
return $row;
});
$typeConverter = new MappingValueConverter([
'rdfs:class' => 'class',
'rdfs:property' => 'property',
'class' => 'class',
'property' => 'property',
'Class' => 'class',
'Property' => 'property',
'subclass' => 'class',
'subproperty' => 'property',
'' => '',
]);
$vocabWriter = new Writer\CallbackWriter(function ($row) {
$this->setPrologColumns();
//todo: $row[59] (status) and $row[62] (uri) sre hardcoded but the array key should be set somewhere globally
if ( ! isset( $row[59] )) {
$row[59] = $this->prolog['defaults']['statusId'];
}
$rowStatus = $row[59];
$language = $this->prolog['defaults']['lang'];
foreach ($row as $key => &$element) {
$this->updateRowUris($key, $element);
}
$uri = $this->getFqn($row[62]);
$property = null;
if ( ! empty( $row['reg_id'] )) {
$property = \ConceptPeer::retrieveByPK($row['reg_id']);
} else {
//check for an existing property by uri
/** @var \Concept $property */
$property = \ConceptPeer::getConceptByUri($uri);
}
//even if we found a property, we kill it if it's in a different schema than we're populating
if ($property and $property->getVocabularyId() !== $this->vocabId) {
//todo: we should log this event
unset( $property );
}
if (empty( $property )) { //it's a new property
$property = new \Concept();
$property->setVocabularyId($this->vocabId);
$property->setCreatedUserId($this->userId);
$property->setUpdatedUserId($this->userId);
$property->setStatusId($rowStatus);
$property->setLanguage($language);
$property->setUri($uri);
$property->save();
}
unset( $row['reg_id'] );
if ($property) {
// if (8 == $rowStatus) {
// //it's been deprecated and we don't do anything else
// $property->setStatusId($rowStatus);
// $this->updateElement($element, $dbElement, $property);
// } else {
$dbElements = $property->getElementsForImport($this->profileProperties);
foreach ($dbElements as $key => $dbElement) {
/** @var string | array $rowElement */
$rowElement = isset( $row[$key] ) ? $row[$key] : null;
if (is_array($rowElement)) {
foreach ($rowElement as $elementKey => &$element) {
if ($this->updateConceptProperty($element, $dbElement, $property)) {
unset( $rowElement[$elementKey] );
}
}
} else {
if ($this->updateConceptProperty($rowElement, $dbElement, $property)) {
unset( $row[$key] );
}
}
}
foreach ($row as $key => $value) {
$dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null;
if ( ! empty( $this->prolog['columns'][$key]['property'] )) {
$profileProperty = $this->prolog['columns'][$key]['property'];
if (is_array($value)) {
foreach ($value as &$oneValue) {
$language = $this->prolog['columns'][$key]['lang'][0];
$this->upsertConceptFromRow($dbElement,
$oneValue,
$rowStatus,
$property,
$profileProperty,
$language,
$key);
}
} else {
$language = $this->prolog['columns'][$key]['lang'];
$this->upsertConceptFromRow($dbElement,
$value,
$rowStatus,
$property,
$profileProperty,
$language,
$key);
}
}
}
}
$property->setUri($uri);
$property->setStatusId($rowStatus);
$affectedRows = $property->save();
if (empty( $property->getPrefLabel() )) {
$prefLabel = \ConceptPropertyPeer::lookupProperty($property->getId(),
19,
$this->prolog['defaults']['lang']);
if ( ! empty( $prefLabel )) {
$property->setPrefLabel($prefLabel->getObject());
$property->setPrefLabelId($prefLabel->getId());
$property->setLanguage($this->prolog['defaults']['lang']);
$property->save();
}
}
return;
//**************
// CRUFT ALERT
// The rest of this is never run!!
//**************
//build an array of references
$newElements = [ ];
$newElements2 = [ ];
if ( ! isset( $row['status'] )) {
$row[14] = $this->prolog['defaults']['statusId'];
}
foreach ($row as $key => $element) {
//skip it there's no property id
$columnKey = $this->prolog['columns'][$key];
if ( ! $columnKey['id']) {
continue;
}
if ( ! empty( $columnKey['type'] ) and $this->useCuries) {
$element = $this->getFqn($element);
}
$key2 = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element);
$newElements[$key2] = [ ];
$newElements[$key2] += $columnKey;
$newElements[$key2]['val'] = $element;
/** @var \ProfileProperty $profileProperty */
if (isset( $columnKey['property'] )) {
$profileProperty = $columnKey['property'];
$var = [
'matchkey' => $key2,
'val' => $newElements[$key2],
];
if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) {
$newElements2[$columnKey['id']][$columnKey['lang']][] = $var;
} else {
$newElements2[$columnKey['id']][] = $var;
}
}
}
if ( ! empty( $row['reg_id'] )) {
$property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
if ($property) {
$dbElements = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty();
$dbElements2 = [ ];
/** @var \SchemaPropertyElement $dbElement */
foreach ($dbElements as $dbElement) {
if ($dbElement->getProfileProperty()->getHasLanguage()) {
$dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement;
} else {
$dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement;
}
}
/** @var \SchemaPropertyElement $element */
foreach ($dbElements as $element) {
$language = $element->getLanguage();
$profilePropertyId = $element->getProfilePropertyId();
$key = md5(strval($profilePropertyId) . strval($language) . $element->getObject());
//if the newelement key matches then
if (isset( $newElements[$key] )) {
if ($element->getProfileProperty()->getHasLanguage()) {
$newElements2Array = $newElements2[$profilePropertyId][$language];
} else {
$newElements2Array = $newElements2[$profilePropertyId];
}
$count = count($newElements2Array);
for ($I = 0; $I < $count; $I++) {
if ($newElements2Array[$I]['matchkey'] == $key) {
unset( $newElements2Array[$I] );
}
}
unset( $newElements[$key] );
$element->importStatus = 'match';
continue;
} else {
if ($element->getProfileProperty()->getHasLanguage()) {
if (isset( $newElements2[$profilePropertyId][$language] )) {
$count = count($newElements2[$profilePropertyId][$language]);
for ($I = 0; $I < $count; $I++) {
if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) {
unset( $newElements2[$profilePropertyId][$language][$I] );
$element->importStatus = 'match';
if ( ! count($newElements2[$profilePropertyId][$language])) {
unset( $newElements2[$profilePropertyId][$language] );
}
continue;
}
}
}
} else {
//compare the old values with the new with the same key
$count = count($newElements2[$profilePropertyId]);
for ($I = 0; $I < $count; $I++) {
if (isset( $newElements2[$profilePropertyId][$I] )) {
if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) {
unset( $newElements2[$profilePropertyId][$I] );
$element->importStatus = 'match';
continue;
}
}
}
}
//if the key matches then
//if the value matches
//delete the newElement
//else the value doesn't match
//if the newElement value is empty
//delete the dbElement
}
$element->matchKey = $key;
}
//update the property values
$property->save();
} else {
//there's no existing property an we have to create a new one
$property = new \SchemaProperty();
}
foreach ($newElements as $key => $newElement) {
if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) {
$profileProperty = $newElement['property'];
//walk the old elements looking for a match on predicate + language
/** @var \SchemaPropertyElement[] $oldElement */
foreach ($dbElements as $oldElement) {
/** @var \SchemaPropertyElement $oldOne */
$oldOne = &$oldElement['element'];
if ($newElement['id'] == $oldOne->getProfilePropertyId()) {
/** @var \ProfileProperty $profileProperty */
if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) {
if ( ! empty( $newElement['val'] )) {
$oldOne->setObject($newElement['val']);
$oldOne->setUpdatedUserId($this->userId);
$oldOne->setStatusId($row['status']);
//$oldOne->save();
$oldElement['status'] = "updated";
} else {
$oldOne->delete();
$oldElement['status'] = "deleted";
}
//update the property value
if ($profileProperty->getIsInForm()) {
$this->setPropertyValue($newElement['val'],
$property,
$profileProperty->getName(),
! $profileProperty->getIsObjectProp());
}
break;
}
}
}
//we looked through them all, add a new one
if ( ! empty( $newElement['val'] )) {
$addMe = new \SchemaPropertyElement();
$addMe->setObject($newElement['val']);
//$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property);
$addMe->setCreatedUserId($this->userId);
$addMe->setUpdatedUserId($this->userId);
$addMe->setLanguage($newElement['lang']);
$addMe->setProfilePropertyId($newElement['id']);
$addMe->setStatusId($row['status']);
$addMe->importId = $this->importId;
//$addMe->save();
$property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe);
//update the property value
if ($profileProperty->getIsInForm()) {
$this->setPropertyValue($newElement['val'],
$property,
$profileProperty->getName(),
! $profileProperty->getIsObjectProp());
}
}
}
}
//update the property
if ($property) {
$property->setStatusId($row['status']);
$property->save();
}
}
//var_dump($row);
});
$schemaWriter = new Writer\CallbackWriter(function ($row) {
$this->setPrologColumns();
if ( ! isset( $row[14] )) {
$row[14] = $this->prolog['defaults']['statusId'];
}
$rowStatus = $row[14];
$language = $this->prolog['defaults']['lang'];
foreach ($row as $key => &$element) {
$this->updateRowUris($key, $element);
}
$uri = $row[13];
$property = null;
$schemaId = $this->vocabId;
if ( ! empty( $row['reg_id'] )) {
$property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
} else {
//check for an existing property by uri
/** @var \SchemaProperty $property */
$property = \SchemaPropertyPeer::retrieveByUri($uri);
if ($property) {
$schemaId = $property->getSchemaId();
}
}
//even if we found a property, we kill it if it's in a different schema than we're populating
if ($property and $schemaId !== $this->vocabId) {
//todo: we should log this event
unset( $property );
}
if (empty( $property )) { //it's a new property
$property = new \SchemaProperty();
$property->setSchemaId($this->vocabId);
$property->setCreatedUserId($this->userId);
$property->setUpdatedUserId($this->userId);
$property->setStatusId($rowStatus);
$property->setLanguage($language);
$property->save();
}
unset( $row['reg_id'] );
if ($property) {
// if (8 == $rowStatus) {
// //it's been deprecated and we don't do anything else
// $property->setStatusId($rowStatus);
// $this->updateElement($element, $dbElement, $property);
// } else {
$dbElements = $property->getElementsForImport($this->profileProperties);
foreach ($dbElements as $key => $dbElement) {
/** @var string | array $rowElement */
$rowElement = isset( $row[$key] ) ? $row[$key] : null;
if (is_array($rowElement)) {
foreach ($rowElement as $elementKey => &$element) {
if ($this->updateElement($element, $dbElement, $property)) {
unset( $rowElement[$elementKey] );
}
}
} else {
if ($this->updateElement($rowElement, $dbElement, $property)) {
unset( $row[$key] );
}
}
}
foreach ($row as $key => $value) {
$dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null;
if ( ! empty( $this->prolog['columns'][$key]['property'] )) {
$profileProperty = $this->prolog['columns'][$key]['property'];
if (is_array($value)) {
foreach ($value as &$oneValue) {
$language = $this->prolog['columns'][$key]['lang'][0];
$this->upsertElementFromRow($dbElement,
$oneValue,
$rowStatus,
$property,
$profileProperty,
$language,
$key);
}
} else {
$language = $this->prolog['columns'][$key]['lang'];
$this->upsertElementFromRow($dbElement,
$value,
$rowStatus,
$property,
$profileProperty,
$language,
$key);
}
if ($key == 'parent_class' and strtolower($property->getType()) == 'class' and $row[$key] != $property->getParentUri()) {
$property->setParentUri($row[$key]);
//we'll set this later
$property->setIsSubpropertyOf(null);
}
if ($key == 'parent_property' and strtolower($property->getType()) == 'property' and $row[$key] != $property->getParentUri()) {
$property->setParentUri($row[$key]);
//we'll set this later
$property->setIsSubpropertyOf(null);
}
}
}
}
$affectedRows = $property->save();
// }
return;
//build an array of references
$newElements = [ ];
$newElements2 = [ ];
if ( ! isset( $row['status'] )) {
$row[14] = $this->prolog['defaults']['statusId'];
}
foreach ($row as $key => $element) {
//skip it there's no property id
$columnKey = $this->prolog['columns'][$key];
if ( ! $columnKey['id']) {
continue;
}
if ( ! empty( $columnKey['type'] ) and $this->useCuries) {
$element = $this->getFqn($element);
}
$key2 = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element);
$newElements[$key2] = [ ];
$newElements[$key2] += $columnKey;
$newElements[$key2]['val'] = $element;
/** @var \ProfileProperty $profileProperty */
if (isset( $columnKey['property'] )) {
$profileProperty = $columnKey['property'];
$var = [
'matchkey' => $key2,
'val' => $newElements[$key2],
];
if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) {
$newElements2[$columnKey['id']][$columnKey['lang']][] = $var;
} else {
$newElements2[$columnKey['id']][] = $var;
}
}
}
if ( ! empty( $row['reg_id'] )) {
$property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
if ($property) {
$dbElements = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty();
$dbElements2 = [ ];
/** @var \SchemaPropertyElement $dbElement */
foreach ($dbElements as $dbElement) {
if ($dbElement->getProfileProperty()->getHasLanguage()) {
$dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement;
} else {
$dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement;
}
}
/** @var \SchemaPropertyElement $element */
foreach ($dbElements as $element) {
$language = $element->getLanguage();
$profilePropertyId = $element->getProfilePropertyId();
$key = md5(strval($profilePropertyId) . strval($language) . $element->getObject());
//if the newelement key matches then
if (isset( $newElements[$key] )) {
if ($element->getProfileProperty()->getHasLanguage()) {
$newElements2Array = $newElements2[$profilePropertyId][$language];
} else {
$newElements2Array = $newElements2[$profilePropertyId];
}
$count = count($newElements2Array);
for ($I = 0; $I < $count; $I++) {
if ($newElements2Array[$I]['matchkey'] == $key) {
unset( $newElements2Array[$I] );
}
}
unset( $newElements[$key] );
$element->importStatus = 'match';
continue;
} else {
if ($element->getProfileProperty()->getHasLanguage()) {
if (isset( $newElements2[$profilePropertyId][$language] )) {
$count = count($newElements2[$profilePropertyId][$language]);
for ($I = 0; $I < $count; $I++) {
if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) {
unset( $newElements2[$profilePropertyId][$language][$I] );
$element->importStatus = 'match';
if ( ! count($newElements2[$profilePropertyId][$language])) {
unset( $newElements2[$profilePropertyId][$language] );
}
continue;
}
}
}
} else {
//compare the old values with the new with the same key
$count = count($newElements2[$profilePropertyId]);
for ($I = 0; $I < $count; $I++) {
if (isset( $newElements2[$profilePropertyId][$I] )) {
if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) {
unset( $newElements2[$profilePropertyId][$I] );
$element->importStatus = 'match';
continue;
}
}
}
}
//if the key matches then
//if the value matches
//delete the newElement
//else the value doesn't match
//if the newElement value is empty
//delete the dbElement
}
$element->matchKey = $key;
}
//update the property values
$property->save();
} else {
//there's no existing property an we have to create a new one
$property = new \SchemaProperty();
}
foreach ($newElements as $key => $newElement) {
if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) {
$profileProperty = $newElement['property'];
//walk the old elements looking for a match on predicate + language
/** @var \SchemaPropertyElement[] $oldElement */
foreach ($dbElements as $oldElement) {
/** @var \SchemaPropertyElement $oldOne */
$oldOne = &$oldElement['element'];
if ($newElement['id'] == $oldOne->getProfilePropertyId()) {
/** @var \ProfileProperty $profileProperty */
if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) {
if ( ! empty( $newElement['val'] )) {
$oldOne->setObject($newElement['val']);
$oldOne->setUpdatedUserId($this->userId);
$oldOne->setStatusId($row['status']);
//$oldOne->save();
$oldElement['status'] = "updated";
} else {
$oldOne->delete();
$oldElement['status'] = "deleted";
}
//update the property value
if ($profileProperty->getIsInForm()) {
$this->setPropertyValue($newElement['val'],
$property,
$profileProperty->getName(),
! $profileProperty->getIsObjectProp());
}
break;
}
}
}
//we looked through them all, add a new one
if ( ! empty( $newElement['val'] )) {
$addMe = new \SchemaPropertyElement();
$addMe->setObject($newElement['val']);
//$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property);
$addMe->setCreatedUserId($this->userId);
$addMe->setUpdatedUserId($this->userId);
$addMe->setLanguage($newElement['lang']);
$addMe->setProfilePropertyId($newElement['id']);
$addMe->setStatusId($row['status']);
$addMe->importId = $this->importId;
//$addMe->save();
$property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe);
//update the property value
if ($profileProperty->getIsInForm()) {
$this->setPropertyValue($newElement['val'],
$property,
$profileProperty->getName(),
! $profileProperty->getIsObjectProp());
}
}
}
}
//update the property
if ($property) {
$property->setStatusId($row['status']);
$property->save();
}
}
//var_dump($row);
});
$workflow->addItemConverter($this->mapping);
$workflow->addFilter($filter);
$workflow->addItemConverter($trimConverter);
$workflow->addItemConverter($lexicalConverter);
$workflow->addWriter(new Writer\ConsoleProgressWriter($output, $this->reader));
$workflow->addValueConverter("4", $typeConverter);
//add a database writer
if ('schema' == $this->type) {
$workflow->addWriter($schemaWriter);
} else {
$workflow->addWriter($vocabWriter);
}
/** @todo we need to make a second pass through to delete missing rows
* for each schemaproperty in the database
* match to a row in the csv
* if the row value is empty and $this->deleteMissing
* delete the entire schemaproperty
*/
$workResults = $workflow->process();
$this->DataWorkflowResults = $workResults;
/** @TODO need to make a second pass through to lookup and set $element->related_schema_property_id */
$this->processParents($this->vocabId);
//use the prolog to configure namespaces, look up correct resources in the database
//store the row number of the first non-meta line
return $workResults;
}