/** * @return \Ddeboer\DataImport\Result * @throws \Ddeboer\DataImport\Exception\ExceptionInterface * @throws \Exception */ public function processData() { $workflow = new Workflow($this->reader); $output = new ConsoleOutput(); // Don’t import the non-metadata $filter = new Filter\CallbackFilter(function ($row) { if (is_numeric($row['reg_id'])) { return true; } if ( ! trim($row['reg_id'])) { foreach ($row as $item) { if ( ! is_array($item)) { if (trim($item)) { return true; } } else { foreach ($item as $foo) { if (trim($foo)) { return true; } } } } } return false; }); $trimConverter = new CallbackItemConverter(function ($row) { foreach ($row as $key => $value) { $row[$key] = trim($row[$key]); } return $row; }); $lexicalConverter = new CallbackItemConverter(function ($row) { foreach ($row as $key => $value) { if (isset( $this->prolog['columns'][$key]['id'] ) and $this->prolog['columns'][$key]['id'] == 27) { $lang = $this->prolog['columns'][$key]['lang']; //check for existing language suffix if ($value and ! preg_match("/\\." . $lang . "$/u", $value)) { $row[$key] = $value . "." . $lang; } } } return $row; }); $typeConverter = new MappingValueConverter([ 'rdfs:class' => 'class', 'rdfs:property' => 'property', 'class' => 'class', 'property' => 'property', 'Class' => 'class', 'Property' => 'property', 'subclass' => 'class', 'subproperty' => 'property', '' => '', ]); $vocabWriter = new Writer\CallbackWriter(function ($row) { $this->setPrologColumns(); //todo: $row[59] (status) and $row[62] (uri) sre hardcoded but the array key should be set somewhere globally if ( ! isset( $row[59] )) { $row[59] = $this->prolog['defaults']['statusId']; } $rowStatus = $row[59]; $language = $this->prolog['defaults']['lang']; foreach ($row as $key => &$element) { $this->updateRowUris($key, $element); } $uri = $this->getFqn($row[62]); $property = null; if ( ! empty( $row['reg_id'] )) { $property = \ConceptPeer::retrieveByPK($row['reg_id']); } else { //check for an existing property by uri /** @var \Concept $property */ $property = \ConceptPeer::getConceptByUri($uri); } //even if we found a property, we kill it if it's in a different schema than we're populating if ($property and $property->getVocabularyId() !== $this->vocabId) { //todo: we should log this event unset( $property ); } if (empty( $property )) { //it's a new property $property = new \Concept(); $property->setVocabularyId($this->vocabId); $property->setCreatedUserId($this->userId); $property->setUpdatedUserId($this->userId); $property->setStatusId($rowStatus); $property->setLanguage($language); $property->setUri($uri); $property->save(); } unset( $row['reg_id'] ); if ($property) { // if (8 == $rowStatus) { // //it's been deprecated and we don't do anything else // $property->setStatusId($rowStatus); // $this->updateElement($element, $dbElement, $property); // } else { $dbElements = $property->getElementsForImport($this->profileProperties); foreach ($dbElements as $key => $dbElement) { /** @var string | array $rowElement */ $rowElement = isset( $row[$key] ) ? $row[$key] : null; if (is_array($rowElement)) { foreach ($rowElement as $elementKey => &$element) { if ($this->updateConceptProperty($element, $dbElement, $property)) { unset( $rowElement[$elementKey] ); } } } else { if ($this->updateConceptProperty($rowElement, $dbElement, $property)) { unset( $row[$key] ); } } } foreach ($row as $key => $value) { $dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null; if ( ! empty( $this->prolog['columns'][$key]['property'] )) { $profileProperty = $this->prolog['columns'][$key]['property']; if (is_array($value)) { foreach ($value as &$oneValue) { $language = $this->prolog['columns'][$key]['lang'][0]; $this->upsertConceptFromRow($dbElement, $oneValue, $rowStatus, $property, $profileProperty, $language, $key); } } else { $language = $this->prolog['columns'][$key]['lang']; $this->upsertConceptFromRow($dbElement, $value, $rowStatus, $property, $profileProperty, $language, $key); } } } } $property->setUri($uri); $property->setStatusId($rowStatus); $affectedRows = $property->save(); if (empty( $property->getPrefLabel() )) { $prefLabel = \ConceptPropertyPeer::lookupProperty($property->getId(), 19, $this->prolog['defaults']['lang']); if ( ! empty( $prefLabel )) { $property->setPrefLabel($prefLabel->getObject()); $property->setPrefLabelId($prefLabel->getId()); $property->setLanguage($this->prolog['defaults']['lang']); $property->save(); } } return; //************** // CRUFT ALERT // The rest of this is never run!! //************** //build an array of references $newElements = [ ]; $newElements2 = [ ]; if ( ! isset( $row['status'] )) { $row[14] = $this->prolog['defaults']['statusId']; } foreach ($row as $key => $element) { //skip it there's no property id $columnKey = $this->prolog['columns'][$key]; if ( ! $columnKey['id']) { continue; } if ( ! empty( $columnKey['type'] ) and $this->useCuries) { $element = $this->getFqn($element); } $key2 = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element); $newElements[$key2] = [ ]; $newElements[$key2] += $columnKey; $newElements[$key2]['val'] = $element; /** @var \ProfileProperty $profileProperty */ if (isset( $columnKey['property'] )) { $profileProperty = $columnKey['property']; $var = [ 'matchkey' => $key2, 'val' => $newElements[$key2], ]; if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) { $newElements2[$columnKey['id']][$columnKey['lang']][] = $var; } else { $newElements2[$columnKey['id']][] = $var; } } } if ( ! empty( $row['reg_id'] )) { $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']); if ($property) { $dbElements = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty(); $dbElements2 = [ ]; /** @var \SchemaPropertyElement $dbElement */ foreach ($dbElements as $dbElement) { if ($dbElement->getProfileProperty()->getHasLanguage()) { $dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement; } else { $dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement; } } /** @var \SchemaPropertyElement $element */ foreach ($dbElements as $element) { $language = $element->getLanguage(); $profilePropertyId = $element->getProfilePropertyId(); $key = md5(strval($profilePropertyId) . strval($language) . $element->getObject()); //if the newelement key matches then if (isset( $newElements[$key] )) { if ($element->getProfileProperty()->getHasLanguage()) { $newElements2Array = $newElements2[$profilePropertyId][$language]; } else { $newElements2Array = $newElements2[$profilePropertyId]; } $count = count($newElements2Array); for ($I = 0; $I < $count; $I++) { if ($newElements2Array[$I]['matchkey'] == $key) { unset( $newElements2Array[$I] ); } } unset( $newElements[$key] ); $element->importStatus = 'match'; continue; } else { if ($element->getProfileProperty()->getHasLanguage()) { if (isset( $newElements2[$profilePropertyId][$language] )) { $count = count($newElements2[$profilePropertyId][$language]); for ($I = 0; $I < $count; $I++) { if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) { unset( $newElements2[$profilePropertyId][$language][$I] ); $element->importStatus = 'match'; if ( ! count($newElements2[$profilePropertyId][$language])) { unset( $newElements2[$profilePropertyId][$language] ); } continue; } } } } else { //compare the old values with the new with the same key $count = count($newElements2[$profilePropertyId]); for ($I = 0; $I < $count; $I++) { if (isset( $newElements2[$profilePropertyId][$I] )) { if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) { unset( $newElements2[$profilePropertyId][$I] ); $element->importStatus = 'match'; continue; } } } } //if the key matches then //if the value matches //delete the newElement //else the value doesn't match //if the newElement value is empty //delete the dbElement } $element->matchKey = $key; } //update the property values $property->save(); } else { //there's no existing property an we have to create a new one $property = new \SchemaProperty(); } foreach ($newElements as $key => $newElement) { if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) { $profileProperty = $newElement['property']; //walk the old elements looking for a match on predicate + language /** @var \SchemaPropertyElement[] $oldElement */ foreach ($dbElements as $oldElement) { /** @var \SchemaPropertyElement $oldOne */ $oldOne = &$oldElement['element']; if ($newElement['id'] == $oldOne->getProfilePropertyId()) { /** @var \ProfileProperty $profileProperty */ if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) { if ( ! empty( $newElement['val'] )) { $oldOne->setObject($newElement['val']); $oldOne->setUpdatedUserId($this->userId); $oldOne->setStatusId($row['status']); //$oldOne->save(); $oldElement['status'] = "updated"; } else { $oldOne->delete(); $oldElement['status'] = "deleted"; } //update the property value if ($profileProperty->getIsInForm()) { $this->setPropertyValue($newElement['val'], $property, $profileProperty->getName(), ! $profileProperty->getIsObjectProp()); } break; } } } //we looked through them all, add a new one if ( ! empty( $newElement['val'] )) { $addMe = new \SchemaPropertyElement(); $addMe->setObject($newElement['val']); //$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property); $addMe->setCreatedUserId($this->userId); $addMe->setUpdatedUserId($this->userId); $addMe->setLanguage($newElement['lang']); $addMe->setProfilePropertyId($newElement['id']); $addMe->setStatusId($row['status']); $addMe->importId = $this->importId; //$addMe->save(); $property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe); //update the property value if ($profileProperty->getIsInForm()) { $this->setPropertyValue($newElement['val'], $property, $profileProperty->getName(), ! $profileProperty->getIsObjectProp()); } } } } //update the property if ($property) { $property->setStatusId($row['status']); $property->save(); } } //var_dump($row); }); $schemaWriter = new Writer\CallbackWriter(function ($row) { $this->setPrologColumns(); if ( ! isset( $row[14] )) { $row[14] = $this->prolog['defaults']['statusId']; } $rowStatus = $row[14]; $language = $this->prolog['defaults']['lang']; foreach ($row as $key => &$element) { $this->updateRowUris($key, $element); } $uri = $row[13]; $property = null; $schemaId = $this->vocabId; if ( ! empty( $row['reg_id'] )) { $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']); } else { //check for an existing property by uri /** @var \SchemaProperty $property */ $property = \SchemaPropertyPeer::retrieveByUri($uri); if ($property) { $schemaId = $property->getSchemaId(); } } //even if we found a property, we kill it if it's in a different schema than we're populating if ($property and $schemaId !== $this->vocabId) { //todo: we should log this event unset( $property ); } if (empty( $property )) { //it's a new property $property = new \SchemaProperty(); $property->setSchemaId($this->vocabId); $property->setCreatedUserId($this->userId); $property->setUpdatedUserId($this->userId); $property->setStatusId($rowStatus); $property->setLanguage($language); $property->save(); } unset( $row['reg_id'] ); if ($property) { // if (8 == $rowStatus) { // //it's been deprecated and we don't do anything else // $property->setStatusId($rowStatus); // $this->updateElement($element, $dbElement, $property); // } else { $dbElements = $property->getElementsForImport($this->profileProperties); foreach ($dbElements as $key => $dbElement) { /** @var string | array $rowElement */ $rowElement = isset( $row[$key] ) ? $row[$key] : null; if (is_array($rowElement)) { foreach ($rowElement as $elementKey => &$element) { if ($this->updateElement($element, $dbElement, $property)) { unset( $rowElement[$elementKey] ); } } } else { if ($this->updateElement($rowElement, $dbElement, $property)) { unset( $row[$key] ); } } } foreach ($row as $key => $value) { $dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null; if ( ! empty( $this->prolog['columns'][$key]['property'] )) { $profileProperty = $this->prolog['columns'][$key]['property']; if (is_array($value)) { foreach ($value as &$oneValue) { $language = $this->prolog['columns'][$key]['lang'][0]; $this->upsertElementFromRow($dbElement, $oneValue, $rowStatus, $property, $profileProperty, $language, $key); } } else { $language = $this->prolog['columns'][$key]['lang']; $this->upsertElementFromRow($dbElement, $value, $rowStatus, $property, $profileProperty, $language, $key); } if ($key == 'parent_class' and strtolower($property->getType()) == 'class' and $row[$key] != $property->getParentUri()) { $property->setParentUri($row[$key]); //we'll set this later $property->setIsSubpropertyOf(null); } if ($key == 'parent_property' and strtolower($property->getType()) == 'property' and $row[$key] != $property->getParentUri()) { $property->setParentUri($row[$key]); //we'll set this later $property->setIsSubpropertyOf(null); } } } } $affectedRows = $property->save(); // } return; //build an array of references $newElements = [ ]; $newElements2 = [ ]; if ( ! isset( $row['status'] )) { $row[14] = $this->prolog['defaults']['statusId']; } foreach ($row as $key => $element) { //skip it there's no property id $columnKey = $this->prolog['columns'][$key]; if ( ! $columnKey['id']) { continue; } if ( ! empty( $columnKey['type'] ) and $this->useCuries) { $element = $this->getFqn($element); } $key2 = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element); $newElements[$key2] = [ ]; $newElements[$key2] += $columnKey; $newElements[$key2]['val'] = $element; /** @var \ProfileProperty $profileProperty */ if (isset( $columnKey['property'] )) { $profileProperty = $columnKey['property']; $var = [ 'matchkey' => $key2, 'val' => $newElements[$key2], ]; if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) { $newElements2[$columnKey['id']][$columnKey['lang']][] = $var; } else { $newElements2[$columnKey['id']][] = $var; } } } if ( ! empty( $row['reg_id'] )) { $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']); if ($property) { $dbElements = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty(); $dbElements2 = [ ]; /** @var \SchemaPropertyElement $dbElement */ foreach ($dbElements as $dbElement) { if ($dbElement->getProfileProperty()->getHasLanguage()) { $dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement; } else { $dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement; } } /** @var \SchemaPropertyElement $element */ foreach ($dbElements as $element) { $language = $element->getLanguage(); $profilePropertyId = $element->getProfilePropertyId(); $key = md5(strval($profilePropertyId) . strval($language) . $element->getObject()); //if the newelement key matches then if (isset( $newElements[$key] )) { if ($element->getProfileProperty()->getHasLanguage()) { $newElements2Array = $newElements2[$profilePropertyId][$language]; } else { $newElements2Array = $newElements2[$profilePropertyId]; } $count = count($newElements2Array); for ($I = 0; $I < $count; $I++) { if ($newElements2Array[$I]['matchkey'] == $key) { unset( $newElements2Array[$I] ); } } unset( $newElements[$key] ); $element->importStatus = 'match'; continue; } else { if ($element->getProfileProperty()->getHasLanguage()) { if (isset( $newElements2[$profilePropertyId][$language] )) { $count = count($newElements2[$profilePropertyId][$language]); for ($I = 0; $I < $count; $I++) { if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) { unset( $newElements2[$profilePropertyId][$language][$I] ); $element->importStatus = 'match'; if ( ! count($newElements2[$profilePropertyId][$language])) { unset( $newElements2[$profilePropertyId][$language] ); } continue; } } } } else { //compare the old values with the new with the same key $count = count($newElements2[$profilePropertyId]); for ($I = 0; $I < $count; $I++) { if (isset( $newElements2[$profilePropertyId][$I] )) { if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) { unset( $newElements2[$profilePropertyId][$I] ); $element->importStatus = 'match'; continue; } } } } //if the key matches then //if the value matches //delete the newElement //else the value doesn't match //if the newElement value is empty //delete the dbElement } $element->matchKey = $key; } //update the property values $property->save(); } else { //there's no existing property an we have to create a new one $property = new \SchemaProperty(); } foreach ($newElements as $key => $newElement) { if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) { $profileProperty = $newElement['property']; //walk the old elements looking for a match on predicate + language /** @var \SchemaPropertyElement[] $oldElement */ foreach ($dbElements as $oldElement) { /** @var \SchemaPropertyElement $oldOne */ $oldOne = &$oldElement['element']; if ($newElement['id'] == $oldOne->getProfilePropertyId()) { /** @var \ProfileProperty $profileProperty */ if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) { if ( ! empty( $newElement['val'] )) { $oldOne->setObject($newElement['val']); $oldOne->setUpdatedUserId($this->userId); $oldOne->setStatusId($row['status']); //$oldOne->save(); $oldElement['status'] = "updated"; } else { $oldOne->delete(); $oldElement['status'] = "deleted"; } //update the property value if ($profileProperty->getIsInForm()) { $this->setPropertyValue($newElement['val'], $property, $profileProperty->getName(), ! $profileProperty->getIsObjectProp()); } break; } } } //we looked through them all, add a new one if ( ! empty( $newElement['val'] )) { $addMe = new \SchemaPropertyElement(); $addMe->setObject($newElement['val']); //$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property); $addMe->setCreatedUserId($this->userId); $addMe->setUpdatedUserId($this->userId); $addMe->setLanguage($newElement['lang']); $addMe->setProfilePropertyId($newElement['id']); $addMe->setStatusId($row['status']); $addMe->importId = $this->importId; //$addMe->save(); $property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe); //update the property value if ($profileProperty->getIsInForm()) { $this->setPropertyValue($newElement['val'], $property, $profileProperty->getName(), ! $profileProperty->getIsObjectProp()); } } } } //update the property if ($property) { $property->setStatusId($row['status']); $property->save(); } } //var_dump($row); }); $workflow->addItemConverter($this->mapping); $workflow->addFilter($filter); $workflow->addItemConverter($trimConverter); $workflow->addItemConverter($lexicalConverter); $workflow->addWriter(new Writer\ConsoleProgressWriter($output, $this->reader)); $workflow->addValueConverter("4", $typeConverter); //add a database writer if ('schema' == $this->type) { $workflow->addWriter($schemaWriter); } else { $workflow->addWriter($vocabWriter); } /** @todo we need to make a second pass through to delete missing rows * for each schemaproperty in the database * match to a row in the csv * if the row value is empty and $this->deleteMissing * delete the entire schemaproperty */ $workResults = $workflow->process(); $this->DataWorkflowResults = $workResults; /** @TODO need to make a second pass through to lookup and set $element->related_schema_property_id */ $this->processParents($this->vocabId); //use the prolog to configure namespaces, look up correct resources in the database //store the row number of the first non-meta line return $workResults; }