Ejemplo n.º 1
0
    /**
     * @return \Ddeboer\DataImport\Result
     * @throws \Ddeboer\DataImport\Exception\ExceptionInterface
     * @throws \Exception
     */
    public function processData()
    {
        $workflow = new Workflow($this->reader);
        $output   = new ConsoleOutput();
        // Don’t import the non-metadata
        $filter = new Filter\CallbackFilter(function ($row) {
            if (is_numeric($row['reg_id'])) {
                return true;
            }
            if ( ! trim($row['reg_id'])) {
                foreach ($row as $item) {
                    if ( ! is_array($item)) {
                        if (trim($item)) {
                            return true;
                        }
                    } else {
                        foreach ($item as $foo) {
                            if (trim($foo)) {
                                return true;
                            }
                        }
                    }
                }
            }

            return false;
        });

        $trimConverter = new CallbackItemConverter(function ($row) {
            foreach ($row as $key => $value) {
                $row[$key] = trim($row[$key]);
            }

            return $row;
        });

        $lexicalConverter = new CallbackItemConverter(function ($row) {
            foreach ($row as $key => $value) {
                if (isset( $this->prolog['columns'][$key]['id'] ) and $this->prolog['columns'][$key]['id'] == 27) {
                    $lang = $this->prolog['columns'][$key]['lang'];
                    //check for existing language suffix
                    if ($value and ! preg_match("/\\." . $lang . "$/u", $value)) {
                        $row[$key] = $value . "." . $lang;
                    }
                }
            }

            return $row;
        });

        $typeConverter = new MappingValueConverter([
                                                       'rdfs:class'    => 'class',
                                                       'rdfs:property' => 'property',
                                                       'class'         => 'class',
                                                       'property'      => 'property',
                                                       'Class'         => 'class',
                                                       'Property'      => 'property',
                                                       'subclass'      => 'class',
                                                       'subproperty'   => 'property',
                                                       ''              => '',
                                                   ]);

        $vocabWriter  = new Writer\CallbackWriter(function ($row) {
            $this->setPrologColumns();

            //todo: $row[59] (status) and $row[62] (uri) sre hardcoded but the array key should be set somewhere globally
            if ( ! isset( $row[59] )) {
                $row[59] = $this->prolog['defaults']['statusId'];
            }
            $rowStatus = $row[59];
            $language  = $this->prolog['defaults']['lang'];

            foreach ($row as $key => &$element) {
                $this->updateRowUris($key, $element);
            }

            $uri      = $this->getFqn($row[62]);
            $property = null;

            if ( ! empty( $row['reg_id'] )) {
                $property = \ConceptPeer::retrieveByPK($row['reg_id']);
            } else {
                //check for an existing property by uri
                /** @var \Concept $property */
                $property = \ConceptPeer::getConceptByUri($uri);
            }

            //even if we found a property, we kill it if it's in a different schema than we're populating
            if ($property and $property->getVocabularyId() !== $this->vocabId) {
                //todo: we should log this event
                unset( $property );
            }

            if (empty( $property )) { //it's a new property
                $property = new \Concept();
                $property->setVocabularyId($this->vocabId);
                $property->setCreatedUserId($this->userId);
                $property->setUpdatedUserId($this->userId);
                $property->setStatusId($rowStatus);
                $property->setLanguage($language);
                $property->setUri($uri);
                $property->save();
            }

            unset( $row['reg_id'] );

            if ($property) {
                //        if (8 == $rowStatus) {
//          //it's been deprecated and we don't do anything else
//          $property->setStatusId($rowStatus);
//          $this->updateElement($element, $dbElement, $property);
//        } else {
                $dbElements = $property->getElementsForImport($this->profileProperties);
                foreach ($dbElements as $key => $dbElement) {
                    /** @var string | array $rowElement */
                    $rowElement = isset( $row[$key] ) ? $row[$key] : null;
                    if (is_array($rowElement)) {
                        foreach ($rowElement as $elementKey => &$element) {
                            if ($this->updateConceptProperty($element, $dbElement, $property)) {
                                unset( $rowElement[$elementKey] );
                            }
                        }
                    } else {
                        if ($this->updateConceptProperty($rowElement, $dbElement, $property)) {
                            unset( $row[$key] );
                        }
                    }
                }
                foreach ($row as $key => $value) {
                    $dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null;
                    if ( ! empty( $this->prolog['columns'][$key]['property'] )) {
                        $profileProperty = $this->prolog['columns'][$key]['property'];
                        if (is_array($value)) {
                            foreach ($value as &$oneValue) {
                                $language = $this->prolog['columns'][$key]['lang'][0];
                                $this->upsertConceptFromRow($dbElement,
                                                            $oneValue,
                                                            $rowStatus,
                                                            $property,
                                                            $profileProperty,
                                                            $language,
                                                            $key);
                            }
                        } else {
                            $language = $this->prolog['columns'][$key]['lang'];
                            $this->upsertConceptFromRow($dbElement,
                                                        $value,
                                                        $rowStatus,
                                                        $property,
                                                        $profileProperty,
                                                        $language,
                                                        $key);
                        }
                    }
                }
            }

            $property->setUri($uri);
            $property->setStatusId($rowStatus);

            $affectedRows = $property->save();

            if (empty( $property->getPrefLabel() )) {
                $prefLabel = \ConceptPropertyPeer::lookupProperty($property->getId(),
                                                                  19,
                                                                  $this->prolog['defaults']['lang']);
                if ( ! empty( $prefLabel )) {
                    $property->setPrefLabel($prefLabel->getObject());
                    $property->setPrefLabelId($prefLabel->getId());
                    $property->setLanguage($this->prolog['defaults']['lang']);
                    $property->save();
                }
            }

            return;

            //**************
            // CRUFT ALERT
            // The rest of this is never run!!
            //**************

            //build an array of references
            $newElements  = [ ];
            $newElements2 = [ ];
            if ( ! isset( $row['status'] )) {
                $row[14] = $this->prolog['defaults']['statusId'];
            }
            foreach ($row as $key => $element) {
                //skip it there's no property id
                $columnKey = $this->prolog['columns'][$key];
                if ( ! $columnKey['id']) {
                    continue;
                }

                if ( ! empty( $columnKey['type'] ) and $this->useCuries) {
                    $element = $this->getFqn($element);
                }

                $key2               = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element);
                $newElements[$key2] = [ ];
                $newElements[$key2] += $columnKey;
                $newElements[$key2]['val'] = $element;
                /** @var \ProfileProperty $profileProperty */
                if (isset( $columnKey['property'] )) {
                    $profileProperty = $columnKey['property'];
                    $var             = [
                        'matchkey' => $key2,
                        'val'      => $newElements[$key2],
                    ];
                    if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) {
                        $newElements2[$columnKey['id']][$columnKey['lang']][] = $var;
                    } else {
                        $newElements2[$columnKey['id']][] = $var;
                    }
                }
            }
            if ( ! empty( $row['reg_id'] )) {
                $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
                if ($property) {
                    $dbElements  = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty();
                    $dbElements2 = [ ];
                    /** @var \SchemaPropertyElement $dbElement */
                    foreach ($dbElements as $dbElement) {
                        if ($dbElement->getProfileProperty()->getHasLanguage()) {
                            $dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement;
                        } else {
                            $dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement;
                        }
                    }

                    /** @var \SchemaPropertyElement $element */
                    foreach ($dbElements as $element) {
                        $language          = $element->getLanguage();
                        $profilePropertyId = $element->getProfilePropertyId();
                        $key               = md5(strval($profilePropertyId) . strval($language) . $element->getObject());
                        //if the newelement key matches then
                        if (isset( $newElements[$key] )) {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                $newElements2Array = $newElements2[$profilePropertyId][$language];
                            } else {
                                $newElements2Array = $newElements2[$profilePropertyId];
                            }
                            $count = count($newElements2Array);
                            for ($I = 0; $I < $count; $I++) {
                                if ($newElements2Array[$I]['matchkey'] == $key) {
                                    unset( $newElements2Array[$I] );
                                }
                            }
                            unset( $newElements[$key] );
                            $element->importStatus = 'match';
                            continue;
                        } else {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                if (isset( $newElements2[$profilePropertyId][$language] )) {
                                    $count = count($newElements2[$profilePropertyId][$language]);
                                    for ($I = 0; $I < $count; $I++) {
                                        if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$language][$I] );
                                            $element->importStatus = 'match';
                                            if ( ! count($newElements2[$profilePropertyId][$language])) {
                                                unset( $newElements2[$profilePropertyId][$language] );
                                            }
                                            continue;
                                        }
                                    }
                                }
                            } else {
                                //compare the old values with the new with the same key
                                $count = count($newElements2[$profilePropertyId]);
                                for ($I = 0; $I < $count; $I++) {
                                    if (isset( $newElements2[$profilePropertyId][$I] )) {
                                        if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$I] );
                                            $element->importStatus = 'match';
                                            continue;
                                        }
                                    }
                                }
                            }
                            //if the key matches then
                            //if the value matches
                            //delete the newElement
                            //else the value doesn't match
                            //if the newElement value is empty
                            //delete the dbElement
                        }

                        $element->matchKey = $key;
                    }
                    //update the property values
                    $property->save();
                } else {
                    //there's no existing property an we have to create a new one
                    $property = new \SchemaProperty();
                }
                foreach ($newElements as $key => $newElement) {
                    if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) {
                        $profileProperty = $newElement['property'];
                        //walk the old elements looking for a match on predicate + language
                        /** @var \SchemaPropertyElement[] $oldElement */
                        foreach ($dbElements as $oldElement) {
                            /** @var \SchemaPropertyElement $oldOne */
                            $oldOne = &$oldElement['element'];
                            if ($newElement['id'] == $oldOne->getProfilePropertyId()) {
                                /** @var \ProfileProperty $profileProperty */
                                if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) {
                                    if ( ! empty( $newElement['val'] )) {
                                        $oldOne->setObject($newElement['val']);
                                        $oldOne->setUpdatedUserId($this->userId);
                                        $oldOne->setStatusId($row['status']);
                                        //$oldOne->save();
                                        $oldElement['status'] = "updated";
                                    } else {
                                        $oldOne->delete();
                                        $oldElement['status'] = "deleted";
                                    }
                                    //update the property value
                                    if ($profileProperty->getIsInForm()) {
                                        $this->setPropertyValue($newElement['val'],
                                                                $property,
                                                                $profileProperty->getName(),
                                                                ! $profileProperty->getIsObjectProp());
                                    }
                                    break;
                                }
                            }
                        }
                        //we looked through them all, add a new one
                        if ( ! empty( $newElement['val'] )) {
                            $addMe = new \SchemaPropertyElement();
                            $addMe->setObject($newElement['val']);
                            //$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property);
                            $addMe->setCreatedUserId($this->userId);
                            $addMe->setUpdatedUserId($this->userId);
                            $addMe->setLanguage($newElement['lang']);
                            $addMe->setProfilePropertyId($newElement['id']);
                            $addMe->setStatusId($row['status']);
                            $addMe->importId = $this->importId;
                            //$addMe->save();
                            $property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe);
                            //update the property value
                            if ($profileProperty->getIsInForm()) {
                                $this->setPropertyValue($newElement['val'],
                                                        $property,
                                                        $profileProperty->getName(),
                                                        ! $profileProperty->getIsObjectProp());
                            }
                        }
                    }
                }
                //update the property
                if ($property) {
                    $property->setStatusId($row['status']);
                    $property->save();
                }
            }
            //var_dump($row);
        });
        $schemaWriter = new Writer\CallbackWriter(function ($row) {
            $this->setPrologColumns();

            if ( ! isset( $row[14] )) {
                $row[14] = $this->prolog['defaults']['statusId'];
            }
            $rowStatus = $row[14];
            $language  = $this->prolog['defaults']['lang'];

            foreach ($row as $key => &$element) {
                $this->updateRowUris($key, $element);
            }

            $uri      = $row[13];
            $property = null;
            $schemaId = $this->vocabId;

            if ( ! empty( $row['reg_id'] )) {
                $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
            } else {
                //check for an existing property by uri
                /** @var \SchemaProperty $property */
                $property = \SchemaPropertyPeer::retrieveByUri($uri);
                if ($property) {
                    $schemaId = $property->getSchemaId();
                }
            }

            //even if we found a property, we kill it if it's in a different schema than we're populating
            if ($property and $schemaId !== $this->vocabId) {
                //todo: we should log this event
                unset( $property );
            }

            if (empty( $property )) { //it's a new property
                $property = new \SchemaProperty();
                $property->setSchemaId($this->vocabId);
                $property->setCreatedUserId($this->userId);
                $property->setUpdatedUserId($this->userId);
                $property->setStatusId($rowStatus);
                $property->setLanguage($language);
                $property->save();
            }

            unset( $row['reg_id'] );

            if ($property) {
                //        if (8 == $rowStatus) {
//          //it's been deprecated and we don't do anything else
//          $property->setStatusId($rowStatus);
//          $this->updateElement($element, $dbElement, $property);
//        } else {
                $dbElements = $property->getElementsForImport($this->profileProperties);
                foreach ($dbElements as $key => $dbElement) {
                    /** @var string | array $rowElement */
                    $rowElement = isset( $row[$key] ) ? $row[$key] : null;
                    if (is_array($rowElement)) {
                        foreach ($rowElement as $elementKey => &$element) {
                            if ($this->updateElement($element, $dbElement, $property)) {
                                unset( $rowElement[$elementKey] );
                            }
                        }
                    } else {
                        if ($this->updateElement($rowElement, $dbElement, $property)) {
                            unset( $row[$key] );
                        }
                    }
                }
                foreach ($row as $key => $value) {
                    $dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null;
                    if ( ! empty( $this->prolog['columns'][$key]['property'] )) {
                        $profileProperty = $this->prolog['columns'][$key]['property'];
                        if (is_array($value)) {
                            foreach ($value as &$oneValue) {
                                $language = $this->prolog['columns'][$key]['lang'][0];
                                $this->upsertElementFromRow($dbElement,
                                                            $oneValue,
                                                            $rowStatus,
                                                            $property,
                                                            $profileProperty,
                                                            $language,
                                                            $key);
                            }
                        } else {
                            $language = $this->prolog['columns'][$key]['lang'];
                            $this->upsertElementFromRow($dbElement,
                                                        $value,
                                                        $rowStatus,
                                                        $property,
                                                        $profileProperty,
                                                        $language,
                                                        $key);
                        }
                        if ($key == 'parent_class' and strtolower($property->getType()) == 'class' and $row[$key] != $property->getParentUri()) {
                            $property->setParentUri($row[$key]);
                            //we'll set this later
                            $property->setIsSubpropertyOf(null);
                        }
                        if ($key == 'parent_property' and strtolower($property->getType()) == 'property' and $row[$key] != $property->getParentUri()) {
                            $property->setParentUri($row[$key]);
                            //we'll set this later
                            $property->setIsSubpropertyOf(null);
                        }
                    }
                }
            }

            $affectedRows = $property->save();

//      }

            return;
            //build an array of references
            $newElements  = [ ];
            $newElements2 = [ ];
            if ( ! isset( $row['status'] )) {
                $row[14] = $this->prolog['defaults']['statusId'];
            }
            foreach ($row as $key => $element) {
                //skip it there's no property id
                $columnKey = $this->prolog['columns'][$key];
                if ( ! $columnKey['id']) {
                    continue;
                }

                if ( ! empty( $columnKey['type'] ) and $this->useCuries) {
                    $element = $this->getFqn($element);
                }

                $key2               = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element);
                $newElements[$key2] = [ ];
                $newElements[$key2] += $columnKey;
                $newElements[$key2]['val'] = $element;
                /** @var \ProfileProperty $profileProperty */
                if (isset( $columnKey['property'] )) {
                    $profileProperty = $columnKey['property'];
                    $var             = [
                        'matchkey' => $key2,
                        'val'      => $newElements[$key2],
                    ];
                    if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) {
                        $newElements2[$columnKey['id']][$columnKey['lang']][] = $var;
                    } else {
                        $newElements2[$columnKey['id']][] = $var;
                    }
                }
            }
            if ( ! empty( $row['reg_id'] )) {
                $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
                if ($property) {
                    $dbElements  = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty();
                    $dbElements2 = [ ];
                    /** @var \SchemaPropertyElement $dbElement */
                    foreach ($dbElements as $dbElement) {
                        if ($dbElement->getProfileProperty()->getHasLanguage()) {
                            $dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement;
                        } else {
                            $dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement;
                        }
                    }

                    /** @var \SchemaPropertyElement $element */
                    foreach ($dbElements as $element) {
                        $language          = $element->getLanguage();
                        $profilePropertyId = $element->getProfilePropertyId();
                        $key               = md5(strval($profilePropertyId) . strval($language) . $element->getObject());
                        //if the newelement key matches then
                        if (isset( $newElements[$key] )) {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                $newElements2Array = $newElements2[$profilePropertyId][$language];
                            } else {
                                $newElements2Array = $newElements2[$profilePropertyId];
                            }
                            $count = count($newElements2Array);
                            for ($I = 0; $I < $count; $I++) {
                                if ($newElements2Array[$I]['matchkey'] == $key) {
                                    unset( $newElements2Array[$I] );
                                }
                            }
                            unset( $newElements[$key] );
                            $element->importStatus = 'match';
                            continue;
                        } else {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                if (isset( $newElements2[$profilePropertyId][$language] )) {
                                    $count = count($newElements2[$profilePropertyId][$language]);
                                    for ($I = 0; $I < $count; $I++) {
                                        if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$language][$I] );
                                            $element->importStatus = 'match';
                                            if ( ! count($newElements2[$profilePropertyId][$language])) {
                                                unset( $newElements2[$profilePropertyId][$language] );
                                            }
                                            continue;
                                        }
                                    }
                                }
                            } else {
                                //compare the old values with the new with the same key
                                $count = count($newElements2[$profilePropertyId]);
                                for ($I = 0; $I < $count; $I++) {
                                    if (isset( $newElements2[$profilePropertyId][$I] )) {
                                        if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$I] );
                                            $element->importStatus = 'match';
                                            continue;
                                        }
                                    }
                                }
                            }
                            //if the key matches then
                            //if the value matches
                            //delete the newElement
                            //else the value doesn't match
                            //if the newElement value is empty
                            //delete the dbElement
                        }

                        $element->matchKey = $key;
                    }
                    //update the property values
                    $property->save();
                } else {
                    //there's no existing property an we have to create a new one
                    $property = new \SchemaProperty();
                }
                foreach ($newElements as $key => $newElement) {
                    if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) {
                        $profileProperty = $newElement['property'];
                        //walk the old elements looking for a match on predicate + language
                        /** @var \SchemaPropertyElement[] $oldElement */
                        foreach ($dbElements as $oldElement) {
                            /** @var \SchemaPropertyElement $oldOne */
                            $oldOne = &$oldElement['element'];
                            if ($newElement['id'] == $oldOne->getProfilePropertyId()) {
                                /** @var \ProfileProperty $profileProperty */
                                if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) {
                                    if ( ! empty( $newElement['val'] )) {
                                        $oldOne->setObject($newElement['val']);
                                        $oldOne->setUpdatedUserId($this->userId);
                                        $oldOne->setStatusId($row['status']);
                                        //$oldOne->save();
                                        $oldElement['status'] = "updated";
                                    } else {
                                        $oldOne->delete();
                                        $oldElement['status'] = "deleted";
                                    }
                                    //update the property value
                                    if ($profileProperty->getIsInForm()) {
                                        $this->setPropertyValue($newElement['val'],
                                                                $property,
                                                                $profileProperty->getName(),
                                                                ! $profileProperty->getIsObjectProp());
                                    }
                                    break;
                                }
                            }
                        }
                        //we looked through them all, add a new one
                        if ( ! empty( $newElement['val'] )) {
                            $addMe = new \SchemaPropertyElement();
                            $addMe->setObject($newElement['val']);
                            //$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property);
                            $addMe->setCreatedUserId($this->userId);
                            $addMe->setUpdatedUserId($this->userId);
                            $addMe->setLanguage($newElement['lang']);
                            $addMe->setProfilePropertyId($newElement['id']);
                            $addMe->setStatusId($row['status']);
                            $addMe->importId = $this->importId;
                            //$addMe->save();
                            $property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe);
                            //update the property value
                            if ($profileProperty->getIsInForm()) {
                                $this->setPropertyValue($newElement['val'],
                                                        $property,
                                                        $profileProperty->getName(),
                                                        ! $profileProperty->getIsObjectProp());
                            }
                        }
                    }
                }
                //update the property
                if ($property) {
                    $property->setStatusId($row['status']);
                    $property->save();
                }
            }
            //var_dump($row);
        });

        $workflow->addItemConverter($this->mapping);
        $workflow->addFilter($filter);
        $workflow->addItemConverter($trimConverter);
        $workflow->addItemConverter($lexicalConverter);
        $workflow->addWriter(new Writer\ConsoleProgressWriter($output, $this->reader));
        $workflow->addValueConverter("4", $typeConverter);
        //add a database writer
        if ('schema' == $this->type) {
            $workflow->addWriter($schemaWriter);
        } else {
            $workflow->addWriter($vocabWriter);
        }
        /** @todo we need to make a second pass through to delete missing rows
         * for each schemaproperty in the database
         *   match to a row in the csv
         *   if the row value is empty and $this->deleteMissing
         *     delete the entire schemaproperty
         */

        $workResults               = $workflow->process();
        $this->DataWorkflowResults = $workResults;

        /** @TODO need to make a second pass through to lookup and set $element->related_schema_property_id */
        $this->processParents($this->vocabId);
        //use the prolog to configure namespaces, look up correct resources in the database
        //store the row number of the first non-meta line

        return $workResults;
    }