/**
 * Set defaults
 *
 * @param  Concept $concept
 */
 public function setDefaults($concept)
 {
     $vocabObj = $this->getCurrentVocabulary();
     $vocabId = $vocabObj->getId();
     $concept->setVocabularyId($vocabId);
     $conceptParam = $this->getContext()->getRequest()->getParameter('concept');
     if (!$this->getContext()->getRequest()->getErrors() and !isset($conceptParam['uri'])) {
         $vocabDomain = $vocabObj->getBaseDomain();
         $vocabToken = $vocabObj->getToken();
         //get the next id
         $nextUriId = VocabularyPeer::getNextConceptId($vocabId);
         //URI looks like: agent(base_domain) / vocabulary(token) / vocabulary(next_concept_id) / skos_property_id # concept(next_property_id)
         $vSlash = preg_match('@(/$)@i', $vocabDomain) ? '' : '/';
         $tSlash = preg_match('@(/$)@i', $vocabToken) ? '' : '/';
         //to support hash URIs just a wee bit better...
         $tSlash = preg_match('/#$/', $vocabToken) ? '' : $tSlash;
         $newURI = $vocabDomain . $vSlash . $vocabToken . $tSlash . $nextUriId;
         //registry base domain is http://metadataregistry.org/uri/
         //next_concept_id is always initialized to 100000, allowing for 999,999 concepts
         //vocabulary carries denormalized base_domain from agent
         $concept->setUri($newURI);
         $concept->setprefLabel('');
         //set to the vocabulary defaults
         $concept->setLanguage($vocabObj->getLanguage());
         $concept->setStatusId($vocabObj->getStatusId());
     }
     parent::setDefaults($concept);
 }
/**
 * @param $task
 * @param $args
 *
 * Arg[0] is one of "schema" (element set), "vocab" or "vocabulary"
 * arg[1] is the vocabulary name.
 *        The file type is determined by the extension and must be one of "json", "rdf", "csv", "xml"
 * arg[2] is the vocabulary id
 * arg[3] is the batch id
 * arg[4] [optional] is -d
 *
 * @throws Exception
 */
function run_import_vocabulary($task, $args)
{
    //xdebug_break();

    //check the argument counts
    if (count($args) < 1) {
        throw new Exception('You must provide a vocabulary type.');
    }

    if (count($args) < 2) {
        throw new Exception('You must provide a file name.');
    }

    if (count($args) < 3) {
        throw new Exception('You must provide a vocabulary id.');
    }

    //set the arguments
    $type          = strtolower($args[0]);
    $filePath      = $args[1];
    $vocabId       = $args[2];
    $batchId       = isset($args[3]) ? $args[3] : "";
    $deleteMissing = (isset($args[4]) && ("-d" == $args[4]));

    //do some basic validity checks

    if (! in_array(
      $type,
      array(
        "schema",
        "vocab",
        "vocabulary"
      )
    )
    ) {
        throw new Exception('You must import into a schema or a vocab');
    }

    if ("vocabulary" == $type) {
        $type = "vocab";
    }

    if (! is_numeric($vocabId)) {
        throw new Exception('You must provide a valid ID');
    }

    //does the file exist?
    if (! file_exists($filePath)) {
        //default to the site upload path
        $filePath = $GLOBALS['uploadPath'] . $filePath;
        if (! file_exists($filePath)) {
        throw new Exception('You must supply a valid file to import: ' . $filePath);
        }
    }

    //is the file a valid type?
    if (preg_match('/^.+\.([[:alpha:]]{2,4})$/', $filePath, $matches)) {
        if (! in_array(
          strtolower($matches[1]),
          array(
            "json",
            "rdf",
            "csv",
            "xml"
          )
        )
        ) {
            throw new Exception('You must provide a valid file type based on the extension');
        }
    } else {
        throw new Exception("File type cannot be determined from the file extension");
    }

    $fileType = $matches[1];

    //is the object a valid object?
    if ('vocab' == $type) {
        $vocabObj = VocabularyPeer::retrieveByPK($vocabId);
        if (is_null($vocabObj)) {
            throw new Exception('Invalid vocabulary ID');
        }

        //set some defaults
        $baseDomain = $vocabObj->getBaseDomain();
        $language   = $vocabObj->getLanguage();
        $statusId   = $vocabObj->getStatusId();
        $userId     = $vocabObj->getCreatedUserId();
        $agentId    = $vocabObj->getAgentId();

        //get a skos property id map
        $skosMap = SkosPropertyPeer::getPropertyNames();

        //there has to be a hash or a slash
        $tSlash = preg_match('@(/$)@i', $vocabObj->getUri()) ? '' : '/';
        $tSlash = preg_match('/#$/', $vocabObj->getUri()) ? '' : $tSlash;
    } else {
        $import               = new ImportVocab($type, $filePath, $vocabId);
    }

    /* From here on the process is the same regardless of UI */
    //     check to see if file has been uploaded before
    //          check import history for file name
    $importHistory = FileImportHistoryPeer::retrieveByLastFilePath($filePath);
    //          if reimport
    //               get last import history for filename
    //               unserialize column map
    //               match column names to AP based on map
    //     look for matches in unmatched field/column names to AP (ideal)
    //     csv table of data --
    //          row1: parsed field names/column headers
    //          row2: select dropdown with available fields from object AP (pre-select known matches)
    //                each select identified by column number
    //          row3: display datatype of selected field (updated dynamically when field selected)
    //          row4-13: first 10 rows of parsed data from file
    //     require a column that can match to 'URI' (maybe we'll allow an algorithm later)
    //     require columns that are required by AP
    //     on reimport there should be a flag to 'delete missing properties' from the current data
    //     note: at some point there will be a reimport process that allows URI changing
    //          this will require that there be an OMR identifier embedded in the incoming data

    switch ($fileType) {
        case "csv":
            if ('vocab' == $type) {
                // Get array of heading names found
                $headings = $reader->getHeadings();
                $fields   = ConceptPeer::getFieldNames();

                //set the map
                //      $map[] = array("property" => "Uri", "column" => "URILocal");
                //      $map[] = array("property" => "prefLabel", "column" => "skos:prefLabel");
                //      $map[] = array("property" => "definition", "column" => "skos:definition");
                //      $map[] = array("property" => "notation", "column" => "skos:notation");
                //      $map[] = array("property" => "scopeNote", "column" => "skos:scopeNote");

                $map = array(
                  "uri"        => "URILocal",
                  "prefLabel"  => "skos:prefLabel",
                  "definition" => "skos:definition",
                  "notation"   => "skos:notation",
                  "scopeNote"  => "skos:scopeNote"
                );

                $rows = 0;

                //executeImport:

                //    serialize the column map
                try {
                    while ($row = $reader->getRow()) {
                        $rows ++;
                        //        lookup the URI (or the OMR ID if available) for a match
                        $uri        = $baseDomain . $row[$map["uri"]];
                        $concept    = ConceptPeer::getConceptByUri($uri);
                        $updateTime = time();
                        $language   = (isset($map['language'])) ? $row[$map['language']] : $vocabObj->getLanguage();

                        if (! $concept) {
                            //          create a new concept or element
                            $concept = new Concept();
                            $concept->setVocabulary($vocabObj);
                            $concept->setUri($uri);
                            /**
                             * @todo Need to handle updates for topconcept here, just like language
                             **/
                            $concept->setIsTopConcept(false);
                            $concept->updateFromRequest(
                                    $userId,
                                      fixEncoding(rtrim($row[$map['prefLabel']])),
                                      $language,
                                      $statusId
                            );
                        } //don't update the concept if the preflabel matches
                        else if ($row[$map['prefLabel']] != $concept->getPrefLabel()) {
                            $concept->updateFromRequest($userId, fixEncoding(rtrim($row[$map['prefLabel']])));
                        }

                        //there needs to be a language to lookup the properties unless it's an objectProperty
                        $rowLanguage = (isset($map['language'])) ? $row[$map['language']] : $concept->getLanguage();

                        foreach ($map as $key => $value) {
                            //we skip because we already did them
                            if (! in_array(
                              $key,
                              array(
                                'uri',
                                'prefLabel',
                                'language'
                              )
                            )
                            ) {
                                $skosId = $skosMap[$key];
                                //check to see if the property already exists
                                $property =
                                  ConceptPropertyPeer::lookupProperty($concept->getId(), $skosId, $rowLanguage);

                                //create a new property for each unmatched column
                                if (! empty($row[$value])) {
                                    if (! $property) {
                                        $property = new ConceptProperty();
                                        $property->setCreatedUserId($userId);
                                        $property->setConceptId($concept->getId());
                                        $property->setCreatedAt($updateTime);
                                        $property->setSkosPropertyId($skosId);
                                    }

                                    if (($row[$value] != $property->getObject()) ||
                                        ($rowLanguage != $property->getLanguage())
                                    ) {
                                        /**
                                         * @todo We need a check here for skos objectproperties and handle differently
                                         **/
                                        if ($rowLanguage != $property->getLanguage()) {
                                            $property->setLanguage($rowLanguage);
                                        }
                                        if ($row[$value] != $property->getObject()) {
                                            $property->setObject(fixEncoding(rtrim($row[$value])));
                                        }
                                        $property->setUpdatedUserId($userId);
                                        $property->setUpdatedAt($updateTime);
                                        $property->save();
                                    }
                                } //the row value is empty
                                else if ($deleteMissing && $property) {
                                    $property->delete();
                                }
                            }
                        }

                        //          else
                        //               lookup and update concept or element
                        //               lookup and update each property
                        //          update the history for each property, action is 'import', should be a single timestamp for all (this should be automatic)
                        //          if 'delete missing properties' is true
                        //               delete each existing, non-required property that wasn't updated by the import
                    }
                } catch(Exception $e) {
                    //          catch
                    //            if there's an error of any kind, write to error log and continue
                    echo "Error on row: " . $rows . ", " . $uri . "\n" . $e . "\n";
                    continue;
                }
                $objects = $vocabObj->countConcepts();
            } else //it's an element set
            {
                $import->setCsvReader($import->file);
                $import->processProlog();
                $import->getDataColumnIds();
                $import->processData();
                //todo: $results should be a class
                $results[$vocabId] = $import->results;
                $bacthId = $import->saveResults($batchId);
            }
            break;
        case "json":
            break;
        case "rdf":
            break;
        case "xml":
            break;
        default:
    }

    /* output to stdout*/
    //          number of objects imported (link to history, filtered on timestamp of import)
    echo " Rows imported: " . count($results[$vocabId]['success']['rows']) . "\n From File:" . $filePath . "\nUse this ID for more in this batch: " . $bacthId;
    //          number of errors (link to error log)

}
function run_import_marc_vocabulary($task, $args)
{


    //check the argument counts
    if (count($args) < 1) {
        throw new Exception('You must provide a vocabulary type.');
    }

    if (count($args) < 2) {
        throw new Exception('You must provide a file name.');
    }

    if (count($args) < 3) {
        throw new Exception('You must provide a vocabulary id.');
    }

    //set the arguments
    $type          = strtolower($args[0]);
    $filePath      = $args[1];
    $id            = $args[2];
    $deleteMissing = (isset($args[3]) && ("-d" == $args[3]));

    //do some basic validity checks

    if (! in_array($type, array("schema", "vocab", "vocabulary"))) {
        throw new Exception('You must import into a schema or a vocab');
    }

    if ("vocabulary" == $type) {
        $type = "vocab";
    }

    if (! is_numeric($id)) {
        throw new Exception('You must provide a valid ID');
    }

    //does the file exist?
    if (! file_exists($filePath)) {
        throw new Exception('You must supply a valid file to import: ' . $filePath);
    }

    //is the file a valid type?
    if (preg_match('/^.+\.([[:alpha:]]{2,4})$/', $filePath, $matches)) {
        if (! in_array(strtolower($matches[1]), array("json", "rdf", "csv", "xml"))) {
            throw new Exception('You must provide a valid file type based on the extension');
        }
    } else {
        throw new Exception("File type cannot be determined from the file extension");
    }

    $fileType = $matches[1];

    //is the object a valid object?
    if ('vocab' == $type) {
        $vocabObj = VocabularyPeer::retrieveByPK($id);
        if (is_null($vocabObj)) {
            throw new Exception('Invalid vocabulary ID');
        }

        //set some defaults
        $baseDomain = $vocabObj->getBaseDomain();
        $language   = $vocabObj->getLanguage();
        $statusId   = $vocabObj->getStatusId();

        //get a skos property id map
        $skosMap = SkosPropertyPeer::getPropertyNames();

        //there has to be a hash or a slash
        $tSlash = preg_match('@(/$)@i', $vocabObj->getUri()) ? '' : '/';
        $tSlash = preg_match('/#$/', $vocabObj->getUri()) ? '' : $tSlash;
    } else {
        $schemaObj = SchemaPeer::retrieveByPK($id);
        if (is_null($schemaObj)) {
            throw new Exception('Invalid schema ID');
        }

        //set some defaults
        $baseDomain = $schemaObj->getUri();
        $language   = $schemaObj->getLanguage();
        $statusId   = $schemaObj->getStatusId();

        //get a element set property id map
        $profileId  = 1;
        $profile    = ProfilePeer::retrieveByPK($profileId);
        $elementMap = $profile->getAllProperties();

        //there has to be a hash or a slash
        $tSlash = preg_match('@(/$)@i', $baseDomain) ? '' : '/';
        $tSlash = preg_match('/#$/', $baseDomain) ? '' : $tSlash;
    }

    //     insert jon's user id
    $userId = 36;

    /* From here on the process is the same regardless of UI */

    //execute
    //     parse file to get the fields/columns and data
    $file = fopen($filePath, "r");
    if (! $file) {
        throw new Exception("Can't read supplied file");
    }

    //     check to see if file has been uploaded before
    //          check import history for file name
    $importHistory = FileImportHistoryPeer::retrieveByLastFilePath($filePath);
    //          if reimport
    //               get last import history for filename
    //               unserialize column map
    //               match column names to AP based on map
    //     look for matches in unmatched field/column names to AP (ideal)
    //     csv table of data --
    //          row1: parsed field names/column headers
    //          row2: select dropdown with available fields from object AP (pre-select known matches)
    //                each select identified by column number
    //          row3: display datatype of selected field (updated dynamically when field selected)
    //          row4-13: first 10 rows of parsed data from file
    //     require a column that can match to 'URI' (maybe we'll allow an algorithm later)
    //     require columns that are required by AP
    //     on reimport there should be a flag to 'delete missing properties' from the current data
    //     note: at some point there will be a reimport process that allows URI changing
    //          this will require that there be an OMR identifier embedded in the incoming data

    switch ($fileType) {
        case "csv":
            try {
                $reader = new aCsvReader($filePath);
            } catch(Exception $e) {
                throw new Exception("Not a happy CSV file!");
            }

            if ('vocab' == $type) {
                // Get array of heading names found
                $headings = $reader->getHeadings();
                $fields   = ConceptPeer::getFieldNames();

                //set the map
                //      $map[] = array("property" => "Uri", "column" => "URILocal");
                //      $map[] = array("property" => "prefLabel", "column" => "skos:prefLabel");
                //      $map[] = array("property" => "definition", "column" => "skos:definition");
                //      $map[] = array("property" => "notation", "column" => "skos:notation");
                //      $map[] = array("property" => "scopeNote", "column" => "skos:scopeNote");

                $map = array(
                  "uri"        => "URILocal",
                  "prefLabel"  => "skos:prefLabel",
                  "definition" => "skos:definition",
                  "notation"   => "skos:notation",
                  "scopeNote"  => "skos:scopeNote"
                );

                $rows = 0;

                //executeImport:

                //    serialize the column map
                try {
                    while ($row = $reader->getRow()) {
                        $rows ++;
                        //        lookup the URI (or the OMR ID if available) for a match
                        $uri        = $baseDomain . $row[$map["uri"]];
                        $concept    = ConceptPeer::getConceptByUri($uri);
                        $updateTime = time();
                        $language   = (isset($map['language'])) ? $row[$map['language']] : $vocabObj->getLanguage();

                        if (! $concept) {
                            //          create a new concept or element
                            $concept = new Concept();
                            $concept->setVocabulary($vocabObj);
                            $concept->setUri($uri);
                            /**
                             * @todo Need to handle updates for topconcept here, just like language
                             **/
                            $concept->setIsTopConcept(false);
                            $concept->updateFromRequest(
                                    $userId,
                                      fixMarcEncoding(rtrim($row[$map['prefLabel']])),
                                      $language,
                                      $statusId
                            );
                        } //don't update the concept if the preflabel matches
                        else if ($row[$map['prefLabel']] != $concept->getPrefLabel()) {
                            $concept->updateFromRequest($userId, fixMarcEncoding(rtrim($row[$map['prefLabel']])));
                        }

                        //there needs to be a language to lookup the properties unless it's an objectProperty
                        $rowLanguage = (isset($map['language'])) ? $row[$map['language']] : $concept->getLanguage();

                        foreach ($map as $key => $value) {
                            //we skip because we already did them
                            if (! in_array($key, array('uri', 'prefLabel', 'language'))) {
                                $skosId = $skosMap[$key];
                                //check to see if the property already exists
                                $property =
                                  ConceptPropertyPeer::lookupProperty($concept->getId(), $skosId, $rowLanguage);

                                //create a new property for each unmatched column
                                if (! empty($row[$value])) {
                                    if (! $property) {
                                        $property = new ConceptProperty();
                                        $property->setCreatedUserId($userId);
                                        $property->setConceptId($concept->getId());
                                        $property->setCreatedAt($updateTime);
                                        $property->setSkosPropertyId($skosId);
                                    }

                                    if (($row[$value] != $property->getObject()) ||
                                        ($rowLanguage != $property->getLanguage())
                                    ) {
                                        /**
                                         * @todo We need a check here for skos objectproperties and handle differently
                                         **/
                                        if ($rowLanguage != $property->getLanguage()) {
                                            $property->setLanguage($rowLanguage);
                                        }
                                        if ($row[$value] != $property->getObject()) {
                                            $property->setObject(fixMarcEncoding(rtrim($row[$value])));
                                        }
                                        $property->setUpdatedUserId($userId);
                                        $property->setUpdatedAt($updateTime);
                                        $property->save();
                                    }
                                } //the row value is empty
                                else if ($deleteMissing && $property) {
                                    $property->delete();
                                }
                            }
                        }

                        //          else
                        //               lookup and update concept or element
                        //               lookup and update each property
                        //          update the history for each property, action is 'import', should be a single timestamp for all (this should be automatic)
                        //          if 'delete missing properties' is true
                        //               delete each existing, non-required property that wasn't updated by the import
                    }
                } catch(Exception $e) {
                    //          catch
                    //            if there's an error of any kind, write to error log and continue
                    echo "Error on row: " . $rows . ", " . $uri . "\n" . $e . "\n";
                    continue;
                }
                $objects = $vocabObj->countConcepts();
            } else //it's an element set
            {
                $map  = array(
                  "uri"        => "uriLocalPart",
                  "name"       => "reg:name",
                  "definition" => "skos:definition",
                  "label"      => "rdfs:label",
                  "note"       => array("tag" => "tagCap", "ind1" => "ind1Cap", "ind2" => "ind2Cap", "sub" => "subCap")
                );
                $rows = 0;

                //executeImport:
                //    serialize the column map
                try {
                    while ($row = $reader->getRow()) {
                        //        lookup the URI (or the OMR ID if available) for a match

                        //There always has to be a URI on either update or create
                        if (! isset($row[$map["uri"]])) {
                            throw new Exception('Missing URI for row: ' . $reader->getRowCount());
                            continue;
                        }

                        $rows ++;
                        $uri         = $baseDomain . $tSlash . $row[$map["uri"]];
                        $property    = SchemaPropertyPeer::retrieveByUri($uri);
                        $updateTime  = time();
                        $rowLanguage = (isset($map['language'])) ? $row[$map['language']] : $language;
                        $rowStatusId = (isset($map['status'])) ? $row[$map['status']] : $statusId;

                        if (! $property) {
                            //          create a new property
                            /** @var SchemaProperty * */
                            $property = new SchemaProperty();
                            $property->setSchema($schemaObj);
                            $property->setUri($uri);
                            $property->setCreatedUserId($userId);
                            $property->setCreatedAt($updateTime);
                        }

                        $property->setLanguage($rowLanguage);
                        $property->setStatusId($rowStatusId);
                        $property->setUpdatedUserId($userId);
                        $property->setUpdatedAt($updateTime);

                        if (isset($row[$map["label"]])) {
                            $property->setLabel($row[$map["label"]]);
                        }

                        if (isset($row[$map["name"]])) {
                            $property->setName($row[$map["name"]]);
                        }

                        if (isset($row[$map["definition"]])) {
                            $property->setDefinition($row[$map["definition"]]);
                        }

                        if (is_array($map["note"])) {
                            $note = '';
                            foreach ($map["note"] as $key => $value) {
                                $caption = ! empty($row[$value]) ? " (" . $row[$value] . ")" : ' (no caption)';
                                $note .= ! empty($row[$key]) ? $key . ": " . $row[$key] . $caption . "<br />" : "";
                            }
                            $property->setNote($note);
                        } else {
                            if (isset($row[$map["note"]])) {
                                $property->setNote($row[$map["note"]]);
                            }
                        }
                        $property->saveSchemaProperty($userId);

                        /**
                         * @todo Need to handle domain and range
                         **/

                        foreach ($map as $key => $value) {
                            //we skip because we already did them
                            if (! in_array(
                              $key,
                              array('uri', 'status', 'language', 'label', 'name', 'definition', 'comment', 'note')
                            )
                            ) {
                                $elementId = $elementMap[$key];
                                //check to see if the property already exists
                                //note that this also checks the object value as well, so there's no way to update or delete an existing triple
                                //the sheet would have to conatin the identifier for the triple
                                $element = SchemaPropertyElementPeer::lookupElement(
                                                                    $schemaObj->getId(),
                                                                      $elementId,
                                                                      $map[$value]
                                );

                                //create a new property for each unmatched column
                                if (! empty($row[$value])) {
                                    if (! $element) {
                                        $element = new SchemaPropertyElement();
                                        $element->setCreatedUserId($userId);
                                        $element->setCreatedAt($updateTime);
                                        $element->setProfilePropertyId($elementId);
                                    }

                                    if (($row[$value] != $element->getObject()) ||
                                        ($rowLanguage != $element->getLanguage())
                                    ) {
                                        /**
                                         * @todo We need a check here for objectproperties and handle differently
                                         **/
                                        if ($rowLanguage != $element->getLanguage()) {
                                            $element->setLanguage($rowLanguage);
                                        }
                                        if ($row[$value] != $element->getObject()) {
                                            $element->setObject($row[$value]);
                                        }
                                        $element->setUpdatedUserId($userId);
                                        $element->setUpdatedAt($updateTime);
                                        $element->save();
                                    }
                                } //the row value is empty
                                else if ($deleteMissing && $element) {
                                    $element->delete();
                                }
                            }
                        }

                        //          else
                        //               lookup and update concept or element
                        //               lookup and update each property
                        //          update the history for each property, action is 'import', should be a single timestamp for all (this should be automatic)
                        //          if 'delete missing properties' is true
                        //               delete each existing, non-required property that wasn't updated by the import
                    }
                } catch(Exception $e) {
                    //          catch
                    //            if there's an error of any kind, write to error log and continue
                    echo "Error on row: " . $rows . ", " . $uri . "\n" . $e . "\n";
                    continue;
                }
                $objects = $schemaObj->countSchemaPropertys();
            }
            //     save the import history file (match timestamp to history entries)
            break;
        case "json":
            break;
        case "rdf":
            break;
        case "xml":
            break;
        default:
    }

    /* output to stdout*/
    //          number of objects imported (link to history, filtered on timestamp of import)
    echo "File:" . $filePath . ";\n     Objects imported: " . $objects . "; Rows read: " . $rows . "\n";
    //          number of errors (link to error log)

}
    /**
     * @return \Ddeboer\DataImport\Result
     * @throws \Ddeboer\DataImport\Exception\ExceptionInterface
     * @throws \Exception
     */
    public function processData()
    {
        $workflow = new Workflow($this->reader);
        $output   = new ConsoleOutput();
        // Don’t import the non-metadata
        $filter = new Filter\CallbackFilter(function ($row) {
            if (is_numeric($row['reg_id'])) {
                return true;
            }
            if ( ! trim($row['reg_id'])) {
                foreach ($row as $item) {
                    if ( ! is_array($item)) {
                        if (trim($item)) {
                            return true;
                        }
                    } else {
                        foreach ($item as $foo) {
                            if (trim($foo)) {
                                return true;
                            }
                        }
                    }
                }
            }

            return false;
        });

        $trimConverter = new CallbackItemConverter(function ($row) {
            foreach ($row as $key => $value) {
                $row[$key] = trim($row[$key]);
            }

            return $row;
        });

        $lexicalConverter = new CallbackItemConverter(function ($row) {
            foreach ($row as $key => $value) {
                if (isset( $this->prolog['columns'][$key]['id'] ) and $this->prolog['columns'][$key]['id'] == 27) {
                    $lang = $this->prolog['columns'][$key]['lang'];
                    //check for existing language suffix
                    if ($value and ! preg_match("/\\." . $lang . "$/u", $value)) {
                        $row[$key] = $value . "." . $lang;
                    }
                }
            }

            return $row;
        });

        $typeConverter = new MappingValueConverter([
                                                       'rdfs:class'    => 'class',
                                                       'rdfs:property' => 'property',
                                                       'class'         => 'class',
                                                       'property'      => 'property',
                                                       'Class'         => 'class',
                                                       'Property'      => 'property',
                                                       'subclass'      => 'class',
                                                       'subproperty'   => 'property',
                                                       ''              => '',
                                                   ]);

        $vocabWriter  = new Writer\CallbackWriter(function ($row) {
            $this->setPrologColumns();

            //todo: $row[59] (status) and $row[62] (uri) sre hardcoded but the array key should be set somewhere globally
            if ( ! isset( $row[59] )) {
                $row[59] = $this->prolog['defaults']['statusId'];
            }
            $rowStatus = $row[59];
            $language  = $this->prolog['defaults']['lang'];

            foreach ($row as $key => &$element) {
                $this->updateRowUris($key, $element);
            }

            $uri      = $this->getFqn($row[62]);
            $property = null;

            if ( ! empty( $row['reg_id'] )) {
                $property = \ConceptPeer::retrieveByPK($row['reg_id']);
            } else {
                //check for an existing property by uri
                /** @var \Concept $property */
                $property = \ConceptPeer::getConceptByUri($uri);
            }

            //even if we found a property, we kill it if it's in a different schema than we're populating
            if ($property and $property->getVocabularyId() !== $this->vocabId) {
                //todo: we should log this event
                unset( $property );
            }

            if (empty( $property )) { //it's a new property
                $property = new \Concept();
                $property->setVocabularyId($this->vocabId);
                $property->setCreatedUserId($this->userId);
                $property->setUpdatedUserId($this->userId);
                $property->setStatusId($rowStatus);
                $property->setLanguage($language);
                $property->setUri($uri);
                $property->save();
            }

            unset( $row['reg_id'] );

            if ($property) {
                //        if (8 == $rowStatus) {
//          //it's been deprecated and we don't do anything else
//          $property->setStatusId($rowStatus);
//          $this->updateElement($element, $dbElement, $property);
//        } else {
                $dbElements = $property->getElementsForImport($this->profileProperties);
                foreach ($dbElements as $key => $dbElement) {
                    /** @var string | array $rowElement */
                    $rowElement = isset( $row[$key] ) ? $row[$key] : null;
                    if (is_array($rowElement)) {
                        foreach ($rowElement as $elementKey => &$element) {
                            if ($this->updateConceptProperty($element, $dbElement, $property)) {
                                unset( $rowElement[$elementKey] );
                            }
                        }
                    } else {
                        if ($this->updateConceptProperty($rowElement, $dbElement, $property)) {
                            unset( $row[$key] );
                        }
                    }
                }
                foreach ($row as $key => $value) {
                    $dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null;
                    if ( ! empty( $this->prolog['columns'][$key]['property'] )) {
                        $profileProperty = $this->prolog['columns'][$key]['property'];
                        if (is_array($value)) {
                            foreach ($value as &$oneValue) {
                                $language = $this->prolog['columns'][$key]['lang'][0];
                                $this->upsertConceptFromRow($dbElement,
                                                            $oneValue,
                                                            $rowStatus,
                                                            $property,
                                                            $profileProperty,
                                                            $language,
                                                            $key);
                            }
                        } else {
                            $language = $this->prolog['columns'][$key]['lang'];
                            $this->upsertConceptFromRow($dbElement,
                                                        $value,
                                                        $rowStatus,
                                                        $property,
                                                        $profileProperty,
                                                        $language,
                                                        $key);
                        }
                    }
                }
            }

            $property->setUri($uri);
            $property->setStatusId($rowStatus);

            $affectedRows = $property->save();

            if (empty( $property->getPrefLabel() )) {
                $prefLabel = \ConceptPropertyPeer::lookupProperty($property->getId(),
                                                                  19,
                                                                  $this->prolog['defaults']['lang']);
                if ( ! empty( $prefLabel )) {
                    $property->setPrefLabel($prefLabel->getObject());
                    $property->setPrefLabelId($prefLabel->getId());
                    $property->setLanguage($this->prolog['defaults']['lang']);
                    $property->save();
                }
            }

            return;

            //**************
            // CRUFT ALERT
            // The rest of this is never run!!
            //**************

            //build an array of references
            $newElements  = [ ];
            $newElements2 = [ ];
            if ( ! isset( $row['status'] )) {
                $row[14] = $this->prolog['defaults']['statusId'];
            }
            foreach ($row as $key => $element) {
                //skip it there's no property id
                $columnKey = $this->prolog['columns'][$key];
                if ( ! $columnKey['id']) {
                    continue;
                }

                if ( ! empty( $columnKey['type'] ) and $this->useCuries) {
                    $element = $this->getFqn($element);
                }

                $key2               = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element);
                $newElements[$key2] = [ ];
                $newElements[$key2] += $columnKey;
                $newElements[$key2]['val'] = $element;
                /** @var \ProfileProperty $profileProperty */
                if (isset( $columnKey['property'] )) {
                    $profileProperty = $columnKey['property'];
                    $var             = [
                        'matchkey' => $key2,
                        'val'      => $newElements[$key2],
                    ];
                    if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) {
                        $newElements2[$columnKey['id']][$columnKey['lang']][] = $var;
                    } else {
                        $newElements2[$columnKey['id']][] = $var;
                    }
                }
            }
            if ( ! empty( $row['reg_id'] )) {
                $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
                if ($property) {
                    $dbElements  = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty();
                    $dbElements2 = [ ];
                    /** @var \SchemaPropertyElement $dbElement */
                    foreach ($dbElements as $dbElement) {
                        if ($dbElement->getProfileProperty()->getHasLanguage()) {
                            $dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement;
                        } else {
                            $dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement;
                        }
                    }

                    /** @var \SchemaPropertyElement $element */
                    foreach ($dbElements as $element) {
                        $language          = $element->getLanguage();
                        $profilePropertyId = $element->getProfilePropertyId();
                        $key               = md5(strval($profilePropertyId) . strval($language) . $element->getObject());
                        //if the newelement key matches then
                        if (isset( $newElements[$key] )) {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                $newElements2Array = $newElements2[$profilePropertyId][$language];
                            } else {
                                $newElements2Array = $newElements2[$profilePropertyId];
                            }
                            $count = count($newElements2Array);
                            for ($I = 0; $I < $count; $I++) {
                                if ($newElements2Array[$I]['matchkey'] == $key) {
                                    unset( $newElements2Array[$I] );
                                }
                            }
                            unset( $newElements[$key] );
                            $element->importStatus = 'match';
                            continue;
                        } else {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                if (isset( $newElements2[$profilePropertyId][$language] )) {
                                    $count = count($newElements2[$profilePropertyId][$language]);
                                    for ($I = 0; $I < $count; $I++) {
                                        if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$language][$I] );
                                            $element->importStatus = 'match';
                                            if ( ! count($newElements2[$profilePropertyId][$language])) {
                                                unset( $newElements2[$profilePropertyId][$language] );
                                            }
                                            continue;
                                        }
                                    }
                                }
                            } else {
                                //compare the old values with the new with the same key
                                $count = count($newElements2[$profilePropertyId]);
                                for ($I = 0; $I < $count; $I++) {
                                    if (isset( $newElements2[$profilePropertyId][$I] )) {
                                        if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$I] );
                                            $element->importStatus = 'match';
                                            continue;
                                        }
                                    }
                                }
                            }
                            //if the key matches then
                            //if the value matches
                            //delete the newElement
                            //else the value doesn't match
                            //if the newElement value is empty
                            //delete the dbElement
                        }

                        $element->matchKey = $key;
                    }
                    //update the property values
                    $property->save();
                } else {
                    //there's no existing property an we have to create a new one
                    $property = new \SchemaProperty();
                }
                foreach ($newElements as $key => $newElement) {
                    if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) {
                        $profileProperty = $newElement['property'];
                        //walk the old elements looking for a match on predicate + language
                        /** @var \SchemaPropertyElement[] $oldElement */
                        foreach ($dbElements as $oldElement) {
                            /** @var \SchemaPropertyElement $oldOne */
                            $oldOne = &$oldElement['element'];
                            if ($newElement['id'] == $oldOne->getProfilePropertyId()) {
                                /** @var \ProfileProperty $profileProperty */
                                if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) {
                                    if ( ! empty( $newElement['val'] )) {
                                        $oldOne->setObject($newElement['val']);
                                        $oldOne->setUpdatedUserId($this->userId);
                                        $oldOne->setStatusId($row['status']);
                                        //$oldOne->save();
                                        $oldElement['status'] = "updated";
                                    } else {
                                        $oldOne->delete();
                                        $oldElement['status'] = "deleted";
                                    }
                                    //update the property value
                                    if ($profileProperty->getIsInForm()) {
                                        $this->setPropertyValue($newElement['val'],
                                                                $property,
                                                                $profileProperty->getName(),
                                                                ! $profileProperty->getIsObjectProp());
                                    }
                                    break;
                                }
                            }
                        }
                        //we looked through them all, add a new one
                        if ( ! empty( $newElement['val'] )) {
                            $addMe = new \SchemaPropertyElement();
                            $addMe->setObject($newElement['val']);
                            //$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property);
                            $addMe->setCreatedUserId($this->userId);
                            $addMe->setUpdatedUserId($this->userId);
                            $addMe->setLanguage($newElement['lang']);
                            $addMe->setProfilePropertyId($newElement['id']);
                            $addMe->setStatusId($row['status']);
                            $addMe->importId = $this->importId;
                            //$addMe->save();
                            $property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe);
                            //update the property value
                            if ($profileProperty->getIsInForm()) {
                                $this->setPropertyValue($newElement['val'],
                                                        $property,
                                                        $profileProperty->getName(),
                                                        ! $profileProperty->getIsObjectProp());
                            }
                        }
                    }
                }
                //update the property
                if ($property) {
                    $property->setStatusId($row['status']);
                    $property->save();
                }
            }
            //var_dump($row);
        });
        $schemaWriter = new Writer\CallbackWriter(function ($row) {
            $this->setPrologColumns();

            if ( ! isset( $row[14] )) {
                $row[14] = $this->prolog['defaults']['statusId'];
            }
            $rowStatus = $row[14];
            $language  = $this->prolog['defaults']['lang'];

            foreach ($row as $key => &$element) {
                $this->updateRowUris($key, $element);
            }

            $uri      = $row[13];
            $property = null;
            $schemaId = $this->vocabId;

            if ( ! empty( $row['reg_id'] )) {
                $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
            } else {
                //check for an existing property by uri
                /** @var \SchemaProperty $property */
                $property = \SchemaPropertyPeer::retrieveByUri($uri);
                if ($property) {
                    $schemaId = $property->getSchemaId();
                }
            }

            //even if we found a property, we kill it if it's in a different schema than we're populating
            if ($property and $schemaId !== $this->vocabId) {
                //todo: we should log this event
                unset( $property );
            }

            if (empty( $property )) { //it's a new property
                $property = new \SchemaProperty();
                $property->setSchemaId($this->vocabId);
                $property->setCreatedUserId($this->userId);
                $property->setUpdatedUserId($this->userId);
                $property->setStatusId($rowStatus);
                $property->setLanguage($language);
                $property->save();
            }

            unset( $row['reg_id'] );

            if ($property) {
                //        if (8 == $rowStatus) {
//          //it's been deprecated and we don't do anything else
//          $property->setStatusId($rowStatus);
//          $this->updateElement($element, $dbElement, $property);
//        } else {
                $dbElements = $property->getElementsForImport($this->profileProperties);
                foreach ($dbElements as $key => $dbElement) {
                    /** @var string | array $rowElement */
                    $rowElement = isset( $row[$key] ) ? $row[$key] : null;
                    if (is_array($rowElement)) {
                        foreach ($rowElement as $elementKey => &$element) {
                            if ($this->updateElement($element, $dbElement, $property)) {
                                unset( $rowElement[$elementKey] );
                            }
                        }
                    } else {
                        if ($this->updateElement($rowElement, $dbElement, $property)) {
                            unset( $row[$key] );
                        }
                    }
                }
                foreach ($row as $key => $value) {
                    $dbElement = isset( $dbElements[$key] ) ? $dbElements[$key] : null;
                    if ( ! empty( $this->prolog['columns'][$key]['property'] )) {
                        $profileProperty = $this->prolog['columns'][$key]['property'];
                        if (is_array($value)) {
                            foreach ($value as &$oneValue) {
                                $language = $this->prolog['columns'][$key]['lang'][0];
                                $this->upsertElementFromRow($dbElement,
                                                            $oneValue,
                                                            $rowStatus,
                                                            $property,
                                                            $profileProperty,
                                                            $language,
                                                            $key);
                            }
                        } else {
                            $language = $this->prolog['columns'][$key]['lang'];
                            $this->upsertElementFromRow($dbElement,
                                                        $value,
                                                        $rowStatus,
                                                        $property,
                                                        $profileProperty,
                                                        $language,
                                                        $key);
                        }
                        if ($key == 'parent_class' and strtolower($property->getType()) == 'class' and $row[$key] != $property->getParentUri()) {
                            $property->setParentUri($row[$key]);
                            //we'll set this later
                            $property->setIsSubpropertyOf(null);
                        }
                        if ($key == 'parent_property' and strtolower($property->getType()) == 'property' and $row[$key] != $property->getParentUri()) {
                            $property->setParentUri($row[$key]);
                            //we'll set this later
                            $property->setIsSubpropertyOf(null);
                        }
                    }
                }
            }

            $affectedRows = $property->save();

//      }

            return;
            //build an array of references
            $newElements  = [ ];
            $newElements2 = [ ];
            if ( ! isset( $row['status'] )) {
                $row[14] = $this->prolog['defaults']['statusId'];
            }
            foreach ($row as $key => $element) {
                //skip it there's no property id
                $columnKey = $this->prolog['columns'][$key];
                if ( ! $columnKey['id']) {
                    continue;
                }

                if ( ! empty( $columnKey['type'] ) and $this->useCuries) {
                    $element = $this->getFqn($element);
                }

                $key2               = md5(strval($columnKey['id']) . strval($columnKey['lang']) . $element);
                $newElements[$key2] = [ ];
                $newElements[$key2] += $columnKey;
                $newElements[$key2]['val'] = $element;
                /** @var \ProfileProperty $profileProperty */
                if (isset( $columnKey['property'] )) {
                    $profileProperty = $columnKey['property'];
                    $var             = [
                        'matchkey' => $key2,
                        'val'      => $newElements[$key2],
                    ];
                    if (isset( $profileProperty ) and $profileProperty->getHasLanguage()) {
                        $newElements2[$columnKey['id']][$columnKey['lang']][] = $var;
                    } else {
                        $newElements2[$columnKey['id']][] = $var;
                    }
                }
            }
            if ( ! empty( $row['reg_id'] )) {
                $property = \SchemaPropertyPeer::retrieveByPK($row['reg_id']);
                if ($property) {
                    $dbElements  = $property->getSchemaPropertyElementsRelatedBySchemaPropertyIdJoinProfileProperty();
                    $dbElements2 = [ ];
                    /** @var \SchemaPropertyElement $dbElement */
                    foreach ($dbElements as $dbElement) {
                        if ($dbElement->getProfileProperty()->getHasLanguage()) {
                            $dbElements2[$dbElement->getProfilePropertyId()][$dbElement->getLanguage()][] = &$dbElement;
                        } else {
                            $dbElements2[$dbElement->getProfilePropertyId()][] = &$dbElement;
                        }
                    }

                    /** @var \SchemaPropertyElement $element */
                    foreach ($dbElements as $element) {
                        $language          = $element->getLanguage();
                        $profilePropertyId = $element->getProfilePropertyId();
                        $key               = md5(strval($profilePropertyId) . strval($language) . $element->getObject());
                        //if the newelement key matches then
                        if (isset( $newElements[$key] )) {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                $newElements2Array = $newElements2[$profilePropertyId][$language];
                            } else {
                                $newElements2Array = $newElements2[$profilePropertyId];
                            }
                            $count = count($newElements2Array);
                            for ($I = 0; $I < $count; $I++) {
                                if ($newElements2Array[$I]['matchkey'] == $key) {
                                    unset( $newElements2Array[$I] );
                                }
                            }
                            unset( $newElements[$key] );
                            $element->importStatus = 'match';
                            continue;
                        } else {
                            if ($element->getProfileProperty()->getHasLanguage()) {
                                if (isset( $newElements2[$profilePropertyId][$language] )) {
                                    $count = count($newElements2[$profilePropertyId][$language]);
                                    for ($I = 0; $I < $count; $I++) {
                                        if ($newElements2[$profilePropertyId][$language][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$language][$I] );
                                            $element->importStatus = 'match';
                                            if ( ! count($newElements2[$profilePropertyId][$language])) {
                                                unset( $newElements2[$profilePropertyId][$language] );
                                            }
                                            continue;
                                        }
                                    }
                                }
                            } else {
                                //compare the old values with the new with the same key
                                $count = count($newElements2[$profilePropertyId]);
                                for ($I = 0; $I < $count; $I++) {
                                    if (isset( $newElements2[$profilePropertyId][$I] )) {
                                        if ($newElements2[$profilePropertyId][$I]['val']['val'] == $element->getObject()) {
                                            unset( $newElements2[$profilePropertyId][$I] );
                                            $element->importStatus = 'match';
                                            continue;
                                        }
                                    }
                                }
                            }
                            //if the key matches then
                            //if the value matches
                            //delete the newElement
                            //else the value doesn't match
                            //if the newElement value is empty
                            //delete the dbElement
                        }

                        $element->matchKey = $key;
                    }
                    //update the property values
                    $property->save();
                } else {
                    //there's no existing property an we have to create a new one
                    $property = new \SchemaProperty();
                }
                foreach ($newElements as $key => $newElement) {
                    if ( ! empty( $newElement['id'] ) and ! isset( $oldElements[$key] )) {
                        $profileProperty = $newElement['property'];
                        //walk the old elements looking for a match on predicate + language
                        /** @var \SchemaPropertyElement[] $oldElement */
                        foreach ($dbElements as $oldElement) {
                            /** @var \SchemaPropertyElement $oldOne */
                            $oldOne = &$oldElement['element'];
                            if ($newElement['id'] == $oldOne->getProfilePropertyId()) {
                                /** @var \ProfileProperty $profileProperty */
                                if (( $profileProperty->getHasLanguage() and $newElement['lang'] == $oldOne->getLanguage() ) or ! $profileProperty->getHasLanguage()) {
                                    if ( ! empty( $newElement['val'] )) {
                                        $oldOne->setObject($newElement['val']);
                                        $oldOne->setUpdatedUserId($this->userId);
                                        $oldOne->setStatusId($row['status']);
                                        //$oldOne->save();
                                        $oldElement['status'] = "updated";
                                    } else {
                                        $oldOne->delete();
                                        $oldElement['status'] = "deleted";
                                    }
                                    //update the property value
                                    if ($profileProperty->getIsInForm()) {
                                        $this->setPropertyValue($newElement['val'],
                                                                $property,
                                                                $profileProperty->getName(),
                                                                ! $profileProperty->getIsObjectProp());
                                    }
                                    break;
                                }
                            }
                        }
                        //we looked through them all, add a new one
                        if ( ! empty( $newElement['val'] )) {
                            $addMe = new \SchemaPropertyElement();
                            $addMe->setObject($newElement['val']);
                            //$addMe->setSchemaPropertyRelatedBySchemaPropertyId($property);
                            $addMe->setCreatedUserId($this->userId);
                            $addMe->setUpdatedUserId($this->userId);
                            $addMe->setLanguage($newElement['lang']);
                            $addMe->setProfilePropertyId($newElement['id']);
                            $addMe->setStatusId($row['status']);
                            $addMe->importId = $this->importId;
                            //$addMe->save();
                            $property->addSchemaPropertyElementRelatedBySchemaPropertyId($addMe);
                            //update the property value
                            if ($profileProperty->getIsInForm()) {
                                $this->setPropertyValue($newElement['val'],
                                                        $property,
                                                        $profileProperty->getName(),
                                                        ! $profileProperty->getIsObjectProp());
                            }
                        }
                    }
                }
                //update the property
                if ($property) {
                    $property->setStatusId($row['status']);
                    $property->save();
                }
            }
            //var_dump($row);
        });

        $workflow->addItemConverter($this->mapping);
        $workflow->addFilter($filter);
        $workflow->addItemConverter($trimConverter);
        $workflow->addItemConverter($lexicalConverter);
        $workflow->addWriter(new Writer\ConsoleProgressWriter($output, $this->reader));
        $workflow->addValueConverter("4", $typeConverter);
        //add a database writer
        if ('schema' == $this->type) {
            $workflow->addWriter($schemaWriter);
        } else {
            $workflow->addWriter($vocabWriter);
        }
        /** @todo we need to make a second pass through to delete missing rows
         * for each schemaproperty in the database
         *   match to a row in the csv
         *   if the row value is empty and $this->deleteMissing
         *     delete the entire schemaproperty
         */

        $workResults               = $workflow->process();
        $this->DataWorkflowResults = $workResults;

        /** @TODO need to make a second pass through to lookup and set $element->related_schema_property_id */
        $this->processParents($this->vocabId);
        //use the prolog to configure namespaces, look up correct resources in the database
        //store the row number of the first non-meta line

        return $workResults;
    }