public function executeList() { $idType = $this->getRequestParameter('IdType', null); $id = $this->getRequestParameter('id', null); if (!$idType) { //a current schema is required to be in the request URL myActionTools::requireSchemaFilter(); } else { $this->getRequest()->getParameterHolder()->set($idType, $id); } $schema = myActionTools::findCurrentSchema(); if ($schema) { $this->schema = $schema; $schemaId = $schema->getId(); } else { $import = FileImportHistoryPeer::retrieveByPK($id); if ($import) { $this->schema = $import->getSchema(); $schemaId = $this->schema->getId(); } } if (in_array($idType, array('schema_property_id', 'schema_property_element_id'))) { $property = myActionTools::findCurrentSchemaProperty(); if ($property) { $this->property = $property; $this->setFlash('hasProperty', true); $schemaId = $this->property->getSchemaId(); } } //get the versions array $c = new Criteria(); $c->add(SchemaHasVersionPeer::SCHEMA_ID, $schemaId); $versions = SchemaHasVersionPeer::doSelect($c); $this->setFlash('versions', $versions); parent::executeList(); }
/** * @param $task * @param $args * * Arg[0] is one of "schema" (element set), "vocab" or "vocabulary" * arg[1] is the vocabulary name. * The file type is determined by the extension and must be one of "json", "rdf", "csv", "xml" * arg[2] is the vocabulary id * arg[3] is the batch id * arg[4] [optional] is -d * * @throws Exception */ function run_import_vocabulary($task, $args) { //xdebug_break(); //check the argument counts if (count($args) < 1) { throw new Exception('You must provide a vocabulary type.'); } if (count($args) < 2) { throw new Exception('You must provide a file name.'); } if (count($args) < 3) { throw new Exception('You must provide a vocabulary id.'); } //set the arguments $type = strtolower($args[0]); $filePath = $args[1]; $vocabId = $args[2]; $batchId = isset($args[3]) ? $args[3] : ""; $deleteMissing = (isset($args[4]) && ("-d" == $args[4])); //do some basic validity checks if (! in_array( $type, array( "schema", "vocab", "vocabulary" ) ) ) { throw new Exception('You must import into a schema or a vocab'); } if ("vocabulary" == $type) { $type = "vocab"; } if (! is_numeric($vocabId)) { throw new Exception('You must provide a valid ID'); } //does the file exist? if (! file_exists($filePath)) { //default to the site upload path $filePath = $GLOBALS['uploadPath'] . $filePath; if (! file_exists($filePath)) { throw new Exception('You must supply a valid file to import: ' . $filePath); } } //is the file a valid type? if (preg_match('/^.+\.([[:alpha:]]{2,4})$/', $filePath, $matches)) { if (! in_array( strtolower($matches[1]), array( "json", "rdf", "csv", "xml" ) ) ) { throw new Exception('You must provide a valid file type based on the extension'); } } else { throw new Exception("File type cannot be determined from the file extension"); } $fileType = $matches[1]; //is the object a valid object? if ('vocab' == $type) { $vocabObj = VocabularyPeer::retrieveByPK($vocabId); if (is_null($vocabObj)) { throw new Exception('Invalid vocabulary ID'); } //set some defaults $baseDomain = $vocabObj->getBaseDomain(); $language = $vocabObj->getLanguage(); $statusId = $vocabObj->getStatusId(); $userId = $vocabObj->getCreatedUserId(); $agentId = $vocabObj->getAgentId(); //get a skos property id map $skosMap = SkosPropertyPeer::getPropertyNames(); //there has to be a hash or a slash $tSlash = preg_match('@(/$)@i', $vocabObj->getUri()) ? '' : '/'; $tSlash = preg_match('/#$/', $vocabObj->getUri()) ? '' : $tSlash; } else { $import = new ImportVocab($type, $filePath, $vocabId); } /* From here on the process is the same regardless of UI */ // check to see if file has been uploaded before // check import history for file name $importHistory = FileImportHistoryPeer::retrieveByLastFilePath($filePath); // if reimport // get last import history for filename // unserialize column map // match column names to AP based on map // look for matches in unmatched field/column names to AP (ideal) // csv table of data -- // row1: parsed field names/column headers // row2: select dropdown with available fields from object AP (pre-select known matches) // each select identified by column number // row3: display datatype of selected field (updated dynamically when field selected) // row4-13: first 10 rows of parsed data from file // require a column that can match to 'URI' (maybe we'll allow an algorithm later) // require columns that are required by AP // on reimport there should be a flag to 'delete missing properties' from the current data // note: at some point there will be a reimport process that allows URI changing // this will require that there be an OMR identifier embedded in the incoming data switch ($fileType) { case "csv": if ('vocab' == $type) { // Get array of heading names found $headings = $reader->getHeadings(); $fields = ConceptPeer::getFieldNames(); //set the map // $map[] = array("property" => "Uri", "column" => "URILocal"); // $map[] = array("property" => "prefLabel", "column" => "skos:prefLabel"); // $map[] = array("property" => "definition", "column" => "skos:definition"); // $map[] = array("property" => "notation", "column" => "skos:notation"); // $map[] = array("property" => "scopeNote", "column" => "skos:scopeNote"); $map = array( "uri" => "URILocal", "prefLabel" => "skos:prefLabel", "definition" => "skos:definition", "notation" => "skos:notation", "scopeNote" => "skos:scopeNote" ); $rows = 0; //executeImport: // serialize the column map try { while ($row = $reader->getRow()) { $rows ++; // lookup the URI (or the OMR ID if available) for a match $uri = $baseDomain . $row[$map["uri"]]; $concept = ConceptPeer::getConceptByUri($uri); $updateTime = time(); $language = (isset($map['language'])) ? $row[$map['language']] : $vocabObj->getLanguage(); if (! $concept) { // create a new concept or element $concept = new Concept(); $concept->setVocabulary($vocabObj); $concept->setUri($uri); /** * @todo Need to handle updates for topconcept here, just like language **/ $concept->setIsTopConcept(false); $concept->updateFromRequest( $userId, fixEncoding(rtrim($row[$map['prefLabel']])), $language, $statusId ); } //don't update the concept if the preflabel matches else if ($row[$map['prefLabel']] != $concept->getPrefLabel()) { $concept->updateFromRequest($userId, fixEncoding(rtrim($row[$map['prefLabel']]))); } //there needs to be a language to lookup the properties unless it's an objectProperty $rowLanguage = (isset($map['language'])) ? $row[$map['language']] : $concept->getLanguage(); foreach ($map as $key => $value) { //we skip because we already did them if (! in_array( $key, array( 'uri', 'prefLabel', 'language' ) ) ) { $skosId = $skosMap[$key]; //check to see if the property already exists $property = ConceptPropertyPeer::lookupProperty($concept->getId(), $skosId, $rowLanguage); //create a new property for each unmatched column if (! empty($row[$value])) { if (! $property) { $property = new ConceptProperty(); $property->setCreatedUserId($userId); $property->setConceptId($concept->getId()); $property->setCreatedAt($updateTime); $property->setSkosPropertyId($skosId); } if (($row[$value] != $property->getObject()) || ($rowLanguage != $property->getLanguage()) ) { /** * @todo We need a check here for skos objectproperties and handle differently **/ if ($rowLanguage != $property->getLanguage()) { $property->setLanguage($rowLanguage); } if ($row[$value] != $property->getObject()) { $property->setObject(fixEncoding(rtrim($row[$value]))); } $property->setUpdatedUserId($userId); $property->setUpdatedAt($updateTime); $property->save(); } } //the row value is empty else if ($deleteMissing && $property) { $property->delete(); } } } // else // lookup and update concept or element // lookup and update each property // update the history for each property, action is 'import', should be a single timestamp for all (this should be automatic) // if 'delete missing properties' is true // delete each existing, non-required property that wasn't updated by the import } } catch(Exception $e) { // catch // if there's an error of any kind, write to error log and continue echo "Error on row: " . $rows . ", " . $uri . "\n" . $e . "\n"; continue; } $objects = $vocabObj->countConcepts(); } else //it's an element set { $import->setCsvReader($import->file); $import->processProlog(); $import->getDataColumnIds(); $import->processData(); //todo: $results should be a class $results[$vocabId] = $import->results; $bacthId = $import->saveResults($batchId); } break; case "json": break; case "rdf": break; case "xml": break; default: } /* output to stdout*/ // number of objects imported (link to history, filtered on timestamp of import) echo " Rows imported: " . count($results[$vocabId]['success']['rows']) . "\n From File:" . $filePath . "\nUse this ID for more in this batch: " . $bacthId; // number of errors (link to error log) }
/** * If this collection has already been initialized with * an identical criteria, it returns the collection. * Otherwise if this Batch is new, it will return * an empty collection; or if this Batch has previously * been saved, it will retrieve related FileImportHistorys from storage. * * This method is protected by default in order to keep the public * api reasonable. You can provide public methods for those you * actually need in Batch. */ public function getFileImportHistorysJoinSchema($criteria = null, $con = null) { // include the Peer class include_once 'lib/model/om/BaseFileImportHistoryPeer.php'; if ($criteria === null) { $criteria = new Criteria(); } elseif ($criteria instanceof Criteria) { $criteria = clone $criteria; } if ($this->collFileImportHistorys === null) { if ($this->isNew()) { $this->collFileImportHistorys = array(); } else { $criteria->add(FileImportHistoryPeer::BATCH_ID, $this->getId()); $this->collFileImportHistorys = FileImportHistoryPeer::doSelectJoinSchema($criteria, $con); } } else { // the following code is to determine if a new query is // called for. If the criteria is the same as the last // one, just return the collection. $criteria->add(FileImportHistoryPeer::BATCH_ID, $this->getId()); if (!isset($this->lastFileImportHistoryCriteria) || !$this->lastFileImportHistoryCriteria->equals($criteria)) { $this->collFileImportHistorys = FileImportHistoryPeer::doSelectJoinSchema($criteria, $con); } } $this->lastFileImportHistoryCriteria = $criteria; return $this->collFileImportHistorys; }
/** * gets the current schema object * * @return schema current schema object */ public function getCurrentSchema() { $schema = false; if ($this->getRequestParameter('id')) { $this->file_import_history = FileImportHistoryPeer::retrieveByPk($this->getRequestParameter('id')); if (isset($this->file_import_history)) { $schema = $this->file_import_history->getSchema(); if ($this->file_import_history->getSchemaId()) { $schema = $this->file_import_history->getSchema(); } if ($this->file_import_history->getVocabularyId()) { $schema = $this->file_import_history->getVocabulary(); } } } if ($this->getRequestParameter('schema_id')) { $schema = SchemaPeer::retrieveByPk($this->getRequestParameter('schema_id')); $this->type = 'schema'; } if ($this->getRequestParameter('vocabulary_id')) { $schema = VocabularyPeer::retrieveByPk($this->getRequestParameter('vocabulary_id')); $this->type = 'vocabulary'; } if ($schema) { myActionTools::setLatestSchema($schema->getId()); } $this->forward404Unless($schema, 'No filter has been selected.'); $this->schema = $schema; $this->schemaID = $schema->getId(); return $schema; }
/** * Selects a collection of SchemaPropertyElementHistory objects pre-filled with all related objects except Status. * * @return array Array of SchemaPropertyElementHistory objects. * @throws PropelException Any exceptions caught during processing will be * rethrown wrapped into a PropelException. */ public static function doSelectJoinAllExceptStatus(Criteria $c, $con = null) { $c = clone $c; // Set the correct dbName if it has not been overridden // $c->getDbName() will return the same object if not set to another value // so == check is okay and faster if ($c->getDbName() == Propel::getDefaultDB()) { $c->setDbName(self::DATABASE_NAME); } SchemaPropertyElementHistoryPeer::addSelectColumns($c); $startcol2 = SchemaPropertyElementHistoryPeer::NUM_COLUMNS - SchemaPropertyElementHistoryPeer::NUM_LAZY_LOAD_COLUMNS + 1; UserPeer::addSelectColumns($c); $startcol3 = $startcol2 + UserPeer::NUM_COLUMNS; SchemaPropertyElementPeer::addSelectColumns($c); $startcol4 = $startcol3 + SchemaPropertyElementPeer::NUM_COLUMNS; SchemaPropertyPeer::addSelectColumns($c); $startcol5 = $startcol4 + SchemaPropertyPeer::NUM_COLUMNS; SchemaPeer::addSelectColumns($c); $startcol6 = $startcol5 + SchemaPeer::NUM_COLUMNS; ProfilePropertyPeer::addSelectColumns($c); $startcol7 = $startcol6 + ProfilePropertyPeer::NUM_COLUMNS; SchemaPropertyPeer::addSelectColumns($c); $startcol8 = $startcol7 + SchemaPropertyPeer::NUM_COLUMNS; FileImportHistoryPeer::addSelectColumns($c); $startcol9 = $startcol8 + FileImportHistoryPeer::NUM_COLUMNS; $c->addJoin(SchemaPropertyElementHistoryPeer::CREATED_USER_ID, UserPeer::ID); $c->addJoin(SchemaPropertyElementHistoryPeer::SCHEMA_PROPERTY_ELEMENT_ID, SchemaPropertyElementPeer::ID); $c->addJoin(SchemaPropertyElementHistoryPeer::SCHEMA_PROPERTY_ID, SchemaPropertyPeer::ID); $c->addJoin(SchemaPropertyElementHistoryPeer::SCHEMA_ID, SchemaPeer::ID); $c->addJoin(SchemaPropertyElementHistoryPeer::PROFILE_PROPERTY_ID, ProfilePropertyPeer::ID); $c->addJoin(SchemaPropertyElementHistoryPeer::RELATED_SCHEMA_PROPERTY_ID, SchemaPropertyPeer::ID); $c->addJoin(SchemaPropertyElementHistoryPeer::IMPORT_ID, FileImportHistoryPeer::ID); $rs = BasePeer::doSelect($c, $con); $results = array(); while ($rs->next()) { $omClass = SchemaPropertyElementHistoryPeer::getOMClass(); $cls = Propel::import($omClass); $obj1 = new $cls(); $obj1->hydrate($rs); $omClass = UserPeer::getOMClass(); $cls = Propel::import($omClass); $obj2 = new $cls(); $obj2->hydrate($rs, $startcol2); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj2 = $temp_obj1->getUser(); //CHECKME if ($temp_obj2->getPrimaryKey() === $obj2->getPrimaryKey()) { $newObject = false; $temp_obj2->addSchemaPropertyElementHistory($obj1); break; } } if ($newObject) { $obj2->initSchemaPropertyElementHistorys(); $obj2->addSchemaPropertyElementHistory($obj1); } $omClass = SchemaPropertyElementPeer::getOMClass(); $cls = Propel::import($omClass); $obj3 = new $cls(); $obj3->hydrate($rs, $startcol3); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj3 = $temp_obj1->getSchemaPropertyElement(); //CHECKME if ($temp_obj3->getPrimaryKey() === $obj3->getPrimaryKey()) { $newObject = false; $temp_obj3->addSchemaPropertyElementHistory($obj1); break; } } if ($newObject) { $obj3->initSchemaPropertyElementHistorys(); $obj3->addSchemaPropertyElementHistory($obj1); } $omClass = SchemaPropertyPeer::getOMClass(); $cls = Propel::import($omClass); $obj4 = new $cls(); $obj4->hydrate($rs, $startcol4); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj4 = $temp_obj1->getSchemaPropertyRelatedBySchemaPropertyId(); //CHECKME if ($temp_obj4->getPrimaryKey() === $obj4->getPrimaryKey()) { $newObject = false; $temp_obj4->addSchemaPropertyElementHistoryRelatedBySchemaPropertyId($obj1); break; } } if ($newObject) { $obj4->initSchemaPropertyElementHistorysRelatedBySchemaPropertyId(); $obj4->addSchemaPropertyElementHistoryRelatedBySchemaPropertyId($obj1); } $omClass = SchemaPeer::getOMClass(); $cls = Propel::import($omClass); $obj5 = new $cls(); $obj5->hydrate($rs, $startcol5); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj5 = $temp_obj1->getSchema(); //CHECKME if ($temp_obj5->getPrimaryKey() === $obj5->getPrimaryKey()) { $newObject = false; $temp_obj5->addSchemaPropertyElementHistory($obj1); break; } } if ($newObject) { $obj5->initSchemaPropertyElementHistorys(); $obj5->addSchemaPropertyElementHistory($obj1); } $omClass = ProfilePropertyPeer::getOMClass(); $cls = Propel::import($omClass); $obj6 = new $cls(); $obj6->hydrate($rs, $startcol6); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj6 = $temp_obj1->getProfileProperty(); //CHECKME if ($temp_obj6->getPrimaryKey() === $obj6->getPrimaryKey()) { $newObject = false; $temp_obj6->addSchemaPropertyElementHistory($obj1); break; } } if ($newObject) { $obj6->initSchemaPropertyElementHistorys(); $obj6->addSchemaPropertyElementHistory($obj1); } $omClass = SchemaPropertyPeer::getOMClass(); $cls = Propel::import($omClass); $obj7 = new $cls(); $obj7->hydrate($rs, $startcol7); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj7 = $temp_obj1->getSchemaPropertyRelatedByRelatedSchemaPropertyId(); //CHECKME if ($temp_obj7->getPrimaryKey() === $obj7->getPrimaryKey()) { $newObject = false; $temp_obj7->addSchemaPropertyElementHistoryRelatedByRelatedSchemaPropertyId($obj1); break; } } if ($newObject) { $obj7->initSchemaPropertyElementHistorysRelatedByRelatedSchemaPropertyId(); $obj7->addSchemaPropertyElementHistoryRelatedByRelatedSchemaPropertyId($obj1); } $omClass = FileImportHistoryPeer::getOMClass(); $cls = Propel::import($omClass); $obj8 = new $cls(); $obj8->hydrate($rs, $startcol8); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj8 = $temp_obj1->getFileImportHistory(); //CHECKME if ($temp_obj8->getPrimaryKey() === $obj8->getPrimaryKey()) { $newObject = false; $temp_obj8->addSchemaPropertyElementHistory($obj1); break; } } if ($newObject) { $obj8->initSchemaPropertyElementHistorys(); $obj8->addSchemaPropertyElementHistory($obj1); } $results[] = $obj1; } return $results; }
public function executeList() { $idType = $this->getRequestParameter('IdType', null); $id = $this->getRequestParameter('id', null); if (!$idType) { //a current vocabulary is required to be in the request URL myActionTools::requireVocabularyFilter(); } else { $this->getRequest()->getParameterHolder()->set($idType, $id); } if ($idType !== 'import_id') { $vocabulary = myActionTools::findCurrentVocabulary(); $this->vocabulary = $vocabulary; if (in_array($idType, array('concept_id', 'property_id'))) { $this->concept = myActionTools::findCurrentConcept(); $this->setFlash('hasConcept', true); } } else { $import = FileImportHistoryPeer::retrieveByPK($id); if ($import) { $vocabulary = $import->getVocabulary(); $this->vocabulary = $vocabulary; } } //get the versions array $c = new Criteria(); $c->add(VocabularyHasVersionPeer::VOCABULARY_ID, $vocabulary->getId()); $versions = VocabularyHasVersionPeer::doSelect($c); $this->setFlash('versions', $versions); parent::executeList(); }
$objectId = $file_import_history->getVocabularyId(); $showVocabularyBc = true; $showVocabularyImportBc = true; //$tab = 'vocabulary'; } } $title = ''; break; case 'importlist': $showBc = true; $tab = false; /** @var \FileImportHistory $file_import_history */ if (!isset($file_import_history)) { $id = 'show' == $action ? $sf_params->get('id') : $paramId; if ($id) { $file_import_history = FileImportHistoryPeer::retrieveByPK($id); } } if ($file_import_history) { if ($file_import_history->getSchemaId() and !isset($schema)) { $schema = $file_import_history->getSchema(); $objectId = $file_import_history->getSchemaId(); $showSchemaBc = true; $showSchemaImportBc = true; //$tab = 'schema'; } if ($file_import_history->getVocabularyId() and !isset($vocabulary)) { $vocabulary = $file_import_history->getVocabulary(); $objectId = $file_import_history->getVocabularyId(); $showVocabularyBc = true; $showVocabularyImportBc = true;
/** * Get the associated FileImportHistory object * * @param Connection Optional Connection object. * @return FileImportHistory The associated FileImportHistory object. * @throws PropelException */ public function getFileImportHistory($con = null) { if ($this->aFileImportHistory === null && $this->import_id !== null) { // include the related Peer class include_once 'lib/model/om/BaseFileImportHistoryPeer.php'; $this->aFileImportHistory = FileImportHistoryPeer::retrieveByPK($this->import_id, $con); /* The following can be used instead of the line above to guarantee the related object contains a reference to this object, but this level of coupling may be undesirable in many circumstances. As it can lead to a db query with many results that may never be used. $obj = FileImportHistoryPeer::retrieveByPK($this->import_id, $con); $obj->addFileImportHistorys($this); */ } return $this->aFileImportHistory; }
function run_import_marc_vocabulary($task, $args) { //check the argument counts if (count($args) < 1) { throw new Exception('You must provide a vocabulary type.'); } if (count($args) < 2) { throw new Exception('You must provide a file name.'); } if (count($args) < 3) { throw new Exception('You must provide a vocabulary id.'); } //set the arguments $type = strtolower($args[0]); $filePath = $args[1]; $id = $args[2]; $deleteMissing = (isset($args[3]) && ("-d" == $args[3])); //do some basic validity checks if (! in_array($type, array("schema", "vocab", "vocabulary"))) { throw new Exception('You must import into a schema or a vocab'); } if ("vocabulary" == $type) { $type = "vocab"; } if (! is_numeric($id)) { throw new Exception('You must provide a valid ID'); } //does the file exist? if (! file_exists($filePath)) { throw new Exception('You must supply a valid file to import: ' . $filePath); } //is the file a valid type? if (preg_match('/^.+\.([[:alpha:]]{2,4})$/', $filePath, $matches)) { if (! in_array(strtolower($matches[1]), array("json", "rdf", "csv", "xml"))) { throw new Exception('You must provide a valid file type based on the extension'); } } else { throw new Exception("File type cannot be determined from the file extension"); } $fileType = $matches[1]; //is the object a valid object? if ('vocab' == $type) { $vocabObj = VocabularyPeer::retrieveByPK($id); if (is_null($vocabObj)) { throw new Exception('Invalid vocabulary ID'); } //set some defaults $baseDomain = $vocabObj->getBaseDomain(); $language = $vocabObj->getLanguage(); $statusId = $vocabObj->getStatusId(); //get a skos property id map $skosMap = SkosPropertyPeer::getPropertyNames(); //there has to be a hash or a slash $tSlash = preg_match('@(/$)@i', $vocabObj->getUri()) ? '' : '/'; $tSlash = preg_match('/#$/', $vocabObj->getUri()) ? '' : $tSlash; } else { $schemaObj = SchemaPeer::retrieveByPK($id); if (is_null($schemaObj)) { throw new Exception('Invalid schema ID'); } //set some defaults $baseDomain = $schemaObj->getUri(); $language = $schemaObj->getLanguage(); $statusId = $schemaObj->getStatusId(); //get a element set property id map $profileId = 1; $profile = ProfilePeer::retrieveByPK($profileId); $elementMap = $profile->getAllProperties(); //there has to be a hash or a slash $tSlash = preg_match('@(/$)@i', $baseDomain) ? '' : '/'; $tSlash = preg_match('/#$/', $baseDomain) ? '' : $tSlash; } // insert jon's user id $userId = 36; /* From here on the process is the same regardless of UI */ //execute // parse file to get the fields/columns and data $file = fopen($filePath, "r"); if (! $file) { throw new Exception("Can't read supplied file"); } // check to see if file has been uploaded before // check import history for file name $importHistory = FileImportHistoryPeer::retrieveByLastFilePath($filePath); // if reimport // get last import history for filename // unserialize column map // match column names to AP based on map // look for matches in unmatched field/column names to AP (ideal) // csv table of data -- // row1: parsed field names/column headers // row2: select dropdown with available fields from object AP (pre-select known matches) // each select identified by column number // row3: display datatype of selected field (updated dynamically when field selected) // row4-13: first 10 rows of parsed data from file // require a column that can match to 'URI' (maybe we'll allow an algorithm later) // require columns that are required by AP // on reimport there should be a flag to 'delete missing properties' from the current data // note: at some point there will be a reimport process that allows URI changing // this will require that there be an OMR identifier embedded in the incoming data switch ($fileType) { case "csv": try { $reader = new aCsvReader($filePath); } catch(Exception $e) { throw new Exception("Not a happy CSV file!"); } if ('vocab' == $type) { // Get array of heading names found $headings = $reader->getHeadings(); $fields = ConceptPeer::getFieldNames(); //set the map // $map[] = array("property" => "Uri", "column" => "URILocal"); // $map[] = array("property" => "prefLabel", "column" => "skos:prefLabel"); // $map[] = array("property" => "definition", "column" => "skos:definition"); // $map[] = array("property" => "notation", "column" => "skos:notation"); // $map[] = array("property" => "scopeNote", "column" => "skos:scopeNote"); $map = array( "uri" => "URILocal", "prefLabel" => "skos:prefLabel", "definition" => "skos:definition", "notation" => "skos:notation", "scopeNote" => "skos:scopeNote" ); $rows = 0; //executeImport: // serialize the column map try { while ($row = $reader->getRow()) { $rows ++; // lookup the URI (or the OMR ID if available) for a match $uri = $baseDomain . $row[$map["uri"]]; $concept = ConceptPeer::getConceptByUri($uri); $updateTime = time(); $language = (isset($map['language'])) ? $row[$map['language']] : $vocabObj->getLanguage(); if (! $concept) { // create a new concept or element $concept = new Concept(); $concept->setVocabulary($vocabObj); $concept->setUri($uri); /** * @todo Need to handle updates for topconcept here, just like language **/ $concept->setIsTopConcept(false); $concept->updateFromRequest( $userId, fixMarcEncoding(rtrim($row[$map['prefLabel']])), $language, $statusId ); } //don't update the concept if the preflabel matches else if ($row[$map['prefLabel']] != $concept->getPrefLabel()) { $concept->updateFromRequest($userId, fixMarcEncoding(rtrim($row[$map['prefLabel']]))); } //there needs to be a language to lookup the properties unless it's an objectProperty $rowLanguage = (isset($map['language'])) ? $row[$map['language']] : $concept->getLanguage(); foreach ($map as $key => $value) { //we skip because we already did them if (! in_array($key, array('uri', 'prefLabel', 'language'))) { $skosId = $skosMap[$key]; //check to see if the property already exists $property = ConceptPropertyPeer::lookupProperty($concept->getId(), $skosId, $rowLanguage); //create a new property for each unmatched column if (! empty($row[$value])) { if (! $property) { $property = new ConceptProperty(); $property->setCreatedUserId($userId); $property->setConceptId($concept->getId()); $property->setCreatedAt($updateTime); $property->setSkosPropertyId($skosId); } if (($row[$value] != $property->getObject()) || ($rowLanguage != $property->getLanguage()) ) { /** * @todo We need a check here for skos objectproperties and handle differently **/ if ($rowLanguage != $property->getLanguage()) { $property->setLanguage($rowLanguage); } if ($row[$value] != $property->getObject()) { $property->setObject(fixMarcEncoding(rtrim($row[$value]))); } $property->setUpdatedUserId($userId); $property->setUpdatedAt($updateTime); $property->save(); } } //the row value is empty else if ($deleteMissing && $property) { $property->delete(); } } } // else // lookup and update concept or element // lookup and update each property // update the history for each property, action is 'import', should be a single timestamp for all (this should be automatic) // if 'delete missing properties' is true // delete each existing, non-required property that wasn't updated by the import } } catch(Exception $e) { // catch // if there's an error of any kind, write to error log and continue echo "Error on row: " . $rows . ", " . $uri . "\n" . $e . "\n"; continue; } $objects = $vocabObj->countConcepts(); } else //it's an element set { $map = array( "uri" => "uriLocalPart", "name" => "reg:name", "definition" => "skos:definition", "label" => "rdfs:label", "note" => array("tag" => "tagCap", "ind1" => "ind1Cap", "ind2" => "ind2Cap", "sub" => "subCap") ); $rows = 0; //executeImport: // serialize the column map try { while ($row = $reader->getRow()) { // lookup the URI (or the OMR ID if available) for a match //There always has to be a URI on either update or create if (! isset($row[$map["uri"]])) { throw new Exception('Missing URI for row: ' . $reader->getRowCount()); continue; } $rows ++; $uri = $baseDomain . $tSlash . $row[$map["uri"]]; $property = SchemaPropertyPeer::retrieveByUri($uri); $updateTime = time(); $rowLanguage = (isset($map['language'])) ? $row[$map['language']] : $language; $rowStatusId = (isset($map['status'])) ? $row[$map['status']] : $statusId; if (! $property) { // create a new property /** @var SchemaProperty * */ $property = new SchemaProperty(); $property->setSchema($schemaObj); $property->setUri($uri); $property->setCreatedUserId($userId); $property->setCreatedAt($updateTime); } $property->setLanguage($rowLanguage); $property->setStatusId($rowStatusId); $property->setUpdatedUserId($userId); $property->setUpdatedAt($updateTime); if (isset($row[$map["label"]])) { $property->setLabel($row[$map["label"]]); } if (isset($row[$map["name"]])) { $property->setName($row[$map["name"]]); } if (isset($row[$map["definition"]])) { $property->setDefinition($row[$map["definition"]]); } if (is_array($map["note"])) { $note = ''; foreach ($map["note"] as $key => $value) { $caption = ! empty($row[$value]) ? " (" . $row[$value] . ")" : ' (no caption)'; $note .= ! empty($row[$key]) ? $key . ": " . $row[$key] . $caption . "<br />" : ""; } $property->setNote($note); } else { if (isset($row[$map["note"]])) { $property->setNote($row[$map["note"]]); } } $property->saveSchemaProperty($userId); /** * @todo Need to handle domain and range **/ foreach ($map as $key => $value) { //we skip because we already did them if (! in_array( $key, array('uri', 'status', 'language', 'label', 'name', 'definition', 'comment', 'note') ) ) { $elementId = $elementMap[$key]; //check to see if the property already exists //note that this also checks the object value as well, so there's no way to update or delete an existing triple //the sheet would have to conatin the identifier for the triple $element = SchemaPropertyElementPeer::lookupElement( $schemaObj->getId(), $elementId, $map[$value] ); //create a new property for each unmatched column if (! empty($row[$value])) { if (! $element) { $element = new SchemaPropertyElement(); $element->setCreatedUserId($userId); $element->setCreatedAt($updateTime); $element->setProfilePropertyId($elementId); } if (($row[$value] != $element->getObject()) || ($rowLanguage != $element->getLanguage()) ) { /** * @todo We need a check here for objectproperties and handle differently **/ if ($rowLanguage != $element->getLanguage()) { $element->setLanguage($rowLanguage); } if ($row[$value] != $element->getObject()) { $element->setObject($row[$value]); } $element->setUpdatedUserId($userId); $element->setUpdatedAt($updateTime); $element->save(); } } //the row value is empty else if ($deleteMissing && $element) { $element->delete(); } } } // else // lookup and update concept or element // lookup and update each property // update the history for each property, action is 'import', should be a single timestamp for all (this should be automatic) // if 'delete missing properties' is true // delete each existing, non-required property that wasn't updated by the import } } catch(Exception $e) { // catch // if there's an error of any kind, write to error log and continue echo "Error on row: " . $rows . ", " . $uri . "\n" . $e . "\n"; continue; } $objects = $schemaObj->countSchemaPropertys(); } // save the import history file (match timestamp to history entries) break; case "json": break; case "rdf": break; case "xml": break; default: } /* output to stdout*/ // number of objects imported (link to history, filtered on timestamp of import) echo "File:" . $filePath . ";\n Objects imported: " . $objects . "; Rows read: " . $rows . "\n"; // number of errors (link to error log) }
/** * Retrieve multiple objects by pkey. * * @param array $pks List of primary keys * @param Connection $con the connection to use * @throws PropelException Any exceptions caught during processing will be * rethrown wrapped into a PropelException. */ public static function retrieveByPKs($pks, $con = null) { if ($con === null) { $con = Propel::getConnection(self::DATABASE_NAME); } $objs = null; if (empty($pks)) { $objs = array(); } else { $criteria = new Criteria(); $criteria->add(FileImportHistoryPeer::ID, $pks, Criteria::IN); $objs = FileImportHistoryPeer::doSelect($criteria, $con); } return $objs; }
/** * Selects a collection of ConceptPropertyHistory objects pre-filled with all related objects except User. * * @return array Array of ConceptPropertyHistory objects. * @throws PropelException Any exceptions caught during processing will be * rethrown wrapped into a PropelException. */ public static function doSelectJoinAllExceptUser(Criteria $c, $con = null) { $c = clone $c; // Set the correct dbName if it has not been overridden // $c->getDbName() will return the same object if not set to another value // so == check is okay and faster if ($c->getDbName() == Propel::getDefaultDB()) { $c->setDbName(self::DATABASE_NAME); } ConceptPropertyHistoryPeer::addSelectColumns($c); $startcol2 = ConceptPropertyHistoryPeer::NUM_COLUMNS - ConceptPropertyHistoryPeer::NUM_LAZY_LOAD_COLUMNS + 1; ConceptPropertyPeer::addSelectColumns($c); $startcol3 = $startcol2 + ConceptPropertyPeer::NUM_COLUMNS; ConceptPeer::addSelectColumns($c); $startcol4 = $startcol3 + ConceptPeer::NUM_COLUMNS; VocabularyPeer::addSelectColumns($c); $startcol5 = $startcol4 + VocabularyPeer::NUM_COLUMNS; SkosPropertyPeer::addSelectColumns($c); $startcol6 = $startcol5 + SkosPropertyPeer::NUM_COLUMNS; VocabularyPeer::addSelectColumns($c); $startcol7 = $startcol6 + VocabularyPeer::NUM_COLUMNS; ConceptPeer::addSelectColumns($c); $startcol8 = $startcol7 + ConceptPeer::NUM_COLUMNS; StatusPeer::addSelectColumns($c); $startcol9 = $startcol8 + StatusPeer::NUM_COLUMNS; FileImportHistoryPeer::addSelectColumns($c); $startcol10 = $startcol9 + FileImportHistoryPeer::NUM_COLUMNS; $c->addJoin(ConceptPropertyHistoryPeer::CONCEPT_PROPERTY_ID, ConceptPropertyPeer::ID); $c->addJoin(ConceptPropertyHistoryPeer::CONCEPT_ID, ConceptPeer::ID); $c->addJoin(ConceptPropertyHistoryPeer::VOCABULARY_ID, VocabularyPeer::ID); $c->addJoin(ConceptPropertyHistoryPeer::SKOS_PROPERTY_ID, SkosPropertyPeer::ID); $c->addJoin(ConceptPropertyHistoryPeer::SCHEME_ID, VocabularyPeer::ID); $c->addJoin(ConceptPropertyHistoryPeer::RELATED_CONCEPT_ID, ConceptPeer::ID); $c->addJoin(ConceptPropertyHistoryPeer::STATUS_ID, StatusPeer::ID); $c->addJoin(ConceptPropertyHistoryPeer::IMPORT_ID, FileImportHistoryPeer::ID); $rs = BasePeer::doSelect($c, $con); $results = array(); while ($rs->next()) { $omClass = ConceptPropertyHistoryPeer::getOMClass(); $cls = Propel::import($omClass); $obj1 = new $cls(); $obj1->hydrate($rs); $omClass = ConceptPropertyPeer::getOMClass(); $cls = Propel::import($omClass); $obj2 = new $cls(); $obj2->hydrate($rs, $startcol2); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj2 = $temp_obj1->getConceptProperty(); //CHECKME if ($temp_obj2->getPrimaryKey() === $obj2->getPrimaryKey()) { $newObject = false; $temp_obj2->addConceptPropertyHistory($obj1); break; } } if ($newObject) { $obj2->initConceptPropertyHistorys(); $obj2->addConceptPropertyHistory($obj1); } $omClass = ConceptPeer::getOMClass(); $cls = Propel::import($omClass); $obj3 = new $cls(); $obj3->hydrate($rs, $startcol3); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj3 = $temp_obj1->getConceptRelatedByConceptId(); //CHECKME if ($temp_obj3->getPrimaryKey() === $obj3->getPrimaryKey()) { $newObject = false; $temp_obj3->addConceptPropertyHistoryRelatedByConceptId($obj1); break; } } if ($newObject) { $obj3->initConceptPropertyHistorysRelatedByConceptId(); $obj3->addConceptPropertyHistoryRelatedByConceptId($obj1); } $omClass = VocabularyPeer::getOMClass(); $cls = Propel::import($omClass); $obj4 = new $cls(); $obj4->hydrate($rs, $startcol4); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj4 = $temp_obj1->getVocabularyRelatedByVocabularyId(); //CHECKME if ($temp_obj4->getPrimaryKey() === $obj4->getPrimaryKey()) { $newObject = false; $temp_obj4->addConceptPropertyHistoryRelatedByVocabularyId($obj1); break; } } if ($newObject) { $obj4->initConceptPropertyHistorysRelatedByVocabularyId(); $obj4->addConceptPropertyHistoryRelatedByVocabularyId($obj1); } $omClass = SkosPropertyPeer::getOMClass(); $cls = Propel::import($omClass); $obj5 = new $cls(); $obj5->hydrate($rs, $startcol5); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj5 = $temp_obj1->getSkosProperty(); //CHECKME if ($temp_obj5->getPrimaryKey() === $obj5->getPrimaryKey()) { $newObject = false; $temp_obj5->addConceptPropertyHistory($obj1); break; } } if ($newObject) { $obj5->initConceptPropertyHistorys(); $obj5->addConceptPropertyHistory($obj1); } $omClass = VocabularyPeer::getOMClass(); $cls = Propel::import($omClass); $obj6 = new $cls(); $obj6->hydrate($rs, $startcol6); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj6 = $temp_obj1->getVocabularyRelatedBySchemeId(); //CHECKME if ($temp_obj6->getPrimaryKey() === $obj6->getPrimaryKey()) { $newObject = false; $temp_obj6->addConceptPropertyHistoryRelatedBySchemeId($obj1); break; } } if ($newObject) { $obj6->initConceptPropertyHistorysRelatedBySchemeId(); $obj6->addConceptPropertyHistoryRelatedBySchemeId($obj1); } $omClass = ConceptPeer::getOMClass(); $cls = Propel::import($omClass); $obj7 = new $cls(); $obj7->hydrate($rs, $startcol7); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj7 = $temp_obj1->getConceptRelatedByRelatedConceptId(); //CHECKME if ($temp_obj7->getPrimaryKey() === $obj7->getPrimaryKey()) { $newObject = false; $temp_obj7->addConceptPropertyHistoryRelatedByRelatedConceptId($obj1); break; } } if ($newObject) { $obj7->initConceptPropertyHistorysRelatedByRelatedConceptId(); $obj7->addConceptPropertyHistoryRelatedByRelatedConceptId($obj1); } $omClass = StatusPeer::getOMClass(); $cls = Propel::import($omClass); $obj8 = new $cls(); $obj8->hydrate($rs, $startcol8); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj8 = $temp_obj1->getStatus(); //CHECKME if ($temp_obj8->getPrimaryKey() === $obj8->getPrimaryKey()) { $newObject = false; $temp_obj8->addConceptPropertyHistory($obj1); break; } } if ($newObject) { $obj8->initConceptPropertyHistorys(); $obj8->addConceptPropertyHistory($obj1); } $omClass = FileImportHistoryPeer::getOMClass(); $cls = Propel::import($omClass); $obj9 = new $cls(); $obj9->hydrate($rs, $startcol9); $newObject = true; for ($j = 0, $resCount = count($results); $j < $resCount; $j++) { $temp_obj1 = $results[$j]; $temp_obj9 = $temp_obj1->getFileImportHistory(); //CHECKME if ($temp_obj9->getPrimaryKey() === $obj9->getPrimaryKey()) { $newObject = false; $temp_obj9->addConceptPropertyHistory($obj1); break; } } if ($newObject) { $obj9->initConceptPropertyHistorys(); $obj9->addConceptPropertyHistory($obj1); } $results[] = $obj1; } return $results; }
public function perform($args) { list($schemaId, $filePath, $importId, $environment, $type) = $args; // Set up environment for this job define('SF_ROOT_DIR', realpath(dirname(__file__) . '/../../../..')); define('SF_APP', 'frontend'); define('SF_ENVIRONMENT', $environment); define('SF_DEBUG', false); //initialize composer require_once(SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'autoload.php'); // initialize symfony require_once(SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'apps' . DIRECTORY_SEPARATOR . SF_APP . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'config.php'); // initialize database manager $databaseManager = new \sfDatabaseManager(); $databaseManager->initialize(); $import = new ImportVocab($type, $filePath, $schemaId); try { $fileImportHistory = \FileImportHistoryPeer::retrieveByPK($importId); } catch (\PropelException $e) { //exit the job with an error throw $e; } try { if ('schema' == $type) { $schema = \SchemaPeer::retrieveByPK($schemaId); } else { $schema = \VocabularyPeer::retrieveByPK($schemaId); } } catch (\PropelException $e) { //exit the job with an error throw $e; } // Perform some job $import->importId = $importId; //todo update the prefixes table with prefixes //todo update the schema table with prefixes $schemaPrefixes = $schema->getPrefixes(); $countSchemaPrefixes = count($schemaPrefixes); /** @var string[] $importPrefixes */ $importPrefixes = $import->prolog['prefix']; foreach ($importPrefixes as $prefix => $url) { if (trim($prefix)) { if ( ! array_key_exists($prefix, $schemaPrefixes)) { $schemaPrefixes[$prefix] = $url; } } } if (count($schemaPrefixes) != $countSchemaPrefixes) { $schema->setPrefixes($schemaPrefixes); $schema->save(); } try { $prolog = $import->processProlog(); $import->processData(); $fileImportHistory->setResults($import->results); $fileImportHistory->setMap($import->mapping); $fileImportHistory->setTotalProcessedCount( $import->DataWorkflowResults->getTotalProcessedCount()); $fileImportHistory->setErrorCount($import->DataWorkflowResults->getErrorCount()); $fileImportHistory->setSuccessCount($import->DataWorkflowResults->getSuccessCount()); $fileImportHistory->setResults('Your file has been imported. It took us: ' . $import->DataWorkflowResults->getElapsed()->format("%h hours; %i minutes; %s seconds")); $fileImportHistory->save(); } catch (\Exception $e) { $fileImportHistory->setResults("There was an error processing the import. Message: " . $e->getMessage()); $fileImportHistory->save(); throw $e; } $agentId = ($fileImportHistory->getSchema()) ? $fileImportHistory->getSchema()->getAgentId() : $fileImportHistory->getVocabulary()->getAgentId(); $newFilePath = \sfConfig::get( 'sf_repos_dir' ) . DIRECTORY_SEPARATOR . 'agents' . DIRECTORY_SEPARATOR . $agentId . DIRECTORY_SEPARATOR . $fileImportHistory->getSourceFileName(); $request = new \myWebRequest(); $result = $request->moveToRepo($filePath, $newFilePath); $schema = \SchemaPeer::retrieveByPK($schemaId); if ($schema) { $schema->setUpdatedAt(time()); $schema->setUpdatedUserId($import->userId); $schema->save(); } unset ($import); unset ($request); unset($databaseManager); }
/** * Populates the object using an array. * * This is particularly useful when populating an object from one of the * request arrays (e.g. $_POST). This method goes through the column * names, checking to see whether a matching key exists in populated * array. If so the setByName() method is called for that column. * * You can specify the key type of the array by additionally passing one * of the class type constants TYPE_PHPNAME, TYPE_COLNAME, TYPE_FIELDNAME, * TYPE_NUM. The default key type is the column's phpname (e.g. 'authorId') * * @param array $arr An array to populate the object from. * @param string $keyType The type of keys the array uses. * @return void */ public function fromArray($arr, $keyType = BasePeer::TYPE_PHPNAME) { $keys = FileImportHistoryPeer::getFieldNames($keyType); if (array_key_exists($keys[0], $arr)) { $this->setId($arr[$keys[0]]); } if (array_key_exists($keys[1], $arr)) { $this->setCreatedAt($arr[$keys[1]]); } if (array_key_exists($keys[2], $arr)) { $this->setMap($arr[$keys[2]]); } if (array_key_exists($keys[3], $arr)) { $this->setUserId($arr[$keys[3]]); } if (array_key_exists($keys[4], $arr)) { $this->setVocabularyId($arr[$keys[4]]); } if (array_key_exists($keys[5], $arr)) { $this->setSchemaId($arr[$keys[5]]); } if (array_key_exists($keys[6], $arr)) { $this->setFileName($arr[$keys[6]]); } if (array_key_exists($keys[7], $arr)) { $this->setSourceFileName($arr[$keys[7]]); } if (array_key_exists($keys[8], $arr)) { $this->setFileType($arr[$keys[8]]); } if (array_key_exists($keys[9], $arr)) { $this->setBatchId($arr[$keys[9]]); } if (array_key_exists($keys[10], $arr)) { $this->setResults($arr[$keys[10]]); } if (array_key_exists($keys[11], $arr)) { $this->setTotalProcessedCount($arr[$keys[11]]); } if (array_key_exists($keys[12], $arr)) { $this->setErrorCount($arr[$keys[12]]); } if (array_key_exists($keys[13], $arr)) { $this->setSuccessCount($arr[$keys[13]]); } }