/** * Analyze a lexicon project and create Sense, Example and Picture guids. Remove id from Sense and Example * @param LexProjectModelForUseWithSenseGuidMigration $project * @param string $projectId * @param string $testMode */ private static function analyzeProject($project, $projectId, $testMode) { $entryModifiedCount = 0; $exampleModifiedCount = 0; $pictureModifiedCount = 0; $entryList = LexEntryCommands::listEntries($projectId); foreach ($entryList->entries as $entryListItem) { $entry = new LexEntryModel($project, $entryListItem['id']); $entryModified = false; if ($entry->hasSenses()) { /** @var LexSense $sense */ foreach ($entry->senses as $sense) { self::createSenseGuids($sense, $entryModified, $exampleModifiedCount, $pictureModifiedCount); } if ($entryModified) { $entryModifiedCount++; } if (!$testMode) { $entry->write(); } } } if (!$testMode) { $project->hasHadSenseGuidsMigrated = true; $project->write(); } print "{$exampleModifiedCount} example and {$pictureModifiedCount} picture guids created.\n"; print "{$entryModifiedCount} of {$entryList->count} entries had sense guids created.\n"; }
/** * Analyze a lexicon project and migrate the semantic domain keys in the senses * @param ProjectModelForUseWithSemanticDomainMigration $project * @param string $projectId * @param string $testMode * @param string $message */ private function analyzeProject($project, $projectId, $testMode, &$message) { $entryModifiedCount = 0; $entryList = LexEntryCommands::listEntries($projectId); foreach ($entryList->entries as $entryListItem) { $entry = new LexEntryModel($project, $entryListItem['id']); $entryModified = false; if ($entry->hasSenses()) { /** @var LexSense $sense */ foreach ($entry->senses as $sense) { $this->migrateSemDomKey($sense, $project->projectName, $message, $entryModified); } } if ($entryModified) { $entryModifiedCount++; if (!$testMode) { $entry->write(); } } } if (!$testMode) { $project->hasMigratedSemanticDomainKeys = true; $project->write(); } if ($entryModifiedCount > 0) { print "{$entryModifiedCount} entries with semantic domains were migrated\n"; } }
public function run($userId, $mode = 'test') { $testMode = $mode != 'run'; $message = "Ensure DB Indexes\n"; $numberOfIndexesCreated = 0; $website = Website::get(); $onDevMachine = strpos($website->domain, 'dev.') !== false; $onLocalMachine = strrpos($website->domain, '.local') !== false; $message .= "\n------------- Main Database:\n"; $mainCollectionName = ProjectModelMongoMapper::instance()->getCollectionName(); $mainIndexes = ProjectModelMongoMapper::instance()->INDEXES_REQUIRED; $mainIndexesToCreate = MongoStore::getIndexesNotSetInCollection(SF_DATABASE, $mainCollectionName, $mainIndexes); $numberOfIndexesCreated += count($mainIndexesToCreate); $message .= count($mainIndexesToCreate) . " main indexes created.\n"; if (($onDevMachine || $onLocalMachine) && MongoStore::hasDB(SF_TEST_DATABASE)) { $message .= "\n------------- Test Database:\n"; $mainIndexesToCreate = MongoStore::getIndexesNotSetInCollection(SF_TEST_DATABASE, $mainCollectionName, $mainIndexes); $numberOfIndexesCreated += count($mainIndexesToCreate); $message .= count($mainIndexesToCreate) . " test indexes created.\n"; } if (!$testMode) { MongoStore::ensureIndexesInCollection(SF_DATABASE, $mainCollectionName, $mainIndexes); if (($onDevMachine || $onLocalMachine) && MongoStore::hasDB(SF_TEST_DATABASE)) { MongoStore::ensureIndexesInCollection(SF_TEST_DATABASE, $mainCollectionName, $mainIndexes); } } // loop over every project $projectList = new ProjectListModel(); $projectList->read(); foreach ($projectList->entries as $projectParams) { $project = ProjectModel::getById($projectParams['id']); if ($project->appName == 'lexicon') { $message .= "\n------------- {$project->projectName} project:\n"; $lexiconCollectionName = LexEntryModel::mapper($project->databaseName())->getCollectionName(); $lexiconIndexes = LexEntryModel::mapper($project->databaseName())->INDEXES_REQUIRED; $lexiconIndexesToCreate = MongoStore::getIndexesNotSetInCollection($project->databaseName(), $lexiconCollectionName, $lexiconIndexes); $numberOfIndexesCreated += count($lexiconIndexesToCreate); $optionListCollectionName = LexOptionListModel::mapper($project->databaseName())->getCollectionName(); $optionListIndexes = LexOptionListModel::mapper($project->databaseName())->INDEXES_REQUIRED; $optionListIndexesToCreate = MongoStore::getIndexesNotSetInCollection($project->databaseName(), $optionListCollectionName, $optionListIndexes); $numberOfIndexesCreated += count($optionListIndexesToCreate); if (count($lexiconIndexesToCreate) + count($optionListIndexesToCreate) > 0) { $message .= count($lexiconIndexesToCreate) . " lexicon indexes created.\n"; $message .= count($optionListIndexesToCreate) . " option list indexes created.\n"; } else { $message .= "No indexes needed creating.\n"; } if (!$testMode) { MongoStore::ensureIndexesInCollection($project->databaseName(), $lexiconCollectionName, $lexiconIndexes); MongoStore::ensureIndexesInCollection($project->databaseName(), $optionListCollectionName, $optionListIndexes); } } } if ($numberOfIndexesCreated > 0) { $message .= "\nCreated {$numberOfIndexesCreated} DB Indexes.\n\n"; } else { $message .= "\nAll indexes were present.\n\n"; } return $message; }
public function encodeIdReference($key, $model) { if ($model->asString() == '') { return ''; } if ($key == 'userRef' || $key == 'userRef2') { $user = new UserModel(); if ($user->exists($model->asString())) { $user->read($model->asString()); return array('id' => $user->id->asString(), 'avatar_ref' => $user->avatar_ref, 'username' => $user->username); } else { return ''; } } elseif ($key == 'projectRef') { $project = new ProjectModel($model->asString()); return array('id' => $project->id->asString(), 'type' => $project->appName); } elseif ($key == 'textRef') { $text = new TextModel($this->_project); if ($text->exists($model->asString())) { return $model->asString(); } else { return ''; } } elseif ($key == 'questionRef') { $question = new QuestionModel($this->_project); if ($question->exists($model->asString())) { return $model->asString(); } else { return ''; } } elseif ($key == 'entryRef') { $entry = new LexEntryModel($this->_project); if ($entry->exists($model->asString())) { return $model->asString(); } else { return ''; } } else { return $model->asString(); } }
/** * Analyze a lexicon project and create Entry guids. * @param LexProjectModelForEntryGuidMigration $project * @param string $testMode * @internal param string $projectId */ private static function analyzeProject($project, $testMode) { $entryModifiedCount = 0; $entryList = new LexAllEntryListModel($project); $entryList->read(); foreach ($entryList->entries as $entryListItem) { $entry = new LexEntryModel($project, $entryListItem['id']); if (!isset($entry->guid) || !$entry->guid || !Guid::isValid($entry->guid)) { $entry->guid = Guid::create(); $entryModifiedCount++; if (!$testMode) { $entry->write(); } } } if (!$testMode) { $project->hasHadEntryGuidMigrated = true; $project->write(); } print "{$entryModifiedCount} of {$entryList->count} entries had guids created.\n"; }
/** * Upload an audio file * * @param string $projectId * @param string $mediaType * @param string $tmpFilePath * @throws \Exception * @return \Api\Model\Shared\Command\UploadResponse */ public static function uploadAudioFile($projectId, $mediaType, $tmpFilePath) { if ($mediaType != 'entry-audio') { throw new \Exception("Unsupported upload type."); } if (!$tmpFilePath) { throw new \Exception("Upload controller did not move the uploaded file."); } $entryId = $_POST['entryId']; $file = $_FILES['file']; $fileName = $file['name']; $finfo = finfo_open(FILEINFO_MIME_TYPE); $fileType = finfo_file($finfo, $tmpFilePath); finfo_close($finfo); $fileName = FileUtilities::replaceSpecialCharacters($fileName); $fileExt = false === ($pos = strrpos($fileName, '.')) ? '' : substr($fileName, $pos); $allowedTypes = array("audio/mpeg", "audio/mp3"); $allowedExtensions = array(".mp3"); $response = new UploadResponse(); if (in_array(strtolower($fileType), $allowedTypes) && in_array(strtolower($fileExt), $allowedExtensions)) { // make the folders if they don't exist $project = new LfProjectModel($projectId); $folderPath = $project->getAssetsFolderPath() . '/audio'; FileUtilities::createAllFolders($folderPath); // cleanup previous files of any allowed extension self::cleanupFiles($folderPath, $entryId, $allowedExtensions); // move uploaded file from tmp location to assets $filePath = self::mediaFilePath($folderPath, $entryId, $fileName); $moveOk = copy($tmpFilePath, $filePath); @unlink($tmpFilePath); // update database with file location $entry = new LexEntryModel($project, $entryId); $entry->audioFileName = ''; if ($moveOk) { $entry->audioFileName = $fileName; } $entry->write(); // construct server response if ($moveOk && $tmpFilePath) { $data = new MediaResult(); $data->path = $project->getAssetsRelativePath(); $data->fileName = $fileName; $response->result = true; } else { $data = new ErrorResult(); $data->errorType = 'UserMessage'; $data->errorMessage = "{$fileName} could not be saved to the right location. Contact your Site Administrator."; $response->result = false; } } else { $allowedExtensionsStr = implode(", ", $allowedExtensions); $data = new ErrorResult(); $data->errorType = 'UserMessage'; if (count($allowedExtensions) < 1) { $data->errorMessage = "{$fileName} is not an allowed audio file. No audio file formats are currently enabled, contact your Site Administrator."; } elseif (count($allowedExtensions) == 1) { $data->errorMessage = "{$fileName} is not an allowed audio file. Ensure the file is a {$allowedExtensionsStr}."; } else { $data->errorMessage = "{$fileName} is not an allowed audio file. Ensure the file is one of the following types: {$allowedExtensionsStr}."; } $response->result = false; } $response->data = $data; return $response; }
public function createDatabaseIndexes() { $collectionName = LexEntryModel::mapper($this->databaseName())->getCollectionName(); $indexes = LexEntryModel::mapper($this->databaseName())->INDEXES_REQUIRED; MongoStore::addIndexesToCollection($this->databaseName(), $collectionName, $indexes); $collectionName = LexOptionListModel::mapper($this->databaseName())->getCollectionName(); $indexes = LexOptionListModel::mapper($this->databaseName())->INDEXES_REQUIRED; MongoStore::addIndexesToCollection($this->databaseName(), $collectionName, $indexes); }
public static function removeEntry($projectId, $entryId, $userId) { $project = new ProjectModel($projectId); $entry = new LexEntryModel($project, $entryId); $entry->isDeleted = true; $entry->write(); ActivityCommands::deleteEntry($project, $userId, $entryId); return true; }
/** * @param string $liftFilePath * @param LexiconProjectModel $projectModel * @param LiftMergeRule $mergeRule * @param boolean $skipSameModTime * @param boolean $deleteMatchingEntry * @return \Api\Model\Languageforge\Lexicon\LiftImport */ public function merge($liftFilePath, $projectModel, $mergeRule = LiftMergeRule::CREATE_DUPLICATES, $skipSameModTime = true, $deleteMatchingEntry = false) { ini_set('max_execution_time', 180); // Sufficient time to import webster. TODO Make this async CP 2014-10 $entryList = new LexEntryListModel($projectModel); $entryList->read(); $hasExistingData = $entryList->count != 0; if (!$hasExistingData) { $projectModel->config->clearAllInputSystems(); // save and clear input systems $savedInputSystems = $projectModel->inputSystems->getArrayCopy(); $projectModel->inputSystems->exchangeArray(array()); } $reader = new \XMLReader(); $reader->open($liftFilePath); $this->liftDecoder = new LiftDecoder($projectModel); $this->stats = new LiftImportStats($entryList->count); $this->report = new ImportErrorReport(); $this->liftImportNodeError = new LiftImportNodeError(LiftImportNodeError::FILE, basename($liftFilePath)); $liftRangeDecoder = new LiftRangeDecoder($projectModel); $liftRangeFiles = array(); // Keys: filenames. Values: parsed files. $liftRanges = array(); // Keys: @id attributes of <range> elements. Values: parsed <range> elements. $liftFolderPath = dirname($liftFilePath); while ($reader->read()) { // Read LIFT ranges in the header of the LIFT file if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'range') { $node = $reader->expand(); $rangeId = $node->attributes->getNamedItem('id')->textContent; $rangeHref = $node->attributes->getNamedItem('href')->textContent; $hrefPath = parse_url($rangeHref, PHP_URL_PATH); $rangeFilename = basename($hrefPath); $rangeImportNodeError = new LiftRangeImportNodeError(LiftRangeImportNodeError::FILE, $rangeFilename); if (!array_key_exists($rangeFilename, $liftRangeFiles)) { // Haven't parsed the .lift-ranges file yet. We'll assume it is alongside the .lift file. $rangeFilePath = $liftFolderPath . "/" . $rangeFilename; if (file_exists($rangeFilePath)) { $sxeNode = simplexml_load_file($rangeFilePath); $parsedRanges = $liftRangeDecoder->decode($sxeNode); $liftRanges = array_merge($liftRanges, $parsedRanges); $liftRangeFiles[] = $rangeFilename; } else { // Range file was NOT found in alongside the .lift file $rangeImportNodeError->addRangeFileNotFound(basename($liftFilePath)); } } // pull out the referenced range if (isset($liftRanges[$rangeId])) { $range = $liftRanges[$rangeId]; } else { $range = null; if (file_exists($rangeFilePath)) { // Range was NOT found in referenced .lift-ranges file after parsing it $rangeImportNodeError->addRangeNotFound($rangeId); } } // Range elements defined in LIFT file override any values defined in .lift-ranges file. if ($node->hasChildNodes()) { $rangeNode = self::domNode_to_sxeNode($node); $range = $liftRangeDecoder->readRange($rangeNode, $range); $liftRanges[$rangeId] = $range; } if ($rangeImportNodeError->hasErrors()) { $this->liftImportNodeError->addSubnodeError($rangeImportNodeError); } } // Read the custom 'fields' spec in the header of the LIFT file if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'fields') { $isInFieldsSectionOfLift = true; $this->liftDecoder->liftFields = array(); while ($isInFieldsSectionOfLift && $reader->read()) { if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'field') { $node = $reader->expand(); $sxeNode = self::domNode_to_sxeNode($node); $LiftFieldTag = (string) $sxeNode['tag']; $liftField = array(); foreach ($sxeNode as $element) { if ($element->getName() === 'form') { $inputSystemTag = (string) $element['lang']; $liftField[$inputSystemTag] = (string) $element->text; } } $this->liftDecoder->liftFields[$LiftFieldTag] = $liftField; } elseif ($reader->nodeType == \XMLReader::END_ELEMENT && $reader->localName == 'fields') { $isInFieldsSectionOfLift = false; } } } // Read an entry node if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'entry') { $this->stats->importEntries++; $node = $reader->expand(); $sxeNode = self::domNode_to_sxeNode($node); $guid = $reader->getAttribute('guid'); $existingEntry = $entryList->searchEntriesFor('guid', $guid); if ($existingEntry) { $entry = new LexEntryModel($projectModel, $existingEntry['id']); $dateModified = $reader->getAttribute('dateModified'); if (self::differentModTime($dateModified, $entry->authorInfo->modifiedDate) || !$skipSameModTime) { if ($mergeRule == LiftMergeRule::CREATE_DUPLICATES) { $entry = new LexEntryModel($projectModel); $this->readEntryWithErrorReport($sxeNode, $entry, $mergeRule); $entry->guid = ''; $entry->write(); $this->stats->entriesDuplicated++; } else { if (isset($sxeNode->{'lexical-unit'})) { $this->readEntryWithErrorReport($sxeNode, $entry, $mergeRule); $entry->write(); $this->stats->entriesMerged++; } elseif (isset($sxeNode->attributes()->dateDeleted) && $deleteMatchingEntry) { LexEntryModel::remove($projectModel, $existingEntry['id']); $this->stats->entriesDeleted++; } } } else { // skip because same mod time and skip enabled if (!isset($sxeNode->{'lexical-unit'}) && isset($sxeNode->attributes()->dateDeleted) && $deleteMatchingEntry) { LexEntryModel::remove($projectModel, $existingEntry['id']); $this->stats->entriesDeleted++; } } } else { if (isset($sxeNode->{'lexical-unit'})) { $entry = new LexEntryModel($projectModel); $this->readEntryWithErrorReport($sxeNode, $entry, $mergeRule); $entry->write(); $this->stats->newEntries++; } } } } $reader->close(); // put back saved input systems if none found in the imported data if (!$hasExistingData && $projectModel->inputSystems->count() <= 0) { $projectModel->inputSystems->exchangeArray($savedInputSystems); } // add lift ranges if ($mergeRule != LiftMergeRule::IMPORT_LOSES) { foreach ($liftRanges as $liftRangeCode => $liftRange) { // add everything except semantic domains if (strpos($liftRangeCode, 'semantic-domain') === false) { self::rangeToOptionList($projectModel, $liftRangeCode, LexiconConfigObj::flexOptionlistName($liftRangeCode), $liftRange); } } } $this->report->nodeErrors[] = $this->liftImportNodeError; if ($this->report->hasError()) { error_log($this->report->toString()); } return $this; }
public function testListEntries_someEntriesWithNoDefinition_Ok() { $e = new LexiconMongoTestEnvironment(); $e->clean(); $project = $e->createProject(SF_TESTPROJECT, SF_TESTPROJECTCODE); $projectId = $project->id->asString(); for ($i = 0; $i < 10; $i++) { $entry = new LexEntryModel($project); $entry->lexeme->form('th', 'Apfel' . $i); if ($i % 2 == 0) { $sense = new Sense(); $entry->senses[] = $sense; } if ($i % 3 == 0) { $sense = new Sense(); $sense->definition->form('en', 'apple'); $sense->partOfSpeech->value = 'noun'; $entry->senses[] = $sense; } $entry->write(); } $result = LexEntryCommands::listEntries($projectId); $this->assertEqual($result->entries[0]['lexeme']['th']['value'], 'Apfel0'); $this->assertTrue(!array_key_exists('definition', $result->entries[0]['senses'][0])); $this->assertEqual($result->entries[3]['senses'][0]['definition']['en']['value'], 'apple'); }
public function testCreateProject_LexProject_IndexesCreated() { self::$environ = new LexiconMongoTestEnvironment(); self::$environ->clean(); $user1Id = self::$environ->createUser("user1name", "User1 Name", "*****@*****.**"); $user1 = new UserModel($user1Id); $srProject = null; $projectId = ProjectCommands::createProject(SF_TESTPROJECT, SF_TESTPROJECTCODE, LexProjectModel::LEXICON_APP, $user1->id->asString(), self::$environ->website, $srProject); $project = new LexProjectModel($projectId); $databaseName = $project->databaseName(); $collectionName = LexEntryModel::mapper($databaseName)->getCollectionName(); $indexCount = iterator_count(MongoStore::getCollectionIndexes($databaseName, $collectionName)); $this->assertTrue($indexCount >= 3); $index = ['key' => ['guid' => 1]]; $this->assertTrue(MongoStore::isAllIndexFieldNamesInCollection($index, $databaseName, $collectionName)); $collectionName = LexOptionListModel::mapper($databaseName)->getCollectionName(); $indexCount = iterator_count(MongoStore::getCollectionIndexes($databaseName, $collectionName)); $this->assertTrue($indexCount >= 2); $index = ['key' => ['code' => 1]]; $this->assertTrue(MongoStore::isAllIndexFieldNamesInCollection($index, $databaseName, $collectionName)); }
function testReadEntry_NoComments_ReadBackOk() { $e = new LexiconMongoTestEnvironment(); $e->clean(); $project = $e->createProject(SF_TESTPROJECT, SF_TESTPROJECTCODE); $projectId = $project->id->asString(); $entry = new LexEntryModel($project); $entry->lexeme->form('th', 'apple'); $sense = new Sense(); $sense->definition->form('en', 'red fruit'); $sense->partOfSpeech = new LexiconField('noun'); $example = new Example(); $example->sentence->form('th', 'example1'); $example->translation->form('en', 'trans1'); $sense->examples[] = $example; $entry->senses[] = $sense; $entryId = $entry->write(); $newEntry = LexEntryCommands::readEntry($projectId, $entryId); $this->assertEqual($newEntry['lexeme']['th']['value'], 'apple'); $this->assertEqual($newEntry['senses'][0]['definition']['en']['value'], 'red fruit'); $this->assertEqual($newEntry['senses'][0]['partOfSpeech']['value'], 'noun'); $this->assertEqual($newEntry['senses'][0]['examples'][0]['sentence']['th']['value'], 'example1'); $this->assertEqual($newEntry['senses'][0]['examples'][0]['translation']['en']['value'], 'trans1'); }