/**
* Analyze a lexicon project and create Sense, Example and Picture guids. Remove id from Sense and Example
* @param LexProjectModelForUseWithSenseGuidMigration $project
* @param string $projectId
* @param string $testMode
*/
private static function analyzeProject($project, $projectId, $testMode)
{
$entryModifiedCount = 0;
$exampleModifiedCount = 0;
$pictureModifiedCount = 0;
$entryList = LexEntryCommands::listEntries($projectId);
foreach ($entryList->entries as $entryListItem) {
$entry = new LexEntryModel($project, $entryListItem['id']);
$entryModified = false;
if ($entry->hasSenses()) {
/** @var LexSense $sense */
foreach ($entry->senses as $sense) {
self::createSenseGuids($sense, $entryModified, $exampleModifiedCount, $pictureModifiedCount);
}
if ($entryModified) {
$entryModifiedCount++;
}
if (!$testMode) {
$entry->write();
}
}
}
if (!$testMode) {
$project->hasHadSenseGuidsMigrated = true;
$project->write();
}
print "{$exampleModifiedCount} example and {$pictureModifiedCount} picture guids created.\n";
print "{$entryModifiedCount} of {$entryList->count} entries had sense guids created.\n";
}
/**
* Analyze a lexicon project and migrate the semantic domain keys in the senses
* @param ProjectModelForUseWithSemanticDomainMigration $project
* @param string $projectId
* @param string $testMode
* @param string $message
*/
private function analyzeProject($project, $projectId, $testMode, &$message)
{
$entryModifiedCount = 0;
$entryList = LexEntryCommands::listEntries($projectId);
foreach ($entryList->entries as $entryListItem) {
$entry = new LexEntryModel($project, $entryListItem['id']);
$entryModified = false;
if ($entry->hasSenses()) {
/** @var LexSense $sense */
foreach ($entry->senses as $sense) {
$this->migrateSemDomKey($sense, $project->projectName, $message, $entryModified);
}
}
if ($entryModified) {
$entryModifiedCount++;
if (!$testMode) {
$entry->write();
}
}
}
if (!$testMode) {
$project->hasMigratedSemanticDomainKeys = true;
$project->write();
}
if ($entryModifiedCount > 0) {
print "{$entryModifiedCount} entries with semantic domains were migrated\n";
}
}
public function run($userId, $mode = 'test')
{
$testMode = $mode != 'run';
$message = "Ensure DB Indexes\n";
$numberOfIndexesCreated = 0;
$website = Website::get();
$onDevMachine = strpos($website->domain, 'dev.') !== false;
$onLocalMachine = strrpos($website->domain, '.local') !== false;
$message .= "\n------------- Main Database:\n";
$mainCollectionName = ProjectModelMongoMapper::instance()->getCollectionName();
$mainIndexes = ProjectModelMongoMapper::instance()->INDEXES_REQUIRED;
$mainIndexesToCreate = MongoStore::getIndexesNotSetInCollection(SF_DATABASE, $mainCollectionName, $mainIndexes);
$numberOfIndexesCreated += count($mainIndexesToCreate);
$message .= count($mainIndexesToCreate) . " main indexes created.\n";
if (($onDevMachine || $onLocalMachine) && MongoStore::hasDB(SF_TEST_DATABASE)) {
$message .= "\n------------- Test Database:\n";
$mainIndexesToCreate = MongoStore::getIndexesNotSetInCollection(SF_TEST_DATABASE, $mainCollectionName, $mainIndexes);
$numberOfIndexesCreated += count($mainIndexesToCreate);
$message .= count($mainIndexesToCreate) . " test indexes created.\n";
}
if (!$testMode) {
MongoStore::ensureIndexesInCollection(SF_DATABASE, $mainCollectionName, $mainIndexes);
if (($onDevMachine || $onLocalMachine) && MongoStore::hasDB(SF_TEST_DATABASE)) {
MongoStore::ensureIndexesInCollection(SF_TEST_DATABASE, $mainCollectionName, $mainIndexes);
}
}
// loop over every project
$projectList = new ProjectListModel();
$projectList->read();
foreach ($projectList->entries as $projectParams) {
$project = ProjectModel::getById($projectParams['id']);
if ($project->appName == 'lexicon') {
$message .= "\n------------- {$project->projectName} project:\n";
$lexiconCollectionName = LexEntryModel::mapper($project->databaseName())->getCollectionName();
$lexiconIndexes = LexEntryModel::mapper($project->databaseName())->INDEXES_REQUIRED;
$lexiconIndexesToCreate = MongoStore::getIndexesNotSetInCollection($project->databaseName(), $lexiconCollectionName, $lexiconIndexes);
$numberOfIndexesCreated += count($lexiconIndexesToCreate);
$optionListCollectionName = LexOptionListModel::mapper($project->databaseName())->getCollectionName();
$optionListIndexes = LexOptionListModel::mapper($project->databaseName())->INDEXES_REQUIRED;
$optionListIndexesToCreate = MongoStore::getIndexesNotSetInCollection($project->databaseName(), $optionListCollectionName, $optionListIndexes);
$numberOfIndexesCreated += count($optionListIndexesToCreate);
if (count($lexiconIndexesToCreate) + count($optionListIndexesToCreate) > 0) {
$message .= count($lexiconIndexesToCreate) . " lexicon indexes created.\n";
$message .= count($optionListIndexesToCreate) . " option list indexes created.\n";
} else {
$message .= "No indexes needed creating.\n";
}
if (!$testMode) {
MongoStore::ensureIndexesInCollection($project->databaseName(), $lexiconCollectionName, $lexiconIndexes);
MongoStore::ensureIndexesInCollection($project->databaseName(), $optionListCollectionName, $optionListIndexes);
}
}
}
if ($numberOfIndexesCreated > 0) {
$message .= "\nCreated {$numberOfIndexesCreated} DB Indexes.\n\n";
} else {
$message .= "\nAll indexes were present.\n\n";
}
return $message;
}
public function encodeIdReference($key, $model)
{
if ($model->asString() == '') {
return '';
}
if ($key == 'userRef' || $key == 'userRef2') {
$user = new UserModel();
if ($user->exists($model->asString())) {
$user->read($model->asString());
return array('id' => $user->id->asString(), 'avatar_ref' => $user->avatar_ref, 'username' => $user->username);
} else {
return '';
}
} elseif ($key == 'projectRef') {
$project = new ProjectModel($model->asString());
return array('id' => $project->id->asString(), 'type' => $project->appName);
} elseif ($key == 'textRef') {
$text = new TextModel($this->_project);
if ($text->exists($model->asString())) {
return $model->asString();
} else {
return '';
}
} elseif ($key == 'questionRef') {
$question = new QuestionModel($this->_project);
if ($question->exists($model->asString())) {
return $model->asString();
} else {
return '';
}
} elseif ($key == 'entryRef') {
$entry = new LexEntryModel($this->_project);
if ($entry->exists($model->asString())) {
return $model->asString();
} else {
return '';
}
} else {
return $model->asString();
}
}
/**
* Analyze a lexicon project and create Entry guids.
* @param LexProjectModelForEntryGuidMigration $project
* @param string $testMode
* @internal param string $projectId
*/
private static function analyzeProject($project, $testMode)
{
$entryModifiedCount = 0;
$entryList = new LexAllEntryListModel($project);
$entryList->read();
foreach ($entryList->entries as $entryListItem) {
$entry = new LexEntryModel($project, $entryListItem['id']);
if (!isset($entry->guid) || !$entry->guid || !Guid::isValid($entry->guid)) {
$entry->guid = Guid::create();
$entryModifiedCount++;
if (!$testMode) {
$entry->write();
}
}
}
if (!$testMode) {
$project->hasHadEntryGuidMigrated = true;
$project->write();
}
print "{$entryModifiedCount} of {$entryList->count} entries had guids created.\n";
}
/**
* Upload an audio file
*
* @param string $projectId
* @param string $mediaType
* @param string $tmpFilePath
* @throws \Exception
* @return \Api\Model\Shared\Command\UploadResponse
*/
public static function uploadAudioFile($projectId, $mediaType, $tmpFilePath)
{
if ($mediaType != 'entry-audio') {
throw new \Exception("Unsupported upload type.");
}
if (!$tmpFilePath) {
throw new \Exception("Upload controller did not move the uploaded file.");
}
$entryId = $_POST['entryId'];
$file = $_FILES['file'];
$fileName = $file['name'];
$finfo = finfo_open(FILEINFO_MIME_TYPE);
$fileType = finfo_file($finfo, $tmpFilePath);
finfo_close($finfo);
$fileName = FileUtilities::replaceSpecialCharacters($fileName);
$fileExt = false === ($pos = strrpos($fileName, '.')) ? '' : substr($fileName, $pos);
$allowedTypes = array("audio/mpeg", "audio/mp3");
$allowedExtensions = array(".mp3");
$response = new UploadResponse();
if (in_array(strtolower($fileType), $allowedTypes) && in_array(strtolower($fileExt), $allowedExtensions)) {
// make the folders if they don't exist
$project = new LfProjectModel($projectId);
$folderPath = $project->getAssetsFolderPath() . '/audio';
FileUtilities::createAllFolders($folderPath);
// cleanup previous files of any allowed extension
self::cleanupFiles($folderPath, $entryId, $allowedExtensions);
// move uploaded file from tmp location to assets
$filePath = self::mediaFilePath($folderPath, $entryId, $fileName);
$moveOk = copy($tmpFilePath, $filePath);
@unlink($tmpFilePath);
// update database with file location
$entry = new LexEntryModel($project, $entryId);
$entry->audioFileName = '';
if ($moveOk) {
$entry->audioFileName = $fileName;
}
$entry->write();
// construct server response
if ($moveOk && $tmpFilePath) {
$data = new MediaResult();
$data->path = $project->getAssetsRelativePath();
$data->fileName = $fileName;
$response->result = true;
} else {
$data = new ErrorResult();
$data->errorType = 'UserMessage';
$data->errorMessage = "{$fileName} could not be saved to the right location. Contact your Site Administrator.";
$response->result = false;
}
} else {
$allowedExtensionsStr = implode(", ", $allowedExtensions);
$data = new ErrorResult();
$data->errorType = 'UserMessage';
if (count($allowedExtensions) < 1) {
$data->errorMessage = "{$fileName} is not an allowed audio file. No audio file formats are currently enabled, contact your Site Administrator.";
} elseif (count($allowedExtensions) == 1) {
$data->errorMessage = "{$fileName} is not an allowed audio file. Ensure the file is a {$allowedExtensionsStr}.";
} else {
$data->errorMessage = "{$fileName} is not an allowed audio file. Ensure the file is one of the following types: {$allowedExtensionsStr}.";
}
$response->result = false;
}
$response->data = $data;
return $response;
}
public function createDatabaseIndexes()
{
$collectionName = LexEntryModel::mapper($this->databaseName())->getCollectionName();
$indexes = LexEntryModel::mapper($this->databaseName())->INDEXES_REQUIRED;
MongoStore::addIndexesToCollection($this->databaseName(), $collectionName, $indexes);
$collectionName = LexOptionListModel::mapper($this->databaseName())->getCollectionName();
$indexes = LexOptionListModel::mapper($this->databaseName())->INDEXES_REQUIRED;
MongoStore::addIndexesToCollection($this->databaseName(), $collectionName, $indexes);
}
public static function removeEntry($projectId, $entryId, $userId)
{
$project = new ProjectModel($projectId);
$entry = new LexEntryModel($project, $entryId);
$entry->isDeleted = true;
$entry->write();
ActivityCommands::deleteEntry($project, $userId, $entryId);
return true;
}
/**
* @param string $liftFilePath
* @param LexiconProjectModel $projectModel
* @param LiftMergeRule $mergeRule
* @param boolean $skipSameModTime
* @param boolean $deleteMatchingEntry
* @return \Api\Model\Languageforge\Lexicon\LiftImport
*/
public function merge($liftFilePath, $projectModel, $mergeRule = LiftMergeRule::CREATE_DUPLICATES, $skipSameModTime = true, $deleteMatchingEntry = false)
{
ini_set('max_execution_time', 180);
// Sufficient time to import webster. TODO Make this async CP 2014-10
$entryList = new LexEntryListModel($projectModel);
$entryList->read();
$hasExistingData = $entryList->count != 0;
if (!$hasExistingData) {
$projectModel->config->clearAllInputSystems();
// save and clear input systems
$savedInputSystems = $projectModel->inputSystems->getArrayCopy();
$projectModel->inputSystems->exchangeArray(array());
}
$reader = new \XMLReader();
$reader->open($liftFilePath);
$this->liftDecoder = new LiftDecoder($projectModel);
$this->stats = new LiftImportStats($entryList->count);
$this->report = new ImportErrorReport();
$this->liftImportNodeError = new LiftImportNodeError(LiftImportNodeError::FILE, basename($liftFilePath));
$liftRangeDecoder = new LiftRangeDecoder($projectModel);
$liftRangeFiles = array();
// Keys: filenames. Values: parsed files.
$liftRanges = array();
// Keys: @id attributes of <range> elements. Values: parsed <range> elements.
$liftFolderPath = dirname($liftFilePath);
while ($reader->read()) {
// Read LIFT ranges in the header of the LIFT file
if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'range') {
$node = $reader->expand();
$rangeId = $node->attributes->getNamedItem('id')->textContent;
$rangeHref = $node->attributes->getNamedItem('href')->textContent;
$hrefPath = parse_url($rangeHref, PHP_URL_PATH);
$rangeFilename = basename($hrefPath);
$rangeImportNodeError = new LiftRangeImportNodeError(LiftRangeImportNodeError::FILE, $rangeFilename);
if (!array_key_exists($rangeFilename, $liftRangeFiles)) {
// Haven't parsed the .lift-ranges file yet. We'll assume it is alongside the .lift file.
$rangeFilePath = $liftFolderPath . "/" . $rangeFilename;
if (file_exists($rangeFilePath)) {
$sxeNode = simplexml_load_file($rangeFilePath);
$parsedRanges = $liftRangeDecoder->decode($sxeNode);
$liftRanges = array_merge($liftRanges, $parsedRanges);
$liftRangeFiles[] = $rangeFilename;
} else {
// Range file was NOT found in alongside the .lift file
$rangeImportNodeError->addRangeFileNotFound(basename($liftFilePath));
}
}
// pull out the referenced range
if (isset($liftRanges[$rangeId])) {
$range = $liftRanges[$rangeId];
} else {
$range = null;
if (file_exists($rangeFilePath)) {
// Range was NOT found in referenced .lift-ranges file after parsing it
$rangeImportNodeError->addRangeNotFound($rangeId);
}
}
// Range elements defined in LIFT file override any values defined in .lift-ranges file.
if ($node->hasChildNodes()) {
$rangeNode = self::domNode_to_sxeNode($node);
$range = $liftRangeDecoder->readRange($rangeNode, $range);
$liftRanges[$rangeId] = $range;
}
if ($rangeImportNodeError->hasErrors()) {
$this->liftImportNodeError->addSubnodeError($rangeImportNodeError);
}
}
// Read the custom 'fields' spec in the header of the LIFT file
if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'fields') {
$isInFieldsSectionOfLift = true;
$this->liftDecoder->liftFields = array();
while ($isInFieldsSectionOfLift && $reader->read()) {
if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'field') {
$node = $reader->expand();
$sxeNode = self::domNode_to_sxeNode($node);
$LiftFieldTag = (string) $sxeNode['tag'];
$liftField = array();
foreach ($sxeNode as $element) {
if ($element->getName() === 'form') {
$inputSystemTag = (string) $element['lang'];
$liftField[$inputSystemTag] = (string) $element->text;
}
}
$this->liftDecoder->liftFields[$LiftFieldTag] = $liftField;
} elseif ($reader->nodeType == \XMLReader::END_ELEMENT && $reader->localName == 'fields') {
$isInFieldsSectionOfLift = false;
}
}
}
// Read an entry node
if ($reader->nodeType == \XMLReader::ELEMENT && $reader->localName == 'entry') {
$this->stats->importEntries++;
$node = $reader->expand();
$sxeNode = self::domNode_to_sxeNode($node);
$guid = $reader->getAttribute('guid');
$existingEntry = $entryList->searchEntriesFor('guid', $guid);
if ($existingEntry) {
$entry = new LexEntryModel($projectModel, $existingEntry['id']);
$dateModified = $reader->getAttribute('dateModified');
if (self::differentModTime($dateModified, $entry->authorInfo->modifiedDate) || !$skipSameModTime) {
if ($mergeRule == LiftMergeRule::CREATE_DUPLICATES) {
$entry = new LexEntryModel($projectModel);
$this->readEntryWithErrorReport($sxeNode, $entry, $mergeRule);
$entry->guid = '';
$entry->write();
$this->stats->entriesDuplicated++;
} else {
if (isset($sxeNode->{'lexical-unit'})) {
$this->readEntryWithErrorReport($sxeNode, $entry, $mergeRule);
$entry->write();
$this->stats->entriesMerged++;
} elseif (isset($sxeNode->attributes()->dateDeleted) && $deleteMatchingEntry) {
LexEntryModel::remove($projectModel, $existingEntry['id']);
$this->stats->entriesDeleted++;
}
}
} else {
// skip because same mod time and skip enabled
if (!isset($sxeNode->{'lexical-unit'}) && isset($sxeNode->attributes()->dateDeleted) && $deleteMatchingEntry) {
LexEntryModel::remove($projectModel, $existingEntry['id']);
$this->stats->entriesDeleted++;
}
}
} else {
if (isset($sxeNode->{'lexical-unit'})) {
$entry = new LexEntryModel($projectModel);
$this->readEntryWithErrorReport($sxeNode, $entry, $mergeRule);
$entry->write();
$this->stats->newEntries++;
}
}
}
}
$reader->close();
// put back saved input systems if none found in the imported data
if (!$hasExistingData && $projectModel->inputSystems->count() <= 0) {
$projectModel->inputSystems->exchangeArray($savedInputSystems);
}
// add lift ranges
if ($mergeRule != LiftMergeRule::IMPORT_LOSES) {
foreach ($liftRanges as $liftRangeCode => $liftRange) {
// add everything except semantic domains
if (strpos($liftRangeCode, 'semantic-domain') === false) {
self::rangeToOptionList($projectModel, $liftRangeCode, LexiconConfigObj::flexOptionlistName($liftRangeCode), $liftRange);
}
}
}
$this->report->nodeErrors[] = $this->liftImportNodeError;
if ($this->report->hasError()) {
error_log($this->report->toString());
}
return $this;
}
public function testListEntries_someEntriesWithNoDefinition_Ok()
{
$e = new LexiconMongoTestEnvironment();
$e->clean();
$project = $e->createProject(SF_TESTPROJECT, SF_TESTPROJECTCODE);
$projectId = $project->id->asString();
for ($i = 0; $i < 10; $i++) {
$entry = new LexEntryModel($project);
$entry->lexeme->form('th', 'Apfel' . $i);
if ($i % 2 == 0) {
$sense = new Sense();
$entry->senses[] = $sense;
}
if ($i % 3 == 0) {
$sense = new Sense();
$sense->definition->form('en', 'apple');
$sense->partOfSpeech->value = 'noun';
$entry->senses[] = $sense;
}
$entry->write();
}
$result = LexEntryCommands::listEntries($projectId);
$this->assertEqual($result->entries[0]['lexeme']['th']['value'], 'Apfel0');
$this->assertTrue(!array_key_exists('definition', $result->entries[0]['senses'][0]));
$this->assertEqual($result->entries[3]['senses'][0]['definition']['en']['value'], 'apple');
}
public function testCreateProject_LexProject_IndexesCreated()
{
self::$environ = new LexiconMongoTestEnvironment();
self::$environ->clean();
$user1Id = self::$environ->createUser("user1name", "User1 Name", "*****@*****.**");
$user1 = new UserModel($user1Id);
$srProject = null;
$projectId = ProjectCommands::createProject(SF_TESTPROJECT, SF_TESTPROJECTCODE, LexProjectModel::LEXICON_APP, $user1->id->asString(), self::$environ->website, $srProject);
$project = new LexProjectModel($projectId);
$databaseName = $project->databaseName();
$collectionName = LexEntryModel::mapper($databaseName)->getCollectionName();
$indexCount = iterator_count(MongoStore::getCollectionIndexes($databaseName, $collectionName));
$this->assertTrue($indexCount >= 3);
$index = ['key' => ['guid' => 1]];
$this->assertTrue(MongoStore::isAllIndexFieldNamesInCollection($index, $databaseName, $collectionName));
$collectionName = LexOptionListModel::mapper($databaseName)->getCollectionName();
$indexCount = iterator_count(MongoStore::getCollectionIndexes($databaseName, $collectionName));
$this->assertTrue($indexCount >= 2);
$index = ['key' => ['code' => 1]];
$this->assertTrue(MongoStore::isAllIndexFieldNamesInCollection($index, $databaseName, $collectionName));
}
function testReadEntry_NoComments_ReadBackOk()
{
$e = new LexiconMongoTestEnvironment();
$e->clean();
$project = $e->createProject(SF_TESTPROJECT, SF_TESTPROJECTCODE);
$projectId = $project->id->asString();
$entry = new LexEntryModel($project);
$entry->lexeme->form('th', 'apple');
$sense = new Sense();
$sense->definition->form('en', 'red fruit');
$sense->partOfSpeech = new LexiconField('noun');
$example = new Example();
$example->sentence->form('th', 'example1');
$example->translation->form('en', 'trans1');
$sense->examples[] = $example;
$entry->senses[] = $sense;
$entryId = $entry->write();
$newEntry = LexEntryCommands::readEntry($projectId, $entryId);
$this->assertEqual($newEntry['lexeme']['th']['value'], 'apple');
$this->assertEqual($newEntry['senses'][0]['definition']['en']['value'], 'red fruit');
$this->assertEqual($newEntry['senses'][0]['partOfSpeech']['value'], 'noun');
$this->assertEqual($newEntry['senses'][0]['examples'][0]['sentence']['th']['value'], 'example1');
$this->assertEqual($newEntry['senses'][0]['examples'][0]['translation']['en']['value'], 'trans1');
}
