public function doAction() { if (empty($this->source_lang)) { $this->result['errors'][] = array("code" => -1, "message" => "missing source_lang"); } if (empty($this->target_lang)) { $this->result['errors'][] = array("code" => -2, "message" => "missing target_lang"); } if (empty($this->source)) { $this->result['errors'][] = array("code" => -3, "message" => "missing source"); } if (empty($this->target)) { $this->result['errors'][] = array("code" => -4, "message" => "missing target"); } //get Job Infos $job_data = getJobData((int) $this->id_job, $this->password); $pCheck = new AjaxPasswordCheck(); //check for Password correctness if (empty($job_data) || !$pCheck->grantJobAccessByJobData($job_data, $this->password)) { $this->result['errors'][] = array("code" => -10, "message" => "wrong password"); return; } $this->tm_keys = $job_data['tm_keys']; $this->checkLogin(); $tms = Engine::getInstance($job_data['id_tms']); $config = $tms->getConfigStruct(); // $config = TMS::getConfigStruct(); $config['segment'] = CatUtils::view2rawxliff($this->source); $config['translation'] = CatUtils::view2rawxliff($this->target); $config['source'] = $this->source_lang; $config['target'] = $this->target_lang; $config['email'] = "*****@*****.**"; $config['id_user'] = array(); //get job's TM keys try { $tm_keys = $this->tm_keys; if (self::isRevision()) { $this->userRole = TmKeyManagement_Filter::ROLE_REVISOR; } elseif ($this->userMail == $job_data['owner']) { $tm_keys = TmKeyManagement_TmKeyManagement::getOwnerKeys(array($tm_keys), 'r', 'tm'); $tm_keys = json_encode($tm_keys); } //get TM keys with read grants $tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($tm_keys, 'r', 'tm', $this->uid, $this->userRole); if (is_array($tm_keys) && !empty($tm_keys)) { foreach ($tm_keys as $tm_key) { $config['id_user'][] = $tm_key->key; } } } catch (Exception $e) { $this->result['errors'][] = array("code" => -11, "message" => "Cannot retrieve TM keys info."); return; } //prepare the errors report $set_code = array(); /** * @var $tm_key TmKeyManagement_TmKeyStruct */ //if there's no key if (empty($tm_keys)) { //try deleting anyway, it may be a public segment and it may work $TMS_RESULT = $tms->delete($config); $set_code[] = $TMS_RESULT; } else { //loop over the list of keys foreach ($tm_keys as $tm_key) { //issue a separate call for each key $config['id_user'] = $tm_key->key; $TMS_RESULT = $tms->delete($config); $set_code[] = $TMS_RESULT; } } $set_successful = true; if (array_search(false, $set_code, true)) { //There's an errors $set_successful = false; } $this->result['data'] = $set_successful ? "OK" : null; $this->result['code'] = $set_successful; }
$amqHandlerSubscriber->incrementAnalyzedCount($pid, 0, 0); $amqHandlerSubscriber->decrementTotalForWaitingProjects($pid); $amqHandlerSubscriber->tryToCloseProject($pid, $my_pid); $amqHandlerSubscriber->ack($msg); continue; } //reset vectors $matches = array(); $tms_match = array(); $mt_result = array(); $_config = array(); $_config['segment'] = $text; $_config['source'] = $source; $_config['target'] = $target; $_config['email'] = "*****@*****.**"; $tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($objQueue['tm_keys'], 'r', 'tm'); if (is_array($tm_keys) && !empty($tm_keys)) { foreach ($tm_keys as $tm_key) { $_config['id_user'][] = $tm_key->key; } } $_config['num_result'] = 3; $id_mt_engine = $objQueue['id_mt_engine']; $id_tms = $objQueue['id_tms']; $_TMS = $id_tms; //request /** * Call Memory Server for matches if it's enabled */ $tms_enabled = false; if ($_TMS == 1) {
public function doAction() { if (!$this->concordance_search) { //execute these lines only in segment contribution search, //in case of user concordance search skip these lines //because segment can be optional if (empty($this->id_segment)) { $this->result['errors'][] = array("code" => -1, "message" => "missing id_segment"); } } if (is_null($this->text) || $this->text === '') { $this->result['errors'][] = array("code" => -2, "message" => "missing text"); } if (empty($this->id_job)) { $this->result['errors'][] = array("code" => -3, "message" => "missing id_job"); } if (empty($this->num_results)) { $this->num_results = INIT::$DEFAULT_NUM_RESULTS_FROM_TM; } if (!empty($this->result['errors'])) { return -1; } //get Job Infos, we need only a row of jobs ( split ) $this->jobData = getJobData($this->id_job, $this->password); $pCheck = new AjaxPasswordCheck(); //check for Password correctness if (empty($this->jobData) || !$pCheck->grantJobAccessByJobData($this->jobData, $this->password)) { $this->result['errors'][] = array("code" => -10, "message" => "wrong password"); return -1; } /* * string manipulation strategy * */ if (!$this->concordance_search) { // $this->text = CatUtils::view2rawxliff($this->text); $this->source = $this->jobData['source']; $this->target = $this->jobData['target']; } else { $regularExpressions = $this->tokenizeSourceSearch(); if ($this->switch_languages) { /* * * switch languages from user concordances search on the target language value * Example: * Job is in * source: it_IT, * target: de_DE * * user perform a right click for concordance help on a german word or phrase * we want result in italian from german source * */ $this->source = $this->jobData['target']; $this->target = $this->jobData['source']; } else { $this->source = $this->jobData['source']; $this->target = $this->jobData['target']; } } $this->id_mt_engine = $this->jobData['id_mt_engine']; $this->id_tms = $this->jobData['id_tms']; $this->tm_keys = $this->jobData['tm_keys']; $config = array(); if ($this->id_tms == 1) { /** * MyMemory Enabled */ $config['get_mt'] = true; $config['mt_only'] = false; if ($this->id_mt_engine != 1) { /** * Don't get MT contribution from MyMemory ( Custom MT ) */ $config['get_mt'] = false; } $_TMS = $this->id_tms; } else { if ($this->id_tms == 0 && $this->id_mt_engine == 1) { /** * MyMemory disabled but MT Enabled and it is NOT a Custom one * So tell to MyMemory to get MT only */ $config['get_mt'] = true; $config['mt_only'] = true; $_TMS = 1; /* MyMemory */ } } /** * if No TM server and No MT selected $_TMS is not defined * so we want not to perform TMS Call * */ if (isset($_TMS)) { /** * @var $tms Engines_MyMemory */ $tms = Engine::getInstance($_TMS); $config = array_merge($tms->getConfigStruct(), $config); $config['segment'] = $this->text; $config['source'] = $this->source; $config['target'] = $this->target; $config['email'] = INIT::$MYMEMORY_API_KEY; $config['id_user'] = array(); $config['num_result'] = $this->num_results; $config['isConcordance'] = $this->concordance_search; //get job's TM keys $this->checkLogin(); try { if (self::isRevision()) { $this->userRole = TmKeyManagement_Filter::ROLE_REVISOR; } $tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($this->tm_keys, 'r', 'tm', $this->uid, $this->userRole); if (is_array($tm_keys) && !empty($tm_keys)) { foreach ($tm_keys as $tm_key) { $config['id_user'][] = $tm_key->key; } } } catch (Exception $e) { $this->result['errors'][] = array("code" => -11, "message" => "Cannot retrieve TM keys info."); Log::doLog($e->getMessage()); return; } $tms_match = $tms->get($config); $tms_match = $tms_match->get_matches_as_array(); } if ($this->id_mt_engine > 1) { /** * @var $mt Engines_Moses */ $mt = Engine::getInstance($this->id_mt_engine); $config = $mt->getConfigStruct(); $config['segment'] = $this->text; $config['source'] = $this->source; $config['target'] = $this->target; $config['id_user'] = INIT::$MYMEMORY_API_KEY; $config['segid'] = $this->id_segment; $mt_result = $mt->get($config); if (isset($mt_result['error']['code'])) { $mt_result['error']['created_by_type'] = 'MT'; $this->result['errors'][] = $mt_result['error']; $mt_result = false; } } $matches = array(); if (!empty($tms_match)) { $matches = $tms_match; } if (!empty($mt_result)) { $matches[] = $mt_result; usort($matches, array("getContributionController", "__compareScore")); //this is necessary since usort sorts is ascending order, thus inverting the ranking $matches = array_reverse($matches); } $matches = array_slice($matches, 0, $this->num_results); isset($matches[0]['match']) ? $firstMatchVal = floatval($matches[0]['match']) : null; if (isset($firstMatchVal) && $firstMatchVal >= 90 && $firstMatchVal < 100) { $srcSearch = strip_tags($this->text); $segmentFound = strip_tags($matches[0]['raw_segment']); $srcSearch = mb_strtolower(preg_replace('#[\\x{20}]{2,}#u', chr(0x20), $srcSearch)); $segmentFound = mb_strtolower(preg_replace('#[\\x{20}]{2,}#u', chr(0x20), $segmentFound)); $fuzzy = levenshtein($srcSearch, $segmentFound) / log10(mb_strlen($srcSearch . $segmentFound) + 1); //levenshtein handle max 255 chars per string and returns -1, so fuzzy var can be less than 0 !! if ($srcSearch == $segmentFound || $fuzzy < 2.5 && $fuzzy >= 0) { $qaRealign = new QA($this->text, html_entity_decode($matches[0]['raw_translation'])); $qaRealign->tryRealignTagID(); $log_prepend = "CLIENT REALIGN IDS PROCEDURE | "; if (!$qaRealign->thereAreErrors()) { /* Log::doLog( $log_prepend . " - Requested Segment: " . var_export( $this->__postInput, true) ); Log::doLog( $log_prepend . "Fuzzy: " . $fuzzy . " - Try to Execute Tag ID Realignment." ); Log::doLog( $log_prepend . "TMS RAW RESULT:" ); Log::doLog( $log_prepend . var_export($matches[0], true) ); Log::doLog( $log_prepend . "Realignment Success:"); */ $matches[0]['segment'] = CatUtils::rawxliff2view($this->text); $matches[0]['translation'] = CatUtils::rawxliff2view($qaRealign->getTrgNormalized()); $matches[0]['match'] = $fuzzy == 0 ? '100%' : '99%'; /* Log::doLog( $log_prepend . "View Segment: " . var_export($matches[0]['segment'], true) ); Log::doLog( $log_prepend . "View Translation: " . var_export($matches[0]['translation'], true) ); */ } else { Log::doLog($log_prepend . 'Realignment Failed. Skip. Segment: ' . $this->__postInput['id_segment']); } } } /* New Feature only if this is not a MT and if it is a ( 90 =< MATCH < 100 ) */ if (!$this->concordance_search) { //execute these lines only in segment contribution search, //in case of user concordance search skip these lines $res = $this->setSuggestionReport($matches); if (is_array($res) and array_key_exists("error", $res)) { // error occurred } // } foreach ($matches as &$match) { if (strpos($match['created_by'], 'MT') !== false) { $match['match'] = 'MT'; $QA = new PostProcess($match['raw_segment'], $match['raw_translation']); $QA->realignMTSpaces(); //this should every time be ok because MT preserve tags, but we use the check on the errors //for logic correctness if (!$QA->thereAreErrors()) { $match['raw_translation'] = $QA->getTrgNormalized(); $match['translation'] = CatUtils::rawxliff2view($match['raw_translation']); } else { Log::doLog($QA->getErrors()); } } if ($match['created_by'] == 'MT!') { $match['created_by'] = 'MT'; //MyMemory returns MT! } else { $match['created_by'] = $this->__changeSuggestionSource($match); } if (!empty($match['sentence_confidence'])) { $match['sentence_confidence'] = round($match['sentence_confidence'], 0) . "%"; } if ($this->concordance_search) { $match['segment'] = strip_tags(html_entity_decode($match['segment'])); $match['segment'] = preg_replace('#[\\x{20}]{2,}#u', chr(0x20), $match['segment']); //Do something with &$match, tokenize strings and send to client $match['segment'] = preg_replace(array_keys($regularExpressions), array_values($regularExpressions), $match['segment']); $match['translation'] = strip_tags(html_entity_decode($match['translation'])); } } $this->result['data']['matches'] = $matches; }
public function testGetJobTmKeys_uidNotNull() { $tmKeys = TmKeyManagement_TmKeyManagement::getJobTmKeys(self::$validJsonTmKeyArrWithUidTranslator, 'rw', 'tm', self::$uid_translator, TmKeyManagement_Filter::ROLE_TRANSLATOR); $this->assertNotNull($tmKeys); $this->assertTrue(is_array($tmKeys)); $this->assertFalse(empty($tmKeys)); $this->assertInstanceOf("TmKeyManagement_TmKeyStruct", $tmKeys[0]); }
public function doAction() { if (is_null($this->source) || $this->source === '') { $this->result['errors'][] = array("code" => -1, "message" => "missing source segment"); } if (is_null($this->target) || $this->target === '') { $this->result['errors'][] = array("code" => -2, "message" => "missing target segment"); } if (empty($this->source_lang)) { $this->result['errors'][] = array("code" => -3, "message" => "missing source lang"); } if (empty($this->target_lang)) { $this->result['errors'][] = array("code" => -2, "message" => "missing target lang"); } if (empty($this->id_job)) { $this->result['errors'][] = array("code" => -4, "message" => "id_job not valid"); $msg = "\n\n Critical. Quit. \n\n " . var_export(array_merge($this->result, $_POST), true); Log::doLog($msg); Utils::sendErrMailReport($msg); // critical. Quit. return -1; } //get Job Infos, we need only a row of jobs ( split ) $job_data = getJobData((int) $this->id_job, $this->password); $pCheck = new AjaxPasswordCheck(); //check for Password correctness if (empty($job_data)) { $this->result['errors'][] = array("code" => -101, "message" => "error fetching job data"); } if (empty($this->result['errors']) && !$pCheck->grantJobAccessByJobData($job_data, $this->password)) { $this->result['errors'][] = array("code" => -10, "message" => "wrong password"); } if (!empty($this->result['errors'])) { $msg = "\n\n Error \n\n " . var_export(array_merge($this->result, $_POST), true); Log::doLog($msg); Utils::sendErrMailReport($msg); return -1; } $id_tms = $job_data['id_tms']; $tm_keys = $job_data['tm_keys']; if ($id_tms != 0) { $tms = Engine::getInstance(1); /** * @var $tms Engines_MyMemory */ $config = $tms->getConfigStruct(); $config['segment'] = CatUtils::view2rawxliff($this->source); $config['translation'] = CatUtils::view2rawxliff($this->target); $config['source'] = $this->source_lang; $config['target'] = $this->target_lang; $config['email'] = "*****@*****.**"; //Props $config['prop'] = json_encode(CatUtils::getTMProps($job_data)); //instantiate TMS object $result = array(); $this->checkLogin(); try { if (self::isRevision()) { $this->userRole = TmKeyManagement_Filter::ROLE_REVISOR; } //find all the job's TMs with write grants and make a contribution to them $tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($tm_keys, 'w', 'tm', $this->uid, $this->userRole); if (!empty($tm_keys)) { unset($config['id_user']); foreach ($tm_keys as $i => $tm_info) { $config['id_user'] = $tm_info->key; $res = $tms->set($config); if (!$res) { $result[] = $res; $this->result['errors'][] = array("code" => -5, "message" => "Set contribution error for key " . $tm_info->name); } } } else { $res = $tms->set($config); if (!$res) { $result[] = $res; $this->result['errors'][] = array("code" => -5, "message" => "Set contribution error"); } } } catch (Exception $e) { $this->result['errors'][] = array("code" => -6, "message" => "Error while retrieving job's TM."); Log::doLog(__METHOD__ . " -> " . $e->getMessage()); } //if translator_username is empty no key is added to MyMemory API SET query string, so, anonymous by default if (count($result)) { $this->result['code'] = -1; $this->result['data'] = "KO"; Log::doLog("Set Contribution Failed."); Log::doLog(var_export($_POST, true)); return -1; } $this->result['code'] = 1; $this->result['data'] = "OK"; } else { $this->result['code'] = 1; $this->result['data'] = "NOCONTRIB_OK"; } }
/** * Get matches from MyMemory and other engines * * @param $queueElement QueueElement * * @return array * @throws Exception */ protected function _getMatches(QueueElement $queueElement) { $_config = array(); $_config['segment'] = $queueElement->params->segment; $_config['source'] = $queueElement->params->source; $_config['target'] = $queueElement->params->target; $_config['email'] = \INIT::$MYMEMORY_TM_API_KEY; $tm_keys = \TmKeyManagement_TmKeyManagement::getJobTmKeys($queueElement->params->tm_keys, 'r', 'tm'); if (is_array($tm_keys) && !empty($tm_keys)) { foreach ($tm_keys as $tm_key) { $_config['id_user'][] = $tm_key->key; } } $_config['num_result'] = 3; $id_mt_engine = $queueElement->params->id_mt_engine; $id_tms = $queueElement->params->id_tms; $_TMS = $id_tms; //request /** * Call Memory Server for matches if it's enabled */ $tms_enabled = false; if ($_TMS == 1) { /** * MyMemory Enabled */ $_config['get_mt'] = true; $_config['mt_only'] = false; if ($id_mt_engine != 1) { /** * Don't get MT contribution from MyMemory ( Custom MT ) */ $_config['get_mt'] = false; } $tms_enabled = true; } elseif ($_TMS == 0 && $id_mt_engine == 1) { /** * MyMemory disabled but MT Enabled and it is NOT a Custom one * So tell to MyMemory to get MT only */ $_config['get_mt'] = true; $_config['mt_only'] = true; $_TMS = 1; /* MyMemory */ $tms_enabled = true; } /* * This will be ever executed without damages because * fastAnalysis set Project as DONE when * MyMemory is disabled and MT is Disabled Too * * So don't worry, perform TMS Analysis * */ if ($tms_enabled) { /** * @var $tms \Engines_MyMemory */ $tms = \Engine::getInstance($_TMS); $tms->doLog = false; $config = $tms->getConfigStruct(); $config = array_merge($config, $_config); $tms_match = $tms->get($config); /** * If No results found. Re-Queue * * MyMemory can return null if an error occurs (e.g http response code is 404, 410, 500, 503, etc.. ) */ if ($tms_match === null) { $this->_doLog("--- (Worker " . $this->_workerPid . ") : Error from MyMemory. NULL received."); throw new ReQueueException("--- (Worker " . $this->_workerPid . ") : Error from MyMemory. NULL received.", self::ERR_REQUEUE); } $tms_match = $tms_match->get_matches_as_array(); } /** * Call External MT engine if it is a custom one ( mt not requested from MyMemory ) */ if ($id_mt_engine > 1) { try { $mt = \Engine::getInstance($id_mt_engine); $config = $mt->getConfigStruct(); $config = array_merge($config, $_config); $mt_result = $mt->get($config); if (isset($mt_result['error']['code'])) { $mt_result = false; } } catch (\Exception $e) { $this->_doLog($e->getMessage()); } } $matches = array(); if (!empty($tms_match)) { $matches = $tms_match; } if (isset($mt_result) && !empty($mt_result)) { $matches[] = $mt_result; usort($matches, "Executor::_compareScore"); } /** * If No results found. Re-Queue */ if (empty($matches) || !is_array($matches)) { $this->_doLog("--- (Worker " . $this->_workerPid . ") : No contribution found for this segment."); $this->_forceSetSegmentAnalyzed($queueElement); throw new EmptyElementException("--- (Worker " . $this->_workerPid . ") : No contribution found for this segment.", self::ERR_EMPTY_ELEMENT); } return $matches; }
protected function _delete($config) { $tm_keys = $this->job_info['tm_keys']; if (self::isRevision()) { $this->userRole = TmKeyManagement_Filter::ROLE_REVISOR; } //get TM keys with read grants $tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($tm_keys, 'w', 'glos', $this->uid, $this->userRole); $config['segment'] = CatUtils::view2rawxliff($config['segment']); $config['translation'] = CatUtils::view2rawxliff($config['translation']); //prepare the error report $set_code = array(); //set the glossary entry for each key with write grants if (count($tm_keys)) { /** * @var $tm_key TmKeyManagement_TmKeyStruct */ foreach ($tm_keys as $tm_key) { $config['id_user'] = $tm_key->key; $TMS_RESULT = $this->_TMS->delete($config); $set_code[] = $TMS_RESULT; } } $set_successful = true; if (array_search(false, $set_code, true)) { $set_successful = false; } $this->result['code'] = $set_successful; $this->result['data'] = $set_successful ? 'OK' : null; }
$pretranslate_100 = $segment['pretranslate_100']; $text = $segment['segment']; if ($raw_wc == 0) { echo "--- (child {$my_pid}) : empty segment. deleting lock and continue\n"; continue; } //reset vectors $matches = array(); $tms_match = array(); $mt_result = array(); $_config = array(); $_config['segment'] = $text; $_config['source'] = $source; $_config['target'] = $target; $_config['email'] = "*****@*****.**"; $tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($segment['tm_keys'], 'r', 'tm'); if (is_array($tm_keys) && !empty($tm_keys)) { foreach ($tm_keys as $tm_key) { $_config['id_user'][] = $tm_key->key; } } $_config['num_result'] = 3; $id_mt_engine = $segment['id_mt_engine']; $id_tms = $segment['id_tms']; $_TMS = $id_tms; //request /** * Call Memory Server for matches if it's enabled */ $tms_enabled = false; if ($_TMS == 1) {