public function getData($job_id, $password, $options = array())
{
$data = getJobData($job_id, $password);
$wCounter = new WordCount_Counter();
$wStruct = $wCounter->initializeJobWordCount($job_id, $password);
$jobQA = new Revise_JobQA($job_id, $password, $wStruct->getTotal());
$jobQA->retrieveJobErrorTotals();
$jobQA->evalJobVote();
$jobVote = $jobQA->getJobVote();
return array('job_id' => $job_id, 'quality_details' => $jobQA->getQaData(), 'quality_overall' => $jobVote['minText']);
}
/**
* @param WordCount_Struct $oldWCount
*/
public function __construct(WordCount_Struct $oldWCount = null)
{
$reflect = new ReflectionClass('Constants_TranslationStatus');
self::$constCache = array_flip($reflect->getConstants());
if ($oldWCount !== null) {
$this->setOldWordCount($oldWCount);
}
}
public function mergeALL(ArrayObject $projectStructure, $renewPassword = false)
{
$query_job = "SELECT *\n FROM jobs\n WHERE id = %u\n ORDER BY job_first_segment";
$query_job = sprintf($query_job, $projectStructure['job_to_merge']);
//$projectStructure[ 'job_to_split' ]
$rows = $this->dbHandler->fetch_array($query_job);
//get the min and
$first_job = reset($rows);
$job_first_segment = $first_job['job_first_segment'];
//the max segment from job list
$last_job = end($rows);
$job_last_segment = $last_job['job_last_segment'];
//change values of first job
$first_job['job_first_segment'] = $job_first_segment;
// redundant
$first_job['job_last_segment'] = $job_last_segment;
//merge TM keys: preserve only owner's keys
$tm_keys = array();
foreach ($rows as $chunk_info) {
$tm_keys[] = $chunk_info['tm_keys'];
}
try {
$owner_tm_keys = TmKeyManagement_TmKeyManagement::getOwnerKeys($tm_keys);
/**
* @var $owner_key TmKeyManagement_TmKeyStruct
*/
foreach ($owner_tm_keys as $i => $owner_key) {
$owner_tm_keys[$i] = $owner_key->toArray();
}
$first_job['tm_keys'] = json_encode($owner_tm_keys);
} catch (Exception $e) {
Log::doLog(__METHOD__ . " -> Merge Jobs error - TM key problem: " . $e->getMessage());
}
$oldPassword = $first_job['password'];
if ($renewPassword) {
$first_job['password'] = self::_generatePassword();
}
$_data = array();
foreach ($first_job as $field => $value) {
$_data[] = "`{$field}`='{$value}'";
}
//----------------------------------------------------
$queries = array();
$queries[] = "UPDATE jobs SET " . implode(", \n", $_data) . " WHERE id = {$first_job['id']} AND password = '******'";
//ose old password
//delete all old jobs
$queries[] = "DELETE FROM jobs WHERE id = {$first_job['id']} AND password != '{$first_job['password']}' ";
//use new password
foreach ($queries as $query) {
$res = $this->dbHandler->query($query);
if ($res !== true) {
$msg = "Failed to merge job " . $rows[0]['id'] . " from " . count($rows) . " chunks\n";
$msg .= "Tried to perform SQL: \n" . print_r($queries, true) . " \n\n";
$msg .= "Failed Statement is: \n" . print_r($query, true) . "\n";
$msg .= "Original Status for rebuild job and project was: \n" . print_r($rows, true) . "\n";
Utils::sendErrMailReport($msg);
throw new Exception('Failed to merge jobs, project damaged. Contact Matecat Support to rebuild project.', -8);
}
}
$wCountManager = new WordCount_Counter();
$wCountManager->initializeJobWordCount($first_job['id'], $first_job['password']);
Shop_Cart::getInstance('outsource_to_external_cache')->emptyCart();
}
public function tryToCloseProject($pid, $child_process_id)
{
if (!empty($pid)) {
$_pid = $pid;
} else {
$_pid = $this->queueTotalID;
}
$project_totals = array();
$project_totals['project_segments'] = $this->getRedisClient()->get(Constants_AnalysisRedisKeys::PROJECT_TOT_SEGMENTS . $pid);
$project_totals['num_analyzed'] = $this->getRedisClient()->get(Constants_AnalysisRedisKeys::PROJECT_NUM_SEGMENTS_DONE . $pid);
$project_totals['eq_wc'] = $this->getRedisClient()->get(Constants_AnalysisRedisKeys::PROJ_EQ_WORD_COUNT . $pid) / 1000;
$project_totals['st_wc'] = $this->getRedisClient()->get(Constants_AnalysisRedisKeys::PROJ_ST_WORD_COUNT . $pid) / 1000;
Log::doLog("--- (child {$child_process_id}) : count segments in project {$pid} = " . $project_totals['project_segments'] . "");
Log::doLog("--- (child {$child_process_id}) : Analyzed segments in project {$pid} = " . $project_totals['num_analyzed'] . "");
if (empty($project_totals['project_segments'])) {
Log::doLog("--- (child {$child_process_id}) : WARNING !!! error while counting segments in projects {$pid} skipping and continue ");
return;
}
if ($project_totals['project_segments'] - $project_totals['num_analyzed'] == 0 && $this->getRedisClient()->setnx(Constants_AnalysisRedisKeys::PROJECT_ENDING_SEMAPHORE . $pid, 1)) {
$this->getRedisClient()->expire(Constants_AnalysisRedisKeys::PROJECT_ENDING_SEMAPHORE . $pid, 60 * 60 * 24);
$_analyzed_report = getProjectSegmentsTranslationSummary($pid);
$total_segs = array_pop($_analyzed_report);
//remove Rollup
Log::doLog("--- (child {$child_process_id}) : analysis project {$pid} finished : change status to DONE");
changeProjectStatus($pid, Constants_ProjectStatus::STATUS_DONE);
changeTmWc($pid, $project_totals['eq_wc'], $project_totals['st_wc']);
/*
* Remove this job from the project list
*/
$this->getRedisClient()->lrem(Constants_AnalysisRedisKeys::PROJECTS_QUEUE_LIST, 0, $_pid);
Log::doLog("--- (child {$child_process_id}) : trying to initialize job total word count.");
foreach ($_analyzed_report as $job_info) {
$counter = new WordCount_Counter();
$counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
}
}
}
function insertFastAnalysis($pid, &$fastReport, $equivalentWordMapping, $perform_Tms_Analysis = true)
{
$db = Database::obtain();
$data = array();
$amqHandler = new Analysis_QueueHandler();
$total_eq_wc = 0;
$total_standard_wc = 0;
$data['id_segment'] = null;
$data['id_job'] = null;
$data['segment_hash'] = null;
$data['match_type'] = null;
$data['eq_word_count'] = null;
$data['standard_word_count'] = null;
$segment_translations = "INSERT INTO `segment_translations` ( " . implode(", ", array_keys($data)) . " ) VALUES ";
$st_values = array();
foreach ($fastReport as $k => $v) {
$jid_fid = explode("-", $k);
$id_segment = $jid_fid[0];
$list_id_jobs_password = $jid_fid[1];
if (array_key_exists($v['match_type'], $equivalentWordMapping)) {
$eq_word = $v['wc'] * $equivalentWordMapping[$v['match_type']] / 100;
if ($v['match_type'] == "INTERNAL") {
}
} else {
$eq_word = $v['wc'];
}
$total_eq_wc += $eq_word;
$standard_words = $eq_word;
if ($v['match_type'] == "INTERNAL" or $v['match_type'] == "MT") {
$standard_words = $v['wc'] * $equivalentWordMapping["NO_MATCH"] / 100;
}
$total_standard_wc += $standard_words;
unset($fastReport[$k]['wc']);
$list_id_jobs_password = explode(',', $list_id_jobs_password);
foreach ($list_id_jobs_password as $id_job) {
list($id_job, $job_pass) = explode(":", $id_job);
$data['id_job'] = (int) $id_job;
$data['id_segment'] = (int) $fastReport[$k]['id_segment'];
$data['segment_hash'] = $db->escape($v['segment_hash']);
$data['match_type'] = $db->escape($v['match_type']);
if (!empty($v['segment_hash']) && empty($data['segment_hash'])) {
$data['segment_hash'] = $v['segment_hash'];
$msg = "mysql_real_escape_string failed!!! String was empty. Replaced with original {$v['segment_hash']}";
_TimeStampMsg($msg);
Utils::sendErrMailReport("<strong>{$msg}</strong>", "Fast Analysis mysql_real_escape_string failed.");
}
$data['eq_word_count'] = (double) $eq_word;
$data['standard_word_count'] = (double) $standard_words;
$st_values[] = " ( '" . implode("', '", array_values($data)) . "' )";
if ($data['eq_word_count'] > 0 && $perform_Tms_Analysis) {
/**
*
* IMPORTANT
* id_job will be taken from languages ( 80415:fr-FR,80416:it-IT )
*/
$fastReport[$k]['pid'] = (int) $pid;
$fastReport[$k]['date_insert'] = date_create()->format('Y-m-d H:i:s');
$fastReport[$k]['eq_word_count'] = (double) $eq_word;
$fastReport[$k]['standard_word_count'] = (double) $standard_words;
} else {
// Log::doLog( 'Skipped Fast Segment: ' . var_export( $fastReport[ $k ], true ) );
unset($fastReport[$k]);
}
}
}
unset($data);
$chunks_st = array_chunk($st_values, 200);
_TimeStampMsg('Insert Segment Translations: ' . count($st_values));
_TimeStampMsg('Queries: ' . count($chunks_st));
//USE the MySQL InnoDB isolation Level to protect from thread high concurrency access
$db->query('SET autocommit=0');
$db->query('START TRANSACTION');
foreach ($chunks_st as $k => $chunk) {
$query_st = $segment_translations . implode(", ", $chunk) . " ON DUPLICATE KEY UPDATE\n match_type = VALUES( match_type ),\n eq_word_count = VALUES( eq_word_count ),\n standard_word_count = VALUES( standard_word_count )\n ";
$db->query($query_st);
_TimeStampMsg("Executed " . ($k + 1));
$err = $db->get_error();
if ($err['error_code'] != 0) {
_TimeStampMsg($err);
return $err['error_code'] * -1;
}
}
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($st_values);
unset($chunks_st);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
/*
* IF NO TM ANALYSIS, upload the jobs global word count
*/
if (!$perform_Tms_Analysis) {
$_details = getProjectSegmentsTranslationSummary($pid);
_TimeStampMsg("--- trying to initialize job total word count.");
$project_details = array_pop($_details);
//remove rollup
foreach ($_details as $job_info) {
$counter = new WordCount_Counter();
$counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
}
}
/* IF NO TM ANALYSIS, upload the jobs global word count */
//_TimeStampMsg( "Done." );
$data2 = array('fast_analysis_wc' => $total_eq_wc);
$where = " id = {$pid}";
$db->update('projects', $data2, $where);
$err = $db->get_error();
$errno = $err['error_code'];
if ($errno != 0) {
$db->query('ROLLBACK');
$db->query('SET autocommit=1');
_TimeStampMsg($err);
return $errno * -1;
}
$db->query('COMMIT');
$db->query('SET autocommit=1');
if (count($fastReport)) {
// $chunks_st_queue = array_chunk( $fastReport, 10 );
_TimeStampMsg('Insert Segment Translations Queue: ' . count($fastReport));
_TimeStampMsg('Queries: ' . count($fastReport));
try {
$amqHandler->setTotal(array('qid' => $pid, 'queueName' => INIT::$QUEUE_NAME));
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set Total values failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
$time_start = microtime(true);
foreach ($fastReport as $k => $queue_element) {
try {
$languages_job = explode(",", $queue_element['target']);
//now target holds more than one language ex: ( 80415:fr-FR,80416:it-IT )
//in memory replacement avoid duplication of the segment list
//send in queue every element * number of languages
foreach ($languages_job as $_language) {
list($id_job, $language) = explode(":", $_language);
$queue_element['target'] = $language;
$queue_element['id_job'] = $id_job;
$jsonObj = json_encode($queue_element);
Utils::raiseJsonExceptionError();
$amqHandler->send(INIT::$QUEUE_NAME, $jsonObj, array('persistent' => $amqHandler->persistent));
_TimeStampMsg("AMQ Set Executed " . ($k + 1));
}
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set queue failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
}
_TimeStampMsg('Done in ' . (microtime(true) - $time_start) . " seconds.");
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($fastReport);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
}
$amqHandler->disconnect();
return $db->affected_rows;
}
public function doAction()
{
$files_found = array();
$lang_handler = Langs_Languages::getInstance();
$data = getSegmentsInfo($this->jid, $this->password);
if (empty($data) or $data < 0) {
$this->job_not_found = true;
//stop execution
return;
}
//retrieve job owner. It will be useful also if the job is archived or cancelled
$this->job_owner = $data[0]['job_owner'] != "" ? $data[0]['job_owner'] : "*****@*****.**";
if ($data[0]['status'] == Constants_JobStatus::STATUS_CANCELLED) {
$this->job_cancelled = true;
//stop execution
return;
}
if ($data[0]['status'] == Constants_JobStatus::STATUS_ARCHIVED) {
$this->job_archived = true;
//stop execution
return;
}
/*
* I prefer to use a programmatic approach to the check for the archive date instead of a pure query
* because the query to check "Utils::getArchivableJobs($this->jid)" should be
* executed every time a job is loaded ( F5 or CTRL+R on browser ) and it cost some milliseconds ( ~0.1s )
* and it is little heavy for the database.
* We use the data we already have from last query and perform
* the check on the last translation only if the job is older than 30 days
*
*/
$lastUpdate = new DateTime($data[0]['last_update']);
$oneMonthAgo = new DateTime();
$oneMonthAgo->modify('-' . INIT::JOB_ARCHIVABILITY_THRESHOLD . ' days');
if ($lastUpdate < $oneMonthAgo && !$this->job_cancelled) {
$lastTranslationInJob = new Datetime(getLastTranslationDate($this->jid));
if ($lastTranslationInJob < $oneMonthAgo) {
$res = "job";
$new_status = Constants_JobStatus::STATUS_ARCHIVED;
updateJobsStatus($res, $this->jid, $new_status, null, null, $this->password);
$this->job_archived = true;
}
}
foreach ($data as $i => $job) {
$this->project_status = $job;
// get one row values for the project are the same for every row
if (empty($this->pname)) {
$this->pname = $job['pname'];
$this->downloadFileName = $job['pname'] . ".zip";
// will be overwritten below in case of one file job
}
if (empty($this->last_opened_segment)) {
$this->last_opened_segment = $job['last_opened_segment'];
}
if (empty($this->cid)) {
$this->cid = $job['cid'];
}
if (empty($this->pid)) {
$this->pid = $job['pid'];
}
if (empty($this->create_date)) {
$this->create_date = $job['create_date'];
}
if (empty($this->source_code)) {
$this->source_code = $job['source'];
}
if (empty($this->target_code)) {
$this->target_code = $job['target'];
}
if (empty($this->source)) {
$s = explode("-", $job['source']);
$source = strtoupper($s[0]);
$this->source = $source;
$this->source_rtl = $lang_handler->isRTL(strtolower($this->source)) ? ' rtl-source' : '';
}
if (empty($this->target)) {
$t = explode("-", $job['target']);
$target = strtoupper($t[0]);
$this->target = $target;
$this->target_rtl = $lang_handler->isRTL(strtolower($this->target)) ? ' rtl-target' : '';
}
//check if language belongs to supported right-to-left languages
if ($job['status'] == Constants_JobStatus::STATUS_ARCHIVED) {
$this->job_archived = true;
$this->job_owner = $data[0]['job_owner'];
}
$id_file = $job['id_file'];
if (!isset($this->data["{$id_file}"])) {
$files_found[] = $job['filename'];
}
$wStruct = new WordCount_Struct();
$wStruct->setIdJob($this->jid);
$wStruct->setJobPassword($this->password);
$wStruct->setNewWords($job['new_words']);
$wStruct->setDraftWords($job['draft_words']);
$wStruct->setTranslatedWords($job['translated_words']);
$wStruct->setApprovedWords($job['approved_words']);
$wStruct->setRejectedWords($job['rejected_words']);
unset($job['id_file']);
unset($job['source']);
unset($job['target']);
unset($job['source_code']);
unset($job['target_code']);
unset($job['mime_type']);
unset($job['filename']);
unset($job['jid']);
unset($job['pid']);
unset($job['cid']);
unset($job['tid']);
unset($job['pname']);
unset($job['create_date']);
unset($job['owner']);
unset($job['last_opened_segment']);
unset($job['new_words']);
unset($job['draft_words']);
unset($job['translated_words']);
unset($job['approved_words']);
unset($job['rejected_words']);
//For projects created with No tm analysis enabled
if ($wStruct->getTotal() == 0 && ($job['status_analysis'] == Constants_ProjectStatus::STATUS_DONE || $job['status_analysis'] == Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE)) {
$wCounter = new WordCount_Counter();
$wStruct = $wCounter->initializeJobWordCount($this->jid, $this->password);
Log::doLog("BackWard compatibility set Counter.");
}
$this->job_stats = CatUtils::getFastStatsForJob($wStruct);
}
//Needed because a just created job has last_opened segment NULL
if (empty($this->last_opened_segment)) {
$this->last_opened_segment = getFirstSegmentId($this->jid, $this->password);
}
$this->first_job_segment = $this->project_status['job_first_segment'];
$this->last_job_segment = $this->project_status['job_last_segment'];
if (count($files_found) == 1) {
$this->downloadFileName = $files_found[0];
}
/**
* get first segment of every file
*/
$fileInfo = getFirstSegmentOfFilesInJob($this->jid);
$TotalPayable = array();
foreach ($fileInfo as &$file) {
$file['file_name'] = ZipArchiveExtended::getFileName($file['file_name']);
$TotalPayable[$file['id_file']]['TOTAL_FORMATTED'] = $file['TOTAL_FORMATTED'];
}
$this->firstSegmentOfFiles = json_encode($fileInfo);
$this->fileCounter = json_encode($TotalPayable);
list($uid, $user_email) = $this->getLoginUserParams();
if (self::isRevision()) {
$this->userRole = TmKeyManagement_Filter::ROLE_REVISOR;
} elseif ($user_email == $data[0]['job_owner']) {
$this->userRole = TmKeyManagement_Filter::OWNER;
} else {
$this->userRole = TmKeyManagement_Filter::ROLE_TRANSLATOR;
}
/*
* Take the keys of the user
*/
try {
$_keyList = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => $uid));
$keyList = $_keyList->read($dh);
} catch (Exception $e) {
$keyList = array();
Log::doLog($e->getMessage());
}
$reverse_lookup_user_personal_keys = array('pos' => array(), 'elements' => array());
/**
* Set these keys as editable for the client
*
* @var $keyList TmKeyManagement_MemoryKeyStruct[]
*/
foreach ($keyList as $_j => $key) {
/**
* @var $_client_tm_key TmKeyManagement_TmKeyStruct
*/
//create a reverse lookup
$reverse_lookup_user_personal_keys['pos'][$_j] = $key->tm_key->key;
$reverse_lookup_user_personal_keys['elements'][$_j] = $key;
$this->_keyList['totals'][$_j] = new TmKeyManagement_ClientTmKeyStruct($key->tm_key);
}
/*
* Now take the JOB keys
*/
$job_keyList = json_decode($data[0]['tm_keys'], true);
$this->tid = count($job_keyList) > 0;
/**
* Start this N^2 cycle from keys of the job,
* these should be statistically lesser than the keys of the user
*
* @var $keyList array
*/
foreach ($job_keyList as $jobKey) {
$jobKey = new TmKeyManagement_ClientTmKeyStruct($jobKey);
if ($this->isLoggedIn() && count($reverse_lookup_user_personal_keys['pos'])) {
/*
* If user has some personal keys, check for the job keys if they are present, and obfuscate
* when they are not
*/
$_index_position = array_search($jobKey->key, $reverse_lookup_user_personal_keys['pos']);
if ($_index_position !== false) {
//i found a key in the job that is present in my database
//i'm owner?? and the key is an owner type key?
if (!$jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) {
$jobKey->r = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['r']};
$jobKey->w = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['w']};
$jobKey = $jobKey->hideKey($uid);
} else {
if ($jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) {
// I'm not the job owner, but i know the key because it is in my keyring
// so, i can upload and download TMX, but i don't want it to be removed from job
// in tm.html relaxed the control to "key.edit" to enable buttons
// $jobKey = $jobKey->hideKey( $uid ); // enable editing
} else {
if ($jobKey->owner && $this->userRole == TmKeyManagement_Filter::OWNER) {
//do Nothing
}
}
}
unset($this->_keyList['totals'][$_index_position]);
} else {
/*
* This is not a key of that user, set right and obfuscate
*/
$jobKey->r = true;
$jobKey->w = true;
$jobKey = $jobKey->hideKey(-1);
}
$this->_keyList['job_keys'][] = $jobKey;
} else {
/*
* This user is anonymous or it has no keys in its keyring, obfuscate all
*/
$jobKey->r = true;
$jobKey->w = true;
$this->_keyList['job_keys'][] = $jobKey->hideKey(-1);
}
}
//clean unordered keys
$this->_keyList['totals'] = array_values($this->_keyList['totals']);
/**
* Retrieve information about job errors
* ( Note: these information are fed by the revision process )
* @see setRevisionController
*/
$jobQA = new Revise_JobQA($this->jid, $this->password, $wStruct->getTotal());
$jobQA->retrieveJobErrorTotals();
$jobVote = $jobQA->evalJobVote();
$this->qa_data = json_encode($jobQA->getQaData());
$this->qa_overall = $jobVote['minText'];
$engine = new EnginesModel_EngineDAO(Database::obtain());
//this gets all engines of the user
if ($this->isLoggedIn()) {
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->type = 'MT';
$engineQuery->uid = $uid;
$engineQuery->active = 1;
$mt_engines = $engine->read($engineQuery);
} else {
$mt_engines = array();
}
// this gets MyMemory
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->type = 'TM';
$engineQuery->active = 1;
$tms_engine = $engine->setCacheTTL(3600 * 24 * 30)->read($engineQuery);
//this gets MT engine active for the job
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->id = $this->project_status['id_mt_engine'];
$engineQuery->active = 1;
$active_mt_engine = $engine->setCacheTTL(60 * 10)->read($engineQuery);
/*
* array_unique cast EnginesModel_EngineStruct to string
*
* EnginesModel_EngineStruct implements __toString method
*
*/
$this->translation_engines = array_unique(array_merge($active_mt_engine, $tms_engine, $mt_engines));
}
public function doAction()
{
try {
$this->_checkData();
} catch (Exception $e) {
if ($e->getCode() == -1) {
Utils::sendErrMailReport($e->getMessage());
}
Log::doLog($e->getMessage());
return $e->getCode();
}
//check tag mismatch
//get original source segment, first
$segment = getSegment($this->id_segment);
//compare segment-translation and get results
$check = new QA($segment['segment'], $this->translation);
$check->performConsistencyCheck();
if ($check->thereAreWarnings()) {
$err_json = $check->getWarningsJSON();
$translation = $this->translation;
} else {
$err_json = '';
$translation = $check->getTrgNormalized();
}
/*
* begin stats counter
*
* It works good with default InnoDB Isolation level
*
* REPEATABLE-READ offering a row level lock for this id_segment
*
*/
$db = Database::obtain();
$db->begin();
$old_translation = getCurrentTranslation($this->id_job, $this->id_segment);
// $old_version = ( empty( $old_translation['translation_date'] ) ? '0' : date_create( $old_translation['translation_date'] )->getTimestamp() );
// if( $this->client_target_version != $old_version ){
// $this->result[ 'errors' ][ ] = array( "code" => -102, "message" => "Translation version mismatch" );
// $db->rollback();
// return false;
// }
//if volume analysis is not enabled and no translation rows exists
//create the row
if (!INIT::$VOLUME_ANALYSIS_ENABLED && empty($old_translation['status'])) {
$_Translation = array();
$_Translation['id_segment'] = (int) $this->id_segment;
$_Translation['id_job'] = (int) $this->id_job;
$_Translation['status'] = Constants_TranslationStatus::STATUS_NEW;
$_Translation['segment_hash'] = $segment['segment_hash'];
$_Translation['translation'] = $segment['segment'];
$_Translation['standard_word_count'] = $segment['raw_word_count'];
$_Translation['serialized_errors_list'] = '';
$_Translation['suggestion_position'] = 0;
$_Translation['warning'] = false;
$_Translation['translation_date'] = date("Y-m-d H:i:s");
$res = addTranslation($_Translation);
if ($res < 0) {
$this->result['errors'][] = array("code" => -101, "message" => "database errors");
$db->rollback();
return $res;
}
/*
* begin stats counter
*
*/
$old_translation = $_Translation;
}
$_Translation = array();
$_Translation['id_segment'] = $this->id_segment;
$_Translation['id_job'] = $this->id_job;
$_Translation['status'] = $this->status;
$_Translation['time_to_edit'] = $this->time_to_edit;
$_Translation['translation'] = preg_replace('/[ \\t\\n\\r\\0\\x0A\\xA0]+$/u', '', $translation);
$_Translation['serialized_errors_list'] = $err_json;
$_Translation['suggestion_position'] = $this->chosen_suggestion_index;
$_Translation['warning'] = $check->thereAreWarnings();
$_Translation['translation_date'] = date("Y-m-d H:i:s");
/**
* when the status of the translation changes, the auto propagation flag
* must be removed
*/
if ($_Translation['translation'] != $old_translation['translation'] || $this->status == Constants_TranslationStatus::STATUS_TRANSLATED || $this->status == Constants_TranslationStatus::STATUS_APPROVED) {
$_Translation['autopropagated_from'] = 'NULL';
}
$res = CatUtils::addSegmentTranslation($_Translation);
if (!empty($res['errors'])) {
$this->result['errors'] = $res['errors'];
$msg = "\n\n Error addSegmentTranslation \n\n Database Error \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
$db->rollback();
return -1;
}
if (INIT::$DQF_ENABLED && !empty($this->jobData['dqf_key']) && $_Translation['status'] == Constants_TranslationStatus::STATUS_TRANSLATED) {
$dqfSegmentStruct = DQF_DqfSegmentStruct::getStruct();
if ($old_translation['suggestion'] == null) {
$dqfSegmentStruct->target_segment = "";
$dqfSegmentStruct->tm_match = 0;
} else {
$dqfSegmentStruct->target_segment = $old_translation['suggestion'];
$dqfSegmentStruct->tm_match = $old_translation['suggestion_match'];
}
$dqfSegmentStruct->task_id = $this->id_job;
$dqfSegmentStruct->segment_id = $this->id_segment;
$dqfSegmentStruct->source_segment = $segment['segment'];
$dqfSegmentStruct->new_target_segment = $_Translation['translation'];
$dqfSegmentStruct->time = $_Translation['time_to_edit'];
// $dqfSegmentStruct->mtengine = $this->jobData['id_mt_engine'];
$dqfSegmentStruct->mt_engine_version = 1;
try {
$dqfQueueHandler = new Analysis_DqfQueueHandler();
$dqfQueueHandler->createSegment($dqfSegmentStruct);
} catch (Exception $exn) {
$msg = $exn->getMessage() . "\n\n" . $exn->getTraceAsString();
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
$propagateToTranslated = true;
//for the moment, this is set explicitely
if ($this->propagate == false) {
$propagateToTranslated = false;
}
//propagate translations
$TPropagation = array();
//Warning: this value will NOT be used to update values,
//but to exclude current segment from auto-propagation
$_idSegment = $this->id_segment;
$propagateToTranslated ? $TPropagation['status'] = $this->status : null;
$TPropagation['id_job'] = $this->id_job;
$TPropagation['translation'] = $translation;
$TPropagation['autopropagated_from'] = $this->id_segment;
$TPropagation['serialized_errors_list'] = $err_json;
$TPropagation['warning'] = $check->thereAreWarnings();
// $TPropagation[ 'translation_date' ] = date( "Y-m-d H:i:s" );
$TPropagation['segment_hash'] = $old_translation['segment_hash'];
$propagationTotal = array();
if ($propagateToTranslated && in_array($this->status, array(Constants_TranslationStatus::STATUS_TRANSLATED, Constants_TranslationStatus::STATUS_APPROVED, Constants_TranslationStatus::STATUS_REJECTED))) {
try {
$propagationTotal = propagateTranslation($TPropagation, $this->jobData, $_idSegment, $propagateToTranslated);
} catch (Exception $e) {
$msg = $e->getMessage() . "\n\n" . $e->getTraceAsString();
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
//Recount Job Totals
$old_wStruct = new WordCount_Struct();
$old_wStruct->setIdJob($this->id_job);
$old_wStruct->setJobPassword($this->password);
$old_wStruct->setNewWords($this->jobData['new_words']);
$old_wStruct->setDraftWords($this->jobData['draft_words']);
$old_wStruct->setTranslatedWords($this->jobData['translated_words']);
$old_wStruct->setApprovedWords($this->jobData['approved_words']);
$old_wStruct->setRejectedWords($this->jobData['rejected_words']);
$old_wStruct->setIdSegment($this->id_segment);
//redundant, this is made into WordCount_Counter::updateDB
$old_wStruct->setOldStatus($old_translation['status']);
$old_wStruct->setNewStatus($this->status);
//redundant because the update is made only where status = old status
if ($this->status != $old_translation['status']) {
//cambiato status, sposta i conteggi
$old_count = !is_null($old_translation['eq_word_count']) ? $old_translation['eq_word_count'] : $segment['raw_word_count'];
//if there is not a row in segment_translations because volume analysis is disabled
//search for a just created row
$old_status = !empty($old_translation['status']) ? $old_translation['status'] : Constants_TranslationStatus::STATUS_NEW;
$counter = new WordCount_Counter($old_wStruct);
$counter->setOldStatus($old_status);
$counter->setNewStatus($this->status);
$newValues = array();
$newValues[] = $counter->getUpdatedValues($old_count);
foreach ($propagationTotal as $__pos => $old_value) {
$counter->setOldStatus($old_value['status']);
$counter->setNewStatus($this->status);
$newValues[] = $counter->getUpdatedValues($old_value['total']);
}
try {
$newTotals = $counter->updateDB($newValues);
} catch (Exception $e) {
$this->result['errors'][] = array("code" => -101, "message" => "database errors");
// Log::doLog("Lock: Transaction Aborted. " . $e->getMessage() );
// $x1 = explode( "\n" , var_export( $old_translation, true) );
// Log::doLog("Lock: Translation status was " . implode( " ", $x1 ) );
$db->rollback();
return $e->getCode();
}
} else {
$newTotals = $old_wStruct;
}
$job_stats = CatUtils::getFastStatsForJob($newTotals);
$project = getProject($this->jobData['id_project']);
$project = array_pop($project);
$job_stats['ANALYSIS_COMPLETE'] = $project['status_analysis'] == Constants_ProjectStatus::STATUS_DONE || $project['status_analysis'] == Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE ? true : false;
$file_stats = array();
$this->result['stats'] = $job_stats;
$this->result['file_stats'] = $file_stats;
$this->result['code'] = 1;
$this->result['data'] = "OK";
$this->result['version'] = date_create($_Translation['translation_date'])->getTimestamp();
/* FIXME: added for code compatibility with front-end. Remove. */
$_warn = $check->getWarnings();
$warning = $_warn[0];
/* */
$this->result['warning']['cod'] = $warning->outcome;
if ($warning->outcome > 0) {
$this->result['warning']['id'] = $this->id_segment;
} else {
$this->result['warning']['id'] = 0;
}
//strtoupper transforms null to "" so check for the first element to be an empty string
if (!empty($this->split_statuses[0]) && !empty($this->split_num)) {
/* put the split inside the transaction if they are present */
$translationStruct = TranslationsSplit_SplitStruct::getStruct();
$translationStruct->id_segment = $this->id_segment;
$translationStruct->id_job = $this->id_job;
$translationStruct->target_chunk_lengths = array('len' => $this->split_chunk_lengths, 'statuses' => $this->split_statuses);
$translationDao = new TranslationsSplit_SplitDAO(Database::obtain());
$result = $translationDao->update($translationStruct);
}
$db->commit();
if ($job_stats['TRANSLATED_PERC'] == '100') {
$update_completed = setJobCompleteness($this->id_job, 1);
}
if (@$update_completed < 0) {
$msg = "\n\n Error setJobCompleteness \n\n " . var_export($_POST, true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
/**
* @param $_project_id
*/
protected function _tryToCloseProject($_project_id)
{
$project_totals = array();
$project_totals['project_segments'] = $this->_queueHandler->getRedisClient()->get(RedisKeys::PROJECT_TOT_SEGMENTS . $_project_id);
$project_totals['num_analyzed'] = $this->_queueHandler->getRedisClient()->get(RedisKeys::PROJECT_NUM_SEGMENTS_DONE . $_project_id);
$project_totals['eq_wc'] = $this->_queueHandler->getRedisClient()->get(RedisKeys::PROJ_EQ_WORD_COUNT . $_project_id) / 1000;
$project_totals['st_wc'] = $this->_queueHandler->getRedisClient()->get(RedisKeys::PROJ_ST_WORD_COUNT . $_project_id) / 1000;
$this->_doLog("--- (Worker {$this->_workerPid}) : count segments in project {$_project_id} = " . $project_totals['project_segments'] . "");
$this->_doLog("--- (Worker {$this->_workerPid}) : Analyzed segments in project {$_project_id} = " . $project_totals['num_analyzed'] . "");
if (empty($project_totals['project_segments'])) {
$this->_doLog("--- (Worker {$this->_workerPid}) : WARNING !!! error while counting segments in projects {$_project_id} skipping and continue ");
return;
}
if ($project_totals['project_segments'] - $project_totals['num_analyzed'] == 0 && $this->_queueHandler->getRedisClient()->setnx(RedisKeys::PROJECT_ENDING_SEMAPHORE . $_project_id, 1)) {
$this->_queueHandler->getRedisClient()->expire(RedisKeys::PROJECT_ENDING_SEMAPHORE . $_project_id, 60 * 60 * 24);
$_analyzed_report = getProjectSegmentsTranslationSummary($_project_id);
$total_segs = array_pop($_analyzed_report);
//remove Rollup
$this->_doLog("--- (Worker {$this->_workerPid}) : analysis project {$_project_id} finished : change status to DONE");
changeProjectStatus($_project_id, \Constants_ProjectStatus::STATUS_DONE);
changeTmWc($_project_id, $project_totals['eq_wc'], $project_totals['st_wc']);
/*
* Remove this job from the project list
*/
$this->_queueHandler->getRedisClient()->lrem($this->_mySubscribedQueue->redis_key, 0, $_project_id);
$this->_doLog("--- (Worker {$this->_workerPid}) : trying to initialize job total word count.");
foreach ($_analyzed_report as $job_info) {
$counter = new \WordCount_Counter();
$counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
}
}
}
