/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return mixed
*/
function doAction()
{
if (count($this->errors) > 0) {
return null;
}
//get job language and data
//Fixed Bug: need a specific job, because we need The target Language
//Removed from within the foreach cycle, the job is always the same...
$jobData = $this->jobInfo = getJobData($this->jobID, $this->jobPass);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($jobData) || !$pCheck->grantJobAccessByJobData($jobData, $this->jobPass)) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
$projectData = getProject($jobData['id_project']);
$source = $jobData['source'];
$target = $jobData['target'];
$tmsService = new TMSService();
/**
* @var $tmx SplTempFileObject
*/
$this->tmx = $tmsService->exportJobAsTMX($this->jobID, $this->jobPass, $source, $target);
$this->fileName = $projectData[0]['name'] . "-" . $this->jobID . ".tmx";
}
public function doAction()
{
if (empty($this->segment)) {
$this->result['errors'][] = array("code" => -1, "message" => "missing source segment");
}
if (empty($this->translation)) {
$this->result['errors'][] = array("code" => -2, "message" => "missing target translation");
}
if (empty($this->source_lang)) {
$this->result['errors'][] = array("code" => -3, "message" => "missing source lang");
}
if (empty($this->target_lang)) {
$this->result['errors'][] = array("code" => -4, "message" => "missing target lang");
}
if (empty($this->time_to_edit)) {
$this->result['errors'][] = array("code" => -5, "message" => "missing time to edit");
}
if (empty($this->id_segment)) {
$this->result['errors'][] = array("code" => -6, "message" => "missing segment id");
}
//get Job Infos, we need only a row of jobs ( split )
$job_data = getJobData((int) $this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($job_data) || !$pCheck->grantJobAccessByJobData($job_data, $this->password)) {
$this->result['errors'][] = array("code" => -10, "message" => "wrong password");
$msg = "\n\n Error \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return;
}
//mt engine to contribute to
if ($job_data['id_mt_engine'] <= 1) {
return false;
}
$this->mt = Engine::getInstance($job_data['id_mt_engine']);
//array of storicised suggestions for current segment
$this->suggestion_json_array = json_decode(getArrayOfSuggestionsJSON($this->id_segment), true);
//extra parameters
$extra = json_encode(array('id_segment' => $this->id_segment, 'suggestion_json_array' => $this->suggestion_json_array, 'chosen_suggestion_index' => $this->chosen_suggestion_index, 'time_to_edit' => $this->time_to_edit));
//send stuff
$config = $this->mt->getConfigStruct();
$config['segment'] = CatUtils::view2rawxliff($this->segment);
$config['translation'] = CatUtils::view2rawxliff($this->translation);
$config['source'] = $this->source_lang;
$config['target'] = $this->target_lang;
$config['email'] = INIT::$MYMEMORY_API_KEY;
$config['segid'] = $this->id_segment;
$config['extra'] = $extra;
$config['id_user'] = array("TESTKEY");
$outcome = $this->mt->set($config);
if ($outcome->error->code < 0) {
$this->result['errors'] = $outcome->error->get_as_array();
}
}
/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return mixed
*/
function doAction()
{
try {
$this->_checkData("auto_propagation.log");
} catch (Exception $e) {
if ($e->getCode() == -1) {
Utils::sendErrMailReport($e->getMessage());
}
Log::doLog($e->getMessage());
return $e->getCode();
}
$cookie_key = '_auto-propagation-' . $this->id_job . "-" . $this->password;
$boolString = (string) (int) $this->propagateAll;
$cookieLife = new DateTime();
$cookieLife->modify('+15 days');
$db = Database::obtain();
if ($this->propagateAll) {
$db->begin();
$old_translation = getCurrentTranslation($this->id_job, $this->id_segment);
//check tag mismatch
//get original source segment, first
$segment = getSegment($this->id_segment);
//compare segment-translation and get results
$check = new QA($segment['segment'], $this->translation);
$check->performConsistencyCheck();
if ($check->thereAreWarnings()) {
$err_json = $check->getWarningsJSON();
$translation = $this->translation;
} else {
$err_json = '';
$translation = $check->getTrgNormalized();
}
$TPropagation = array();
$TPropagation['id_job'] = $this->id_job;
$TPropagation['translation'] = $translation;
$TPropagation['status'] = Constants_TranslationStatus::STATUS_DRAFT;
$TPropagation['autopropagated_from'] = $this->id_segment;
$_Translation['serialized_errors_list'] = $err_json;
$TPropagation['warning'] = $check->thereAreWarnings();
$TPropagation['translation_date'] = date("Y-m-d H:i:s");
$TPropagation['segment_hash'] = $old_translation['segment_hash'];
try {
propagateTranslation($TPropagation, $this->jobData, $this->id_segment, true);
$db->commit();
} catch (Exception $e) {
$db->rollback();
$msg = $e->getMessage() . "\n\n" . $e->getTraceAsString();
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
setcookie($cookie_key, $boolString, $cookieLife->getTimestamp(), "/", $_SERVER['HTTP_HOST']);
Log::doLog("Auto-propagation for already translated segments on Job " . $this->id_job . " set to '" . var_export($this->propagateAll, true) . "'. Cookie Expire at " . $cookieLife->format('Y-m-d H:i:s'));
$this->result['errors'][] = array("code" => 0, "message" => "OK");
}
/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return mixed
*/
function doAction()
{
$_project_data = getProjectJobData($this->id_project);
$pCheck = new AjaxPasswordCheck();
$access = $pCheck->grantProjectAccess($_project_data, $this->password);
//check for Password correctness
if (!$access) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
$analysisStatus = new Analysis_XTRFStatus($_project_data);
$outputContent = $analysisStatus->fetchData()->getResult();
$this->content = $this->composeZip($_project_data[0]['pname'], $outputContent);
$this->_filename = $_project_data[0]['pname'] . ".zip";
}
function insertFastAnalysis($pid, &$fastReport, $equivalentWordMapping, $perform_Tms_Analysis = true)
{
$db = Database::obtain();
$data = array();
$amqHandler = new Analysis_QueueHandler();
$total_eq_wc = 0;
$total_standard_wc = 0;
$data['id_segment'] = null;
$data['id_job'] = null;
$data['segment_hash'] = null;
$data['match_type'] = null;
$data['eq_word_count'] = null;
$data['standard_word_count'] = null;
$segment_translations = "INSERT INTO `segment_translations` ( " . implode(", ", array_keys($data)) . " ) VALUES ";
$st_values = array();
foreach ($fastReport as $k => $v) {
$jid_fid = explode("-", $k);
$id_segment = $jid_fid[0];
$list_id_jobs_password = $jid_fid[1];
if (array_key_exists($v['match_type'], $equivalentWordMapping)) {
$eq_word = $v['wc'] * $equivalentWordMapping[$v['match_type']] / 100;
if ($v['match_type'] == "INTERNAL") {
}
} else {
$eq_word = $v['wc'];
}
$total_eq_wc += $eq_word;
$standard_words = $eq_word;
if ($v['match_type'] == "INTERNAL" or $v['match_type'] == "MT") {
$standard_words = $v['wc'] * $equivalentWordMapping["NO_MATCH"] / 100;
}
$total_standard_wc += $standard_words;
unset($fastReport[$k]['wc']);
$list_id_jobs_password = explode(',', $list_id_jobs_password);
foreach ($list_id_jobs_password as $id_job) {
list($id_job, $job_pass) = explode(":", $id_job);
$data['id_job'] = (int) $id_job;
$data['id_segment'] = (int) $fastReport[$k]['id_segment'];
$data['segment_hash'] = $db->escape($v['segment_hash']);
$data['match_type'] = $db->escape($v['match_type']);
if (!empty($v['segment_hash']) && empty($data['segment_hash'])) {
$data['segment_hash'] = $v['segment_hash'];
$msg = "mysql_real_escape_string failed!!! String was empty. Replaced with original {$v['segment_hash']}";
_TimeStampMsg($msg);
Utils::sendErrMailReport("<strong>{$msg}</strong>", "Fast Analysis mysql_real_escape_string failed.");
}
$data['eq_word_count'] = (double) $eq_word;
$data['standard_word_count'] = (double) $standard_words;
$st_values[] = " ( '" . implode("', '", array_values($data)) . "' )";
if ($data['eq_word_count'] > 0 && $perform_Tms_Analysis) {
/**
*
* IMPORTANT
* id_job will be taken from languages ( 80415:fr-FR,80416:it-IT )
*/
$fastReport[$k]['pid'] = (int) $pid;
$fastReport[$k]['date_insert'] = date_create()->format('Y-m-d H:i:s');
$fastReport[$k]['eq_word_count'] = (double) $eq_word;
$fastReport[$k]['standard_word_count'] = (double) $standard_words;
} else {
// Log::doLog( 'Skipped Fast Segment: ' . var_export( $fastReport[ $k ], true ) );
unset($fastReport[$k]);
}
}
}
unset($data);
$chunks_st = array_chunk($st_values, 200);
_TimeStampMsg('Insert Segment Translations: ' . count($st_values));
_TimeStampMsg('Queries: ' . count($chunks_st));
//USE the MySQL InnoDB isolation Level to protect from thread high concurrency access
$db->query('SET autocommit=0');
$db->query('START TRANSACTION');
foreach ($chunks_st as $k => $chunk) {
$query_st = $segment_translations . implode(", ", $chunk) . " ON DUPLICATE KEY UPDATE\n match_type = VALUES( match_type ),\n eq_word_count = VALUES( eq_word_count ),\n standard_word_count = VALUES( standard_word_count )\n ";
$db->query($query_st);
_TimeStampMsg("Executed " . ($k + 1));
$err = $db->get_error();
if ($err['error_code'] != 0) {
_TimeStampMsg($err);
return $err['error_code'] * -1;
}
}
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($st_values);
unset($chunks_st);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
/*
* IF NO TM ANALYSIS, upload the jobs global word count
*/
if (!$perform_Tms_Analysis) {
$_details = getProjectSegmentsTranslationSummary($pid);
_TimeStampMsg("--- trying to initialize job total word count.");
$project_details = array_pop($_details);
//remove rollup
foreach ($_details as $job_info) {
$counter = new WordCount_Counter();
$counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
}
}
/* IF NO TM ANALYSIS, upload the jobs global word count */
//_TimeStampMsg( "Done." );
$data2 = array('fast_analysis_wc' => $total_eq_wc);
$where = " id = {$pid}";
$db->update('projects', $data2, $where);
$err = $db->get_error();
$errno = $err['error_code'];
if ($errno != 0) {
$db->query('ROLLBACK');
$db->query('SET autocommit=1');
_TimeStampMsg($err);
return $errno * -1;
}
$db->query('COMMIT');
$db->query('SET autocommit=1');
if (count($fastReport)) {
// $chunks_st_queue = array_chunk( $fastReport, 10 );
_TimeStampMsg('Insert Segment Translations Queue: ' . count($fastReport));
_TimeStampMsg('Queries: ' . count($fastReport));
try {
$amqHandler->setTotal(array('qid' => $pid, 'queueName' => INIT::$QUEUE_NAME));
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set Total values failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
$time_start = microtime(true);
foreach ($fastReport as $k => $queue_element) {
try {
$languages_job = explode(",", $queue_element['target']);
//now target holds more than one language ex: ( 80415:fr-FR,80416:it-IT )
//in memory replacement avoid duplication of the segment list
//send in queue every element * number of languages
foreach ($languages_job as $_language) {
list($id_job, $language) = explode(":", $_language);
$queue_element['target'] = $language;
$queue_element['id_job'] = $id_job;
$jsonObj = json_encode($queue_element);
Utils::raiseJsonExceptionError();
$amqHandler->send(INIT::$QUEUE_NAME, $jsonObj, array('persistent' => $amqHandler->persistent));
_TimeStampMsg("AMQ Set Executed " . ($k + 1));
}
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set queue failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
}
_TimeStampMsg('Done in ' . (microtime(true) - $time_start) . " seconds.");
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($fastReport);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
}
$amqHandler->disconnect();
return $db->affected_rows;
}
/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @throws Exception
*/
public function doAction()
{
if (!empty($this->result['errors'])) {
return;
}
$job_data = getJobData((int) $this->id_job, $this->password_job);
if (empty($job_data)) {
$msg = "Error : empty job data \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
//add check for job status archived.
if (strtolower($job_data['status']) == Constants_JobStatus::STATUS_ARCHIVED) {
$this->result['errors'][] = array("code" => -6, "message" => "job archived");
}
$this->parseIDSegment();
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($job_data) || !$pCheck->grantJobAccessByJobData($job_data, $this->password_job, $this->id_segment)) {
$this->result['errors'][] = array("code" => -7, "message" => "wrong password");
}
$wStruct = new WordCount_Struct();
$wStruct->setIdJob($this->id_job);
$wStruct->setJobPassword($this->password_job);
$wStruct->setNewWords($job_data['new_words']);
$wStruct->setDraftWords($job_data['draft_words']);
$wStruct->setTranslatedWords($job_data['translated_words']);
$wStruct->setApprovedWords($job_data['approved_words']);
$wStruct->setRejectedWords($job_data['rejected_words']);
$reviseDAO = new Revise_ReviseDAO(Database::obtain());
//store segment revision in DB
$revisionStruct = Revise_ReviseStruct::getStruct();
$revisionStruct->id_job = $this->id_job;
$revisionStruct->id_segment = $this->id_segment;
//check if an old revision exists. If it does, retrieve it and save it.
$oldRevision = $reviseDAO->read($revisionStruct);
$oldRevision = isset($oldRevision[0]) ? $oldRevision[0] : Revise_ReviseStruct::setDefaultValues(Revise_ReviseStruct::getStruct());
$revisionStruct->err_typing = $this->err_typing;
$revisionStruct->err_translation = $this->err_translation;
$revisionStruct->err_terminology = $this->err_terminology;
$revisionStruct->err_language = $this->err_language;
$revisionStruct->err_style = $this->err_style;
$revisionStruct->original_translation = $this->original_translation;
//save the new revision in the database.
try {
$reviseDAO->create($revisionStruct);
} catch (Exception $e) {
Log::doLog(__METHOD__ . " -> " . $e->getMessage());
$this->result['errors'][] = array('code' => -4, 'message' => "Insert failed");
return;
}
/**
* Refresh error counters in the job table
*/
$errorCountStruct = new ErrorCount_DiffStruct($oldRevision, $revisionStruct);
$errorCountStruct->setIdJob($this->id_job);
$errorCountStruct->setJobPassword($this->password_job);
$errorCountDao = new ErrorCount_ErrorCountDAO(Database::obtain());
try {
$errorCountDao->update($errorCountStruct);
} catch (Exception $e) {
Log::doLog(__METHOD__ . " -> " . $e->getMessage());
$this->result['errors'][] = array('code' => -5, 'message' => "Did not update job error counters.");
return;
}
/**
* Retrieve information about job errors
* ( Note: these information are fed by the revision process )
* @see setRevisionController
*/
$jobQA = new Revise_JobQA($this->id_job, $this->password_job, $wStruct->getTotal());
$jobQA->retrieveJobErrorTotals();
$jobVote = $jobQA->evalJobVote();
$this->result['data']['message'] = 'OK';
$this->result['data']['stat_quality'] = $jobQA->getQaData();
$this->result['data']['overall_quality'] = $jobVote['minText'];
$this->result['data']['overall_quality_class'] = strtolower(str_replace(' ', '', $jobVote['minText']));
}
private function __notifyError($conversionObject)
{
$remote_user = isset($_SERVER['REMOTE_USER']) ? $_SERVER['REMOTE_USER'] : "******";
$link_file = "http://" . $_SERVER['SERVER_NAME'] . "/" . INIT::$CONVERSIONERRORS_REPOSITORY_WEB . "/" . $_COOKIE['upload_session'] . "/" . rawurlencode($conversionObject->file_name);
$message = "MATECAT : conversion error notifier\n\nDetails:\n\t\t\t- machine_ip : " . $conversionObject->ip_machine . "\n\t\t\t- client ip : " . $conversionObject->ip_client . "\n\t\t\t- source : " . $conversionObject->src_lang . "\n\t\t\t- target : " . $conversionObject->trg_lang . "\n\t\t\t- client user (if any used) : {$remote_user}\n\t\t\t\t\t\t\t\t\t\t - direction : " . $conversionObject->direction . "\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t- error : " . $conversionObject->error_message . "\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t Download file clicking to {$link_file}\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t ";
Utils::sendErrMailReport($message);
}
public function doAction()
{
$debug = array();
$debug['total'][] = time();
//get job language and data
//Fixed Bug: need a specific job, because we need The target Language
//Removed from within the foreach cycle, the job is always the same....
$jobData = $this->jobInfo = getJobData($this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($jobData) || !$pCheck->grantJobAccessByJobData($jobData, $this->password)) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
$debug['get_file'][] = time();
$files_job = getFilesForJob($this->id_job, $this->id_file);
$debug['get_file'][] = time();
$nonew = 0;
$output_content = array();
/*
* the procedure is now as follows:
* 1)original file is loaded from DB into RAM and the flushed in a temp file on disk; a file handler is obtained
* 2)RAM gets freed from original content
* 3)the file is read chunk by chunk by a stream parser: for each tran-unit that is encountered,
* target is replaced (or added) with the corresponding translation among segments
* the current string in the buffer is flushed on standard output
* 4)the temporary file is deleted by another process after some time
*
*/
//file array is chuncked. Each chunk will be used for a parallel conversion request.
$files_job = array_chunk($files_job, self::FILES_CHUNK_SIZE);
foreach ($files_job as $chunk) {
$converter = new FileFormatConverter();
$files_buffer = array();
foreach ($chunk as $file) {
$mime_type = $file['mime_type'];
$fileID = $file['id_file'];
$current_filename = $file['filename'];
$original_xliff = $file['xliff_file'];
//get path
$path = INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/' . $fileID . '/' . $current_filename . "_" . uniqid('', true) . '.sdlxliff';
//make dir if doesn't exist
if (!file_exists(dirname($path))) {
Log::doLog('exec ("chmod 666 ' . escapeshellarg($path) . '");');
mkdir(dirname($path), 0777, true);
exec("chmod 666 " . escapeshellarg($path));
}
//create file
$fp = fopen($path, 'w+');
//flush file to disk
fwrite($fp, $original_xliff);
//free memory, as we can work with file on disk now
unset($original_xliff);
$debug['get_segments'][] = time();
$data = getSegmentsDownload($this->id_job, $this->password, $fileID, $nonew);
$debug['get_segments'][] = time();
//create a secondary indexing mechanism on segments' array; this will be useful
//prepend a string so non-trans unit id ( ex: numerical ) are not overwritten
//clean also not valid xml entities ( charactes with ascii < 32 and different from 0A, 0D and 09
$regexpEntity = '/&#x(0[0-8BCEF]|1[0-9A-F]|7F);/u';
//remove binary chars in some xliff files
$regexpAscii = '/([\\x{00}-\\x{1F}\\x{7F}]{1})/u';
foreach ($data as $i => $k) {
$data['matecat|' . $k['internal_id']][] = $i;
//FIXME: temporary patch
$data[$i]['translation'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['translation']);
$data[$i]['segment'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['segment']);
$sanitized_src = preg_replace($regexpAscii, '', $data[$i]['segment']);
$sanitized_trg = preg_replace($regexpAscii, '', $data[$i]['translation']);
$sanitized_src = preg_replace($regexpEntity, '', $sanitized_src);
$sanitized_trg = preg_replace($regexpEntity, '', $sanitized_trg);
if ($sanitized_src != null) {
$data[$i]['segment'] = $sanitized_src;
}
if ($sanitized_trg != null) {
$data[$i]['translation'] = $sanitized_trg;
}
}
$debug['replace'][] = time();
//instatiate parser
$xsp = new XliffSAXTranslationReplacer($path, $data, Langs_Languages::getInstance()->getLangRegionCode($jobData['target']), $fp);
if ($this->download_type == 'omegat') {
$xsp->setSourceInTarget(true);
}
//run parsing
Log::doLog("work on " . $fileID . " " . $current_filename);
$xsp->replaceTranslation();
fclose($fp);
unset($xsp);
$debug['replace'][] = time();
$output_xliff = file_get_contents($path . '.out.sdlxliff');
$output_content[$fileID]['documentContent'] = $output_xliff;
$output_content[$fileID]['filename'] = $current_filename;
unset($output_xliff);
if ($this->forceXliff) {
$file_info_details = pathinfo($output_content[$fileID]['filename']);
$output_content[$fileID]['filename'] = $file_info_details['filename'] . ".out.sdlxliff";
}
//TODO set a flag in database when file uploaded to know if this file is a proprietary xlf converted
//TODO so we can load from database the original file blob ONLY when needed
/**
* Conversion Enforce
*/
$convertBackToOriginal = true;
try {
//if it is a not converted file ( sdlxliff ) we have an empty field original_file
//so we can simplify all the logic with:
// is empty original_file? if it is, we don't need conversion back because
// we already have an sdlxliff or an accepted file
$file['original_file'] = @gzinflate($file['original_file']);
if (!INIT::$CONVERSION_ENABLED || empty($file['original_file']) && $mime_type == 'sdlxliff' || $this->forceXliff) {
$convertBackToOriginal = false;
Log::doLog("SDLXLIFF: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
} else {
//TODO: dos2unix ??? why??
//force unix type files
Log::doLog("NO SDLXLIFF, Conversion enforced: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
if ($convertBackToOriginal) {
$output_content[$fileID]['out_xliff_name'] = $path . '.out.sdlxliff';
$output_content[$fileID]['source'] = $jobData['source'];
$output_content[$fileID]['target'] = $jobData['target'];
$files_buffer[$fileID] = $output_content[$fileID];
} elseif ($this->forceXliff) {
$this->cleanFilePath($output_content[$fileID]['documentContent']);
}
}
$debug['do_conversion'][] = time();
$convertResult = $converter->multiConvertToOriginal($files_buffer, $chosen_machine = false);
foreach (array_keys($files_buffer) as $fileID) {
$output_content[$fileID]['documentContent'] = $this->removeTargetMarks($convertResult[$fileID]['documentContent'], $files_buffer[$fileID]['filename']);
//in case of .strings, they are required to be in UTF-16
//get extension to perform file detection
$extension = pathinfo($output_content[$fileID]['filename'], PATHINFO_EXTENSION);
if (strtoupper($extension) == 'STRINGS') {
//use this function to convert stuff
$encodingConvertedFile = CatUtils::convertEncoding('UTF-16', $output_content[$fileID]['documentContent']);
//strip previously added BOM
$encodingConvertedFile[1] = $converter->stripBOM($encodingConvertedFile[1], 16);
//store new content
$output_content[$fileID]['documentContent'] = $encodingConvertedFile[1];
//trash temporary data
unset($encodingConvertedFile);
}
}
// $output_content[ $fileID ][ 'documentContent' ] = $convertResult[ 'documentContent' ];
unset($convertResult);
$debug['do_conversion'][] = time();
}
//set the file Name
$pathinfo = pathinfo($this->fname);
$this->filename = $pathinfo['filename'] . "_" . $jobData['target'] . "." . $pathinfo['extension'];
//qui prodest to check download type?
if ($this->download_type == 'omegat') {
$this->filename .= ".zip";
$tmsService = new TMSService();
$tmsService->setOutputType('tm');
/**
* @var $tmFile SplTempFileObject
*/
$tmFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tmsService->setOutputType('mt');
/**
* @var $mtFile SplTempFileObject
*/
$mtFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tm_id = uniqid('tm');
$mt_id = uniqid('mt');
$output_content[$tm_id] = array('documentContent' => '', 'filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_TM . tmx");
foreach ($tmFile as $lineNumber => $content) {
$output_content[$tm_id]['documentContent'] .= $content;
}
$output_content[$mt_id] = array('documentContent' => '', 'filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_MT . tmx");
foreach ($mtFile as $lineNumber => $content) {
$output_content[$mt_id]['documentContent'] .= $content;
}
$this->createOmegaTZip($output_content, $jobData['source'], $jobData['target']);
//add zip archive content here;
} else {
if (count($output_content) > 1) {
if ($pathinfo['extension'] != 'zip') {
if ($this->forceXliff) {
$this->filename = $this->id_job . ".zip";
} else {
$this->filename = $pathinfo['basename'] . ".zip";
}
}
$this->composeZip($output_content, $jobData['source']);
//add zip archive content here;
} else {
//always an array with 1 element, pop it, Ex: array( array() )
$output_content = array_pop($output_content);
$this->setContent($output_content);
}
}
$debug['total'][] = time();
Utils::deleteDir(INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/');
}
function main($args = null)
{
$db = Database::obtain();
$lastProcessedJob = (int) file_get_contents(self::$last_job_file_name);
do {
$queryMaxJob = "select min(id) as min, max(id) as max\n from jobs\n where completed = 1\n and id > %d";
$queryFirst = "select id, password, job_first_segment, job_last_segment\n from jobs\n where completed = 1\n and id >= %d and id <= %d";
$querySegments = "select suggestion,\n translation,\n raw_word_count,\n time_to_edit\n from segment_translations st\n join segments s on st.id_segment = s.id\n and s.id between %d and %d\n where status='translated'\n and id_job = %d\n and show_in_cattool = 1\n and id_segment >= %d\n limit %d";
$queryUpdateJob = "update jobs\n set avg_post_editing_effort = %f,\n total_time_to_edit = %f,\n total_raw_wc = %d\n where id = %d and password = '******'";
$minJobMaxJob = $db->query_first(sprintf($queryMaxJob, (int) $lastProcessedJob));
$maxJob = (int) $minJobMaxJob['max'];
$minJob = (int) $minJobMaxJob['min'];
$start = time();
//get a chunk of self::NR_OF_JOBS each time.
for ($firstJob = $minJob; $this->RUNNING && $firstJob < $maxJob; $firstJob += self::NR_OF_JOBS) {
$jobs = $db->fetch_array(sprintf($queryFirst, $firstJob, $firstJob + self::NR_OF_JOBS));
//iterate over completed jobs, evaluate incremental PEE and save it in the job row
//Incremental PEE = sum( segment_pee * segment_raw_wordcount)
//It will be normalized when necessary
for ($j = 0; $this->RUNNING && $j < count($jobs); $j++) {
$job = $jobs[$j];
//BEGIN TRANSACTION
$db->begin();
$_jid = $job['id'];
$_password = $job['password'];
$_job_first_segment = $job['job_first_segment'];
$_job_last_segment = $job['job_last_segment'];
Log::doLog("job {$_jid} -> " . ($_job_last_segment - $_job_first_segment) . " segments");
echo "job {$_jid} -> " . ($_job_last_segment - $_job_first_segment) . " segments\n";
$raw_post_editing_effort_job = 0;
$raw_wc_job = 0;
$time_to_edit_job = 0;
//get a chunk of self::NR_OF_SEGS segments each time.
for ($firstSeg = $_job_first_segment; $firstSeg <= $_job_last_segment; $firstSeg += self::NR_OF_SEGS) {
if ($firstSeg > $_job_last_segment) {
$firstSeg = $_job_last_segment;
}
Log::doLog("starting from segment {$firstSeg}");
echo "starting from segment {$firstSeg}\n";
$segments = $db->fetch_array(sprintf($querySegments, $_job_first_segment, $_job_last_segment, $_jid, $firstSeg, self::NR_OF_SEGS));
//iterate over segments.
foreach ($segments as $i => $segment) {
$segment = new EditLog_EditLogSegmentStruct($segment);
$raw_wc_job += $segment->raw_word_count;
$time_to_edit_job += $segment->time_to_edit;
if ($segment->isValidForEditLog()) {
$raw_post_editing_effort_job += $segment->getPEE() * $segment->raw_word_count;
}
}
//sleep 100 nanosecs
usleep(100);
}
$job_incremental_pee = $raw_post_editing_effort_job;
Log::doLog("job pee: {$job_incremental_pee}\njob time to edit: {$time_to_edit_job}\njob total wc:{$raw_wc_job}\nWriting into DB");
echo "job pee: {$job_incremental_pee}\njob time to edit: {$time_to_edit_job}\njob total wc:{$raw_wc_job}\nWriting into DB\n";
$db->query(sprintf($queryUpdateJob, $job_incremental_pee, $time_to_edit_job, $raw_wc_job, $_jid, $_password));
Log::doLog("done");
echo "done.\n";
if (!file_put_contents(self::$last_job_file_name, $_jid)) {
$db->rollback();
Utils::sendErrMailReport("", "[JobPostEditingEffortRunner] Failed to process job {$_jid}");
$this->RUNNING = false;
continue;
//exit;
}
//COMMIT TRANSACTION
$db->commit();
}
}
Log::doLog("took " . (time() - $start) / 60 . " seconds");
echo "took " . (time() - $start) / 60 . " seconds\n";
Log::doLog("sleeping for 1 month");
echo "sleeping for 1 month\n";
if ($this->RUNNING) {
sleep(self::$sleeptime);
}
} while ($this->RUNNING);
}
/**
* This check for reboot status, if it is at least the third check failed after a reboot command,
* it return a failure
*
* @param $ip_converter
*
* @return mixed
*/
protected function _checkForRebootFailed($ip_converter)
{
$selectLogs_afterLastUpdate = sprintf(self::selectLogs_afterLastUpdate, $this->resultSet[$ip_converter]['id'], $this->resultSet[$ip_converter]['last_update']);
$failedConversionsAfterRebootLogs = $this->db->fetch_array($selectLogs_afterLastUpdate);
//if there are failed conversions test after the reboot time ( long reboot )
if (count($failedConversionsAfterRebootLogs) != 0) {
$rebootTime = new DateTime($this->resultSet[$ip_converter]['last_update']);
$thisTimeFailure = new DateTime();
//if this failure happened 10 minutes after reboot time
if ($thisTimeFailure->modify('-10 minutes') >= $rebootTime) {
self::_prettyEcho("> *** FAILED REBOOT FOUND....", 4);
$msg = "\n\n *** FAILED REBOOT STATUS FOUND FOR CONVERTER {$ip_converter} -> " . $this->host_machine_map[$ip_converter]['instance_name'] . "\n - instance_name: " . $this->host_machine_map[$ip_converter]['instance_name'] . "\n - ip_machine_host: " . $this->host_machine_map[$ip_converter]['ip_machine_host'] . "\n" . "\n" . "\n *** WARNING: THIS MACHINE IS SET AS OFFLINE, WILL NEVER PUT BACK IN POOL ***";
//send Alert report
Utils::sendErrMailReport($msg);
self::_prettyEcho("> *** MESSAGE SENT....", 4);
$this->resultSet[$ip_converter]['status_active'] = 0;
$this->resultSet[$ip_converter]['status_reboot'] = 0;
$this->resultSet[$ip_converter]['status_offline'] = 1;
$this->db->update('converters', $this->resultSet[$ip_converter], " ip_converter = '{$ip_converter}' LIMIT 1");
return $ip_converter;
}
}
}
/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return mixed
*/
function doAction()
{
Utils::sendErrMailReport($this->error_mail_content);
//echo $this->error_mail_content;exit;
parent::makeTemplate("badConfiguration.html");
}
public function doAction()
{
$uploadFile = new Upload();
try {
$stdResult = $uploadFile->uploadFiles($_FILES);
} catch (Exception $e) {
$stdResult = array();
$this->result = array('errors' => array(array("code" => -1, "message" => $e->getMessage())));
$this->api_output['message'] = $e->getMessage();
}
$arFiles = array();
foreach ($stdResult as $input_name => $input_value) {
$arFiles[] = $input_value->name;
}
//if fileupload was failed this index ( 0 = does not exists )
$default_project_name = @$arFiles[0];
if (count($arFiles) > 1) {
$default_project_name = "MATECAT_PROJ-" . date("Ymdhi");
}
if (empty($this->project_name)) {
$this->project_name = $default_project_name;
//'NO_NAME'.$this->create_project_name();
}
if (empty($this->source_lang)) {
$this->api_output['message'] = "Missing source language.";
$this->result['errors'][] = array("code" => -3, "message" => "Missing source language.");
}
if (empty($this->target_lang)) {
$this->api_output['message'] = "Missing target language.";
$this->result['errors'][] = array("code" => -4, "message" => "Missing target language.");
}
//ONE OR MORE ERRORS OCCURRED : EXITING
//for now we sent to api output only the LAST error message, but we log all
if (!empty($this->result['errors'])) {
$msg = "Error \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return -1;
//exit code
}
/* Do conversions here */
$converter = new ConvertFileWrapper($stdResult);
$converter->intDir = $uploadFile->getUploadPath();
$converter->errDir = INIT::$CONVERSIONERRORS_REPOSITORY . DIRECTORY_SEPARATOR . $uploadFile->getDirUploadToken();
$converter->cookieDir = $uploadFile->getDirUploadToken();
$converter->source_lang = $this->source_lang;
$converter->target_lang = $this->target_lang;
$converter->doAction();
$status = $converter->checkResult();
if (!empty($status)) {
$this->api_output['message'] = 'Project Conversion Failure';
$this->api_output['debug'] = $status;
$this->result['errors'] = $status;
Log::doLog($status);
return -1;
}
/* Do conversions here */
$projectManager = new ProjectManager();
$projectStructure = $projectManager->getProjectStructure();
$projectStructure['project_name'] = $this->project_name;
$projectStructure['result'] = $this->result;
$projectStructure['private_tm_key'] = $this->private_tm_key;
$projectStructure['private_tm_user'] = $this->private_tm_user;
$projectStructure['private_tm_pass'] = $this->private_tm_pass;
$projectStructure['uploadToken'] = $uploadFile->getDirUploadToken();
$projectStructure['array_files'] = $arFiles;
//list of file name
$projectStructure['source_language'] = $this->source_lang;
$projectStructure['target_language'] = explode(',', $this->target_lang);
$projectStructure['mt_engine'] = $this->mt_engine;
$projectStructure['tms_engine'] = $this->tms_engine;
$projectStructure['status'] = Constants_ProjectStatus::STATUS_NOT_READY_FOR_ANALYSIS;
$projectStructure['skip_lang_validation'] = true;
$projectManager = new ProjectManager($projectStructure);
$projectManager->createProject();
$this->result = $projectStructure['result'];
if (!empty($projectStructure['result']['errors'])) {
//errors already logged
$this->api_output['message'] = 'Project Creation Failure';
$this->api_output['debug'] = $projectStructure['result']['errors'];
} else {
//everything ok
$this->api_output['status'] = 'OK';
$this->api_output['message'] = 'Success';
$this->api_output['id_project'] = $projectStructure['result']['id_project'];
$this->api_output['project_pass'] = $projectStructure['result']['ppassword'];
}
}
public function doAction()
{
try {
$this->_checkData();
} catch (Exception $e) {
if ($e->getCode() == -1) {
Utils::sendErrMailReport($e->getMessage());
}
Log::doLog($e->getMessage());
return $e->getCode();
}
//check tag mismatch
//get original source segment, first
$segment = getSegment($this->id_segment);
//compare segment-translation and get results
$check = new QA($segment['segment'], $this->translation);
$check->performConsistencyCheck();
if ($check->thereAreWarnings()) {
$err_json = $check->getWarningsJSON();
$translation = $this->translation;
} else {
$err_json = '';
$translation = $check->getTrgNormalized();
}
/*
* begin stats counter
*
* It works good with default InnoDB Isolation level
*
* REPEATABLE-READ offering a row level lock for this id_segment
*
*/
$db = Database::obtain();
$db->begin();
$old_translation = getCurrentTranslation($this->id_job, $this->id_segment);
// $old_version = ( empty( $old_translation['translation_date'] ) ? '0' : date_create( $old_translation['translation_date'] )->getTimestamp() );
// if( $this->client_target_version != $old_version ){
// $this->result[ 'errors' ][ ] = array( "code" => -102, "message" => "Translation version mismatch" );
// $db->rollback();
// return false;
// }
//if volume analysis is not enabled and no translation rows exists
//create the row
if (!INIT::$VOLUME_ANALYSIS_ENABLED && empty($old_translation['status'])) {
$_Translation = array();
$_Translation['id_segment'] = (int) $this->id_segment;
$_Translation['id_job'] = (int) $this->id_job;
$_Translation['status'] = Constants_TranslationStatus::STATUS_NEW;
$_Translation['segment_hash'] = $segment['segment_hash'];
$_Translation['translation'] = $segment['segment'];
$_Translation['standard_word_count'] = $segment['raw_word_count'];
$_Translation['serialized_errors_list'] = '';
$_Translation['suggestion_position'] = 0;
$_Translation['warning'] = false;
$_Translation['translation_date'] = date("Y-m-d H:i:s");
$res = addTranslation($_Translation);
if ($res < 0) {
$this->result['errors'][] = array("code" => -101, "message" => "database errors");
$db->rollback();
return $res;
}
/*
* begin stats counter
*
*/
$old_translation = $_Translation;
}
$_Translation = array();
$_Translation['id_segment'] = $this->id_segment;
$_Translation['id_job'] = $this->id_job;
$_Translation['status'] = $this->status;
$_Translation['time_to_edit'] = $this->time_to_edit;
$_Translation['translation'] = preg_replace('/[ \\t\\n\\r\\0\\x0A\\xA0]+$/u', '', $translation);
$_Translation['serialized_errors_list'] = $err_json;
$_Translation['suggestion_position'] = $this->chosen_suggestion_index;
$_Translation['warning'] = $check->thereAreWarnings();
$_Translation['translation_date'] = date("Y-m-d H:i:s");
/**
* when the status of the translation changes, the auto propagation flag
* must be removed
*/
if ($_Translation['translation'] != $old_translation['translation'] || $this->status == Constants_TranslationStatus::STATUS_TRANSLATED || $this->status == Constants_TranslationStatus::STATUS_APPROVED) {
$_Translation['autopropagated_from'] = 'NULL';
}
$res = CatUtils::addSegmentTranslation($_Translation);
if (!empty($res['errors'])) {
$this->result['errors'] = $res['errors'];
$msg = "\n\n Error addSegmentTranslation \n\n Database Error \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
$db->rollback();
return -1;
}
if (INIT::$DQF_ENABLED && !empty($this->jobData['dqf_key']) && $_Translation['status'] == Constants_TranslationStatus::STATUS_TRANSLATED) {
$dqfSegmentStruct = DQF_DqfSegmentStruct::getStruct();
if ($old_translation['suggestion'] == null) {
$dqfSegmentStruct->target_segment = "";
$dqfSegmentStruct->tm_match = 0;
} else {
$dqfSegmentStruct->target_segment = $old_translation['suggestion'];
$dqfSegmentStruct->tm_match = $old_translation['suggestion_match'];
}
$dqfSegmentStruct->task_id = $this->id_job;
$dqfSegmentStruct->segment_id = $this->id_segment;
$dqfSegmentStruct->source_segment = $segment['segment'];
$dqfSegmentStruct->new_target_segment = $_Translation['translation'];
$dqfSegmentStruct->time = $_Translation['time_to_edit'];
// $dqfSegmentStruct->mtengine = $this->jobData['id_mt_engine'];
$dqfSegmentStruct->mt_engine_version = 1;
try {
$dqfQueueHandler = new Analysis_DqfQueueHandler();
$dqfQueueHandler->createSegment($dqfSegmentStruct);
} catch (Exception $exn) {
$msg = $exn->getMessage() . "\n\n" . $exn->getTraceAsString();
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
$propagateToTranslated = true;
//for the moment, this is set explicitely
if ($this->propagate == false) {
$propagateToTranslated = false;
}
//propagate translations
$TPropagation = array();
//Warning: this value will NOT be used to update values,
//but to exclude current segment from auto-propagation
$_idSegment = $this->id_segment;
$propagateToTranslated ? $TPropagation['status'] = $this->status : null;
$TPropagation['id_job'] = $this->id_job;
$TPropagation['translation'] = $translation;
$TPropagation['autopropagated_from'] = $this->id_segment;
$TPropagation['serialized_errors_list'] = $err_json;
$TPropagation['warning'] = $check->thereAreWarnings();
// $TPropagation[ 'translation_date' ] = date( "Y-m-d H:i:s" );
$TPropagation['segment_hash'] = $old_translation['segment_hash'];
$propagationTotal = array();
if ($propagateToTranslated && in_array($this->status, array(Constants_TranslationStatus::STATUS_TRANSLATED, Constants_TranslationStatus::STATUS_APPROVED, Constants_TranslationStatus::STATUS_REJECTED))) {
try {
$propagationTotal = propagateTranslation($TPropagation, $this->jobData, $_idSegment, $propagateToTranslated);
} catch (Exception $e) {
$msg = $e->getMessage() . "\n\n" . $e->getTraceAsString();
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
//Recount Job Totals
$old_wStruct = new WordCount_Struct();
$old_wStruct->setIdJob($this->id_job);
$old_wStruct->setJobPassword($this->password);
$old_wStruct->setNewWords($this->jobData['new_words']);
$old_wStruct->setDraftWords($this->jobData['draft_words']);
$old_wStruct->setTranslatedWords($this->jobData['translated_words']);
$old_wStruct->setApprovedWords($this->jobData['approved_words']);
$old_wStruct->setRejectedWords($this->jobData['rejected_words']);
$old_wStruct->setIdSegment($this->id_segment);
//redundant, this is made into WordCount_Counter::updateDB
$old_wStruct->setOldStatus($old_translation['status']);
$old_wStruct->setNewStatus($this->status);
//redundant because the update is made only where status = old status
if ($this->status != $old_translation['status']) {
//cambiato status, sposta i conteggi
$old_count = !is_null($old_translation['eq_word_count']) ? $old_translation['eq_word_count'] : $segment['raw_word_count'];
//if there is not a row in segment_translations because volume analysis is disabled
//search for a just created row
$old_status = !empty($old_translation['status']) ? $old_translation['status'] : Constants_TranslationStatus::STATUS_NEW;
$counter = new WordCount_Counter($old_wStruct);
$counter->setOldStatus($old_status);
$counter->setNewStatus($this->status);
$newValues = array();
$newValues[] = $counter->getUpdatedValues($old_count);
foreach ($propagationTotal as $__pos => $old_value) {
$counter->setOldStatus($old_value['status']);
$counter->setNewStatus($this->status);
$newValues[] = $counter->getUpdatedValues($old_value['total']);
}
try {
$newTotals = $counter->updateDB($newValues);
} catch (Exception $e) {
$this->result['errors'][] = array("code" => -101, "message" => "database errors");
// Log::doLog("Lock: Transaction Aborted. " . $e->getMessage() );
// $x1 = explode( "\n" , var_export( $old_translation, true) );
// Log::doLog("Lock: Translation status was " . implode( " ", $x1 ) );
$db->rollback();
return $e->getCode();
}
} else {
$newTotals = $old_wStruct;
}
$job_stats = CatUtils::getFastStatsForJob($newTotals);
$project = getProject($this->jobData['id_project']);
$project = array_pop($project);
$job_stats['ANALYSIS_COMPLETE'] = $project['status_analysis'] == Constants_ProjectStatus::STATUS_DONE || $project['status_analysis'] == Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE ? true : false;
$file_stats = array();
$this->result['stats'] = $job_stats;
$this->result['file_stats'] = $file_stats;
$this->result['code'] = 1;
$this->result['data'] = "OK";
$this->result['version'] = date_create($_Translation['translation_date'])->getTimestamp();
/* FIXME: added for code compatibility with front-end. Remove. */
$_warn = $check->getWarnings();
$warning = $_warn[0];
/* */
$this->result['warning']['cod'] = $warning->outcome;
if ($warning->outcome > 0) {
$this->result['warning']['id'] = $this->id_segment;
} else {
$this->result['warning']['id'] = 0;
}
//strtoupper transforms null to "" so check for the first element to be an empty string
if (!empty($this->split_statuses[0]) && !empty($this->split_num)) {
/* put the split inside the transaction if they are present */
$translationStruct = TranslationsSplit_SplitStruct::getStruct();
$translationStruct->id_segment = $this->id_segment;
$translationStruct->id_job = $this->id_job;
$translationStruct->target_chunk_lengths = array('len' => $this->split_chunk_lengths, 'statuses' => $this->split_statuses);
$translationDao = new TranslationsSplit_SplitDAO(Database::obtain());
$result = $translationDao->update($translationStruct);
}
$db->commit();
if ($job_stats['TRANSLATED_PERC'] == '100') {
$update_completed = setJobCompleteness($this->id_job, 1);
}
if (@$update_completed < 0) {
$msg = "\n\n Error setJobCompleteness \n\n " . var_export($_POST, true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
function addTranslation(array $_Translation)
{
$db = Database::obtain();
$query = "INSERT INTO `segment_translations` ";
foreach ($_Translation as $key => $val) {
if ($key == 'translation') {
$_Translation[$key] = "'" . $db->escape($val) . "'";
continue;
}
if (strtolower($val) == 'now()' || strtolower($val) == 'current_timestamp()' || strtolower($val) == 'sysdate()') {
$_Translation[$key] = "NOW()";
} elseif (is_numeric($val)) {
$_Translation[$key] = (double) $val;
} elseif (is_bool($val)) {
$_Translation[$key] = var_export($val, true);
} elseif (strtolower($val) == 'null' || empty($val)) {
$_Translation[$key] = "NULL";
} else {
$_Translation[$key] = "'" . $db->escape($val) . "'";
}
}
$query .= "(" . implode(", ", array_keys($_Translation)) . ") VALUES (" . implode(", ", array_values($_Translation)) . ")";
$query .= "\n\t\t\t\tON DUPLICATE KEY UPDATE\n\t\t\t\tstatus = {$_Translation['status']},\n suggestion_position = {$_Translation['suggestion_position']},\n serialized_errors_list = {$_Translation['serialized_errors_list']},\n time_to_edit = time_to_edit + VALUES( time_to_edit ),\n translation = {$_Translation['translation']},\n translation_date = {$_Translation['translation_date']},\n warning = " . $_Translation['warning'];
if (isset($_Translation['autopropagated_from'])) {
$query .= " , autopropagated_from = NULL";
}
if (empty($_Translation['translation']) && !is_numeric($_Translation['translation'])) {
$msg = "\n\n Error setTranslationUpdate \n\n Empty translation found after DB Escape: \n\n " . var_export(array_merge(array('db_query' => $query), $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return -1;
}
Log::doLog($query);
$db->query($query);
$err = $db->get_error();
$errno = $err['error_code'];
// Log::doLog( $err );
// Log::doLog( $db->affected_rows );
if ($errno != 0) {
Log::doLog("{$errno}: " . var_export($err, true));
return $errno * -1;
}
return $db->affected_rows;
}
public function doAction()
{
if (!$this->validateAuthHeader()) {
header('HTTP/1.0 401 Unauthorized');
$this->api_output['message'] = 'Authentication failed';
return -1;
}
if (@count($this->api_output['debug']) > 0) {
return;
}
$uploadFile = new Upload();
try {
$stdResult = $uploadFile->uploadFiles($_FILES);
} catch (Exception $e) {
$stdResult = array();
$this->result = array('errors' => array(array("code" => -1, "message" => $e->getMessage())));
$this->api_output['message'] = $e->getMessage();
}
$arFiles = array();
foreach ($stdResult as $input_name => $input_value) {
$arFiles[] = $input_value->name;
}
//if fileupload was failed this index ( 0 = does not exists )
$default_project_name = @$arFiles[0];
if (count($arFiles) > 1) {
$default_project_name = "MATECAT_PROJ-" . date("Ymdhi");
}
if (empty($this->project_name)) {
$this->project_name = $default_project_name;
//'NO_NAME'.$this->create_project_name();
}
if (empty($this->source_lang)) {
$this->api_output['message'] = "Missing source language.";
$this->result['errors'][] = array("code" => -3, "message" => "Missing source language.");
}
if (empty($this->target_lang)) {
$this->api_output['message'] = "Missing target language.";
$this->result['errors'][] = array("code" => -4, "message" => "Missing target language.");
}
//ONE OR MORE ERRORS OCCURRED : EXITING
//for now we sent to api output only the LAST error message, but we log all
if (!empty($this->result['errors'])) {
$msg = "Error \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return -1;
//exit code
}
$cookieDir = $uploadFile->getDirUploadToken();
$intDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $cookieDir;
$errDir = INIT::$STORAGE_DIR . DIRECTORY_SEPARATOR . 'conversion_errors' . DIRECTORY_SEPARATOR . $cookieDir;
$response_stack = array();
foreach ($arFiles as $file_name) {
$ext = FilesStorage::pathinfo_fix($file_name, PATHINFO_EXTENSION);
$conversionHandler = new ConversionHandler();
$conversionHandler->setFileName($file_name);
$conversionHandler->setSourceLang($this->source_lang);
$conversionHandler->setTargetLang($this->target_lang);
$conversionHandler->setSegmentationRule($this->seg_rule);
$conversionHandler->setCookieDir($cookieDir);
$conversionHandler->setIntDir($intDir);
$conversionHandler->setErrDir($errDir);
$status = array();
if ($ext == "zip") {
// this makes the conversionhandler accumulate eventual errors on files and continue
$conversionHandler->setStopOnFileException(false);
$fileObjects = $conversionHandler->extractZipFile();
//call convertFileWrapper and start conversions for each file
if ($conversionHandler->uploadError) {
$fileErrors = $conversionHandler->getUploadedFiles();
foreach ($fileErrors as $fileError) {
if (count($fileError->error) == 0) {
continue;
}
$brokenFileName = ZipArchiveExtended::getFileName($fileError->name);
/*
* TODO
* return error code is 2 because
* <=0 is for errors
* 1 is OK
*
* In this case, we raise warnings, hence the return code must be a new code
*/
$this->result['code'] = 2;
$this->result['errors'][$brokenFileName] = array('code' => $fileError->error['code'], 'message' => $fileError->error['message'], 'debug' => $brokenFileName);
}
}
$realFileObjectInfo = $fileObjects;
$realFileObjectNames = array_map(array('ZipArchiveExtended', 'getFileName'), $fileObjects);
foreach ($realFileObjectNames as $i => &$fileObject) {
$__fileName = $fileObject;
$__realFileName = $realFileObjectInfo[$i];
$filesize = filesize($intDir . DIRECTORY_SEPARATOR . $__realFileName);
$fileObject = array('name' => $__fileName, 'size' => $filesize);
$realFileObjectInfo[$i] = $fileObject;
}
$this->result['data'][$file_name] = json_encode($realFileObjectNames);
$stdFileObjects = array();
if ($fileObjects !== null) {
foreach ($fileObjects as $fName) {
if (isset($fileErrors) && isset($fileErrors->{$fName}) && !empty($fileErrors->{$fName}->error)) {
continue;
}
$newStdFile = new stdClass();
$newStdFile->name = $fName;
$stdFileObjects[] = $newStdFile;
}
} else {
$errors = $conversionHandler->getResult();
$errors = array_map(array('Upload', 'formatExceptionMessage'), $errors['errors']);
$this->result['errors'] = array_merge($this->result['errors'], $errors);
$this->api_output['message'] = "Zip Error";
$this->api_output['debug'] = $this->result['errors'];
return false;
}
/* Do conversions here */
$converter = new ConvertFileWrapper($stdFileObjects, false);
$converter->intDir = $intDir;
$converter->errDir = $errDir;
$converter->cookieDir = $cookieDir;
$converter->source_lang = $this->source_lang;
$converter->target_lang = $this->target_lang;
$converter->doAction();
$status = $errors = $converter->checkResult();
if (count($errors) > 0) {
// $this->result[ 'errors' ] = array_merge( $this->result[ 'errors' ], $errors );
$this->result['code'] = 2;
foreach ($errors as $__err) {
$brokenFileName = ZipArchiveExtended::getFileName($__err['debug']);
if (!isset($this->result['errors'][$brokenFileName])) {
$this->result['errors'][$brokenFileName] = array('code' => $__err['code'], 'message' => $__err['message'], 'debug' => $brokenFileName);
}
}
}
} else {
$conversionHandler->doAction();
$this->result = $conversionHandler->getResult();
if ($this->result['code'] > 0) {
$this->result = array();
}
}
}
$status = array_values($status);
if (!empty($status)) {
$this->api_output['message'] = 'Project Conversion Failure';
$this->api_output['debug'] = $status;
$this->result['errors'] = $status;
Log::doLog($status);
return -1;
}
/* Do conversions here */
if (isset($this->result['data']) && !empty($this->result['data'])) {
foreach ($this->result['data'] as $zipFileName => $zipFiles) {
$zipFiles = json_decode($zipFiles, true);
$fileNames = Utils::array_column($zipFiles, 'name');
$arFiles = array_merge($arFiles, $fileNames);
}
}
$newArFiles = array();
$linkFiles = scandir($intDir);
foreach ($arFiles as $__fName) {
if ('zip' == FilesStorage::pathinfo_fix($__fName, PATHINFO_EXTENSION)) {
$fs = new FilesStorage();
$fs->cacheZipArchive(sha1_file($intDir . DIRECTORY_SEPARATOR . $__fName), $intDir . DIRECTORY_SEPARATOR . $__fName);
$linkFiles = scandir($intDir);
//fetch cache links, created by converter, from upload directory
foreach ($linkFiles as $storedFileName) {
//check if file begins with the name of the zip file.
// If so, then it was stored in the zip file.
if (strpos($storedFileName, $__fName) !== false && substr($storedFileName, 0, strlen($__fName)) == $__fName) {
//add file name to the files array
$newArFiles[] = $storedFileName;
}
}
} else {
//this file was not in a zip. Add it normally
if (file_exists($intDir . DIRECTORY_SEPARATOR . $__fName)) {
$newArFiles[] = $__fName;
}
}
}
$arFiles = $newArFiles;
$projectManager = new ProjectManager();
$projectStructure = $projectManager->getProjectStructure();
$projectStructure['project_name'] = $this->project_name;
$projectStructure['job_subject'] = $this->subject;
$projectStructure['result'] = $this->result;
$projectStructure['private_tm_key'] = $this->private_tm_key;
$projectStructure['private_tm_user'] = $this->private_tm_user;
$projectStructure['private_tm_pass'] = $this->private_tm_pass;
$projectStructure['uploadToken'] = $uploadFile->getDirUploadToken();
$projectStructure['array_files'] = $arFiles;
//list of file name
$projectStructure['source_language'] = $this->source_lang;
$projectStructure['target_language'] = explode(',', $this->target_lang);
$projectStructure['mt_engine'] = $this->mt_engine;
$projectStructure['tms_engine'] = $this->tms_engine;
$projectStructure['status'] = Constants_ProjectStatus::STATUS_NOT_READY_FOR_ANALYSIS;
$projectStructure['skip_lang_validation'] = true;
$projectStructure['owner'] = $this->owner;
if ($this->current_user != null) {
$projectStructure['id_customer'] = $this->current_user->getEmail();
}
if ($this->current_user != null) {
$projectStructure['id_customer'] = $this->current_user->getEmail();
}
$projectManager = new ProjectManager($projectStructure);
$projectManager->createProject();
$this->result = $projectStructure['result'];
if (!empty($projectStructure['result']['errors'])) {
//errors already logged
$this->api_output['message'] = 'Project Creation Failure';
$this->api_output['debug'] = array_values($projectStructure['result']['errors']);
} else {
//everything ok
$this->api_output['status'] = 'OK';
$this->api_output['message'] = 'Success';
$this->api_output['id_project'] = $projectStructure['result']['id_project'];
$this->api_output['project_pass'] = $projectStructure['result']['ppassword'];
$this->api_output['new_keys'] = $this->new_keys;
$this->api_output['analyze_url'] = INIT::$HTTPHOST . "/analyze/" . $projectStructure['project_name'] . "/" . $projectStructure['result']['id_project'] . "-" . $projectStructure['result']['ppassword'];
}
}
public function doAction()
{
if (is_null($this->source) || $this->source === '') {
$this->result['errors'][] = array("code" => -1, "message" => "missing source segment");
}
if (is_null($this->target) || $this->target === '') {
$this->result['errors'][] = array("code" => -2, "message" => "missing target segment");
}
if (empty($this->source_lang)) {
$this->result['errors'][] = array("code" => -3, "message" => "missing source lang");
}
if (empty($this->target_lang)) {
$this->result['errors'][] = array("code" => -2, "message" => "missing target lang");
}
if (empty($this->id_job)) {
$this->result['errors'][] = array("code" => -4, "message" => "id_job not valid");
$msg = "\n\n Critical. Quit. \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
// critical. Quit.
return -1;
}
//get Job Infos, we need only a row of jobs ( split )
$job_data = getJobData((int) $this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($job_data)) {
$this->result['errors'][] = array("code" => -101, "message" => "error fetching job data");
}
if (empty($this->result['errors']) && !$pCheck->grantJobAccessByJobData($job_data, $this->password)) {
$this->result['errors'][] = array("code" => -10, "message" => "wrong password");
}
if (!empty($this->result['errors'])) {
$msg = "\n\n Error \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return -1;
}
$id_tms = $job_data['id_tms'];
$tm_keys = $job_data['tm_keys'];
if ($id_tms != 0) {
$tms = Engine::getInstance(1);
/**
* @var $tms Engines_MyMemory
*/
$config = $tms->getConfigStruct();
$config['segment'] = CatUtils::view2rawxliff($this->source);
$config['translation'] = CatUtils::view2rawxliff($this->target);
$config['source'] = $this->source_lang;
$config['target'] = $this->target_lang;
$config['email'] = "*****@*****.**";
//Props
$config['prop'] = json_encode(CatUtils::getTMProps($job_data));
//instantiate TMS object
$result = array();
$this->checkLogin();
try {
if (self::isRevision()) {
$this->userRole = TmKeyManagement_Filter::ROLE_REVISOR;
}
//find all the job's TMs with write grants and make a contribution to them
$tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($tm_keys, 'w', 'tm', $this->uid, $this->userRole);
if (!empty($tm_keys)) {
unset($config['id_user']);
foreach ($tm_keys as $i => $tm_info) {
$config['id_user'] = $tm_info->key;
$res = $tms->set($config);
if (!$res) {
$result[] = $res;
$this->result['errors'][] = array("code" => -5, "message" => "Set contribution error for key " . $tm_info->name);
}
}
} else {
$res = $tms->set($config);
if (!$res) {
$result[] = $res;
$this->result['errors'][] = array("code" => -5, "message" => "Set contribution error");
}
}
} catch (Exception $e) {
$this->result['errors'][] = array("code" => -6, "message" => "Error while retrieving job's TM.");
Log::doLog(__METHOD__ . " -> " . $e->getMessage());
}
//if translator_username is empty no key is added to MyMemory API SET query string, so, anonymous by default
if (count($result)) {
$this->result['code'] = -1;
$this->result['data'] = "KO";
Log::doLog("Set Contribution Failed.");
Log::doLog(var_export($_POST, true));
return -1;
}
$this->result['code'] = 1;
$this->result['data'] = "OK";
} else {
$this->result['code'] = 1;
$this->result['data'] = "NOCONTRIB_OK";
}
}
public function doAction()
{
//get job language and data
//Fixed Bug: need a specific job, because we need The target Language
//Removed from within the foreach cycle, the job is always the same....
$jobData = $this->jobInfo = getJobData($this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($jobData) || !$pCheck->grantJobAccessByJobData($jobData, $this->password)) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
//get storage object
$fs = new FilesStorage();
$files_job = $fs->getFilesForJob($this->id_job, $this->id_file);
$nonew = 0;
$output_content = array();
/*
the procedure:
1)original xliff file is read directly from disk; a file handler is obtained
2)the file is read chunk by chunk by a stream parser: for each trans-unit that is encountered, target is replaced (or added) with the corresponding translation obtained from the DB
3)the parsed portion of xliff in the buffer is flushed on temporary file
4)the temporary file is sent to the converter and an original file is obtained
5)the temporary file is deleted
*/
// This array will contain all the files of $files_job split by
// converter version.
$files_job_by_converter_version = array();
// Detect the converter's version to use for each file, then store
// file info accordingly.
foreach ($files_job as $file) {
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
$files_job_by_converter_version[$fileType['converter_version']][] = $file;
}
// Process files according to the converters' versions, one version
// at a time
foreach ($files_job_by_converter_version as $converter_version => $files_job) {
//file array is chuncked. Each chunk will be used for a parallel conversion request.
$files_job = array_chunk($files_job, self::FILES_CHUNK_SIZE);
foreach ($files_job as $chunk) {
$converter = new FileFormatConverter($converter_version);
$files_to_be_converted = array();
foreach ($chunk as $file) {
$mime_type = $file['mime_type'];
$fileID = $file['id_file'];
$current_filename = $file['filename'];
//get path for the output file converted to know it's right extension
$_fileName = explode(DIRECTORY_SEPARATOR, $file['xliffFilePath']);
$outputPath = INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/' . $fileID . '/' . uniqid('', true) . "_.out." . array_pop($_fileName);
//make dir if doesn't exist
if (!file_exists(dirname($outputPath))) {
Log::doLog('Create Directory ' . escapeshellarg(dirname($outputPath)) . '');
mkdir(dirname($outputPath), 0775, true);
}
$data = getSegmentsDownload($this->id_job, $this->password, $fileID, $nonew);
//prepare regexp for nest step
$regexpEntity = '/&#x(0[0-8BCEF]|1[0-9A-F]|7F);/u';
$regexpAscii = '/([\\x{00}-\\x{1F}\\x{7F}]{1})/u';
foreach ($data as $i => $k) {
//create a secondary indexing mechanism on segments' array; this will be useful
//prepend a string so non-trans unit id ( ex: numerical ) are not overwritten
$data['matecat|' . $k['internal_id']][] = $i;
//FIXME: temporary patch
$data[$i]['translation'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['translation']);
$data[$i]['segment'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['segment']);
//remove binary chars in some xliff files
$sanitized_src = preg_replace($regexpAscii, '', $data[$i]['segment']);
$sanitized_trg = preg_replace($regexpAscii, '', $data[$i]['translation']);
//clean invalid xml entities ( charactes with ascii < 32 and different from 0A, 0D and 09
$sanitized_src = preg_replace($regexpEntity, '', $sanitized_src);
$sanitized_trg = preg_replace($regexpEntity, '', $sanitized_trg);
if ($sanitized_src != null) {
$data[$i]['segment'] = $sanitized_src;
}
if ($sanitized_trg != null) {
$data[$i]['translation'] = $sanitized_trg;
}
}
//instatiate parser
$xsp = new SdlXliffSAXTranslationReplacer($file['xliffFilePath'], $data, Langs_Languages::getInstance()->getLangRegionCode($jobData['target']), $outputPath);
if ($this->download_type == 'omegat') {
$xsp->setSourceInTarget(true);
}
//run parsing
Log::doLog("work on " . $fileID . " " . $current_filename);
$xsp->replaceTranslation();
//free memory
unset($xsp);
unset($data);
$output_content[$fileID]['document_content'] = file_get_contents($outputPath);
$output_content[$fileID]['output_filename'] = $current_filename;
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
if ($this->forceXliff) {
//clean the output filename by removing
// the unique hash identifier 55e5739b467109.05614837_.out.Test_English.doc.sdlxliff
$output_content[$fileID]['output_filename'] = preg_replace('#[0-9a-f]+\\.[0-9_]+\\.out\\.#i', '', FilesStorage::basename_fix($outputPath));
if ($fileType['proprietary_short_name'] === 'matecat_converter') {
// Set the XLIFF extension to .xlf
// Internally, MateCat continues using .sdlxliff as default
// extension for the XLIFF behind the projects.
// Changing this behavior requires a huge refactoring that
// it's scheduled for future versions.
// We quickly fixed the behaviour from the user standpoint
// using the following line of code, that changes the XLIFF's
// extension just a moment before it is downloaded by the user.
$output_content[$fileID]['output_filename'] = preg_replace("|\\.sdlxliff\$|i", ".xlf", $output_content[$fileID]['output_filename']);
}
}
/**
* Conversion Enforce
*/
$convertBackToOriginal = true;
//if it is a not converted file ( sdlxliff ) we have originalFile equals to xliffFile (it has just been copied)
$file['original_file'] = file_get_contents($file['originalFilePath']);
// When the 'proprietary' flag is set to false, the xliff
// is not passed to any converter, because is handled
// directly inside MateCAT.
$xliffWasNotConverted = $fileType['proprietary'] === false;
if (!INIT::$CONVERSION_ENABLED || ($file['originalFilePath'] == $file['xliffFilePath'] and $xliffWasNotConverted) or $this->forceXliff) {
$convertBackToOriginal = false;
Log::doLog("SDLXLIFF: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
} else {
//TODO: dos2unix ??? why??
//force unix type files
Log::doLog("NO SDLXLIFF, Conversion enforced: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
}
if ($convertBackToOriginal) {
$output_content[$fileID]['out_xliff_name'] = $outputPath;
$output_content[$fileID]['source'] = $jobData['source'];
$output_content[$fileID]['target'] = $jobData['target'];
$files_to_be_converted[$fileID] = $output_content[$fileID];
} elseif ($this->forceXliff) {
$this->cleanFilePath($output_content[$fileID]['document_content']);
}
}
$convertResult = $converter->multiConvertToOriginal($files_to_be_converted, $chosen_machine = false);
foreach (array_keys($files_to_be_converted) as $fileID) {
$output_content[$fileID]['document_content'] = $this->ifGlobalSightXliffRemoveTargetMarks($convertResult[$fileID]['document_content'], $files_to_be_converted[$fileID]['output_filename']);
//in case of .strings, they are required to be in UTF-16
//get extension to perform file detection
$extension = FilesStorage::pathinfo_fix($output_content[$fileID]['output_filename'], PATHINFO_EXTENSION);
if (strtoupper($extension) == 'STRINGS') {
//use this function to convert stuff
$encodingConvertedFile = CatUtils::convertEncoding('UTF-16', $output_content[$fileID]['document_content']);
//strip previously added BOM
$encodingConvertedFile[1] = $converter->stripBOM($encodingConvertedFile[1], 16);
//store new content
$output_content[$fileID]['document_content'] = $encodingConvertedFile[1];
//trash temporary data
unset($encodingConvertedFile);
}
}
unset($convertResult);
}
}
foreach ($output_content as $idFile => $fileInformations) {
$zipPathInfo = ZipArchiveExtended::zipPathInfo($output_content[$idFile]['output_filename']);
if (is_array($zipPathInfo)) {
$output_content[$idFile]['zipfilename'] = $zipPathInfo['zipfilename'];
$output_content[$idFile]['zipinternalPath'] = $zipPathInfo['dirname'];
$output_content[$idFile]['output_filename'] = $zipPathInfo['basename'];
}
}
//set the file Name
$pathinfo = FilesStorage::pathinfo_fix($this->fname);
$this->_filename = $pathinfo['filename'] . "_" . $jobData['target'] . "." . $pathinfo['extension'];
//qui prodest to check download type?
if ($this->download_type == 'omegat') {
$this->_filename .= ".zip";
$tmsService = new TMSService();
$tmsService->setOutputType('tm');
/**
* @var $tmFile SplTempFileObject
*/
$tmFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tmsService->setOutputType('mt');
/**
* @var $mtFile SplTempFileObject
*/
$mtFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tm_id = uniqid('tm');
$mt_id = uniqid('mt');
$output_content[$tm_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_TM . tmx");
foreach ($tmFile as $lineNumber => $content) {
$output_content[$tm_id]['document_content'] .= $content;
}
$output_content[$mt_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_MT . tmx");
foreach ($mtFile as $lineNumber => $content) {
$output_content[$mt_id]['document_content'] .= $content;
}
$this->createOmegaTZip($output_content, $jobData['source'], $jobData['target']);
//add zip archive content here;
} else {
try {
$output_content = $this->getOutputContentsWithZipFiles($output_content);
if (count($output_content) > 1) {
//cast $output_content elements to ZipContentObject
foreach ($output_content as $key => $__output_content_elem) {
$output_content[$key] = new ZipContentObject($__output_content_elem);
}
if ($pathinfo['extension'] != 'zip') {
if ($this->forceXliff) {
$this->_filename = $this->id_job . ".zip";
} else {
$this->_filename = $pathinfo['basename'] . ".zip";
}
}
$this->content = self::composeZip($output_content);
//add zip archive content here;
} else {
//always an array with 1 element, pop it, Ex: array( array() )
$output_content = array_pop($output_content);
$this->setContent($output_content);
}
} catch (Exception $e) {
$msg = "\n\n Error retrieving file content, Conversion failed??? \n\n Error: {$e->getMessage()} \n\n" . var_export($e->getTraceAsString(), true);
$msg .= "\n\n Request: " . var_export($_REQUEST, true);
Log::$fileName = 'fatal_errors.txt';
Log::doLog($msg);
Utils::sendErrMailReport($msg);
$this->unlockToken(array("code" => -110, "message" => "Download failed. Please contact " . INIT::$SUPPORT_MAIL));
throw $e;
// avoid sent Headers and empty file content with finalize method
}
}
try {
Utils::deleteDir(INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/');
} catch (Exception $e) {
Log::doLog('Failed to delete dir:' . $e->getMessage());
}
}
public function setTemplateVars()
{
$this->template->jobs = $this->jobs;
$this->template->fast_analysis_wc = $this->fast_analysis_wc;
$this->template->fast_analysis_wc_print = $this->fast_analysis_wc_print;
$this->template->tm_analysis_wc = $this->tm_analysis_wc;
$this->template->tm_analysis_wc_print = $this->tm_analysis_wc_print;
$this->template->standard_analysis_wc = $this->standard_analysis_wc;
$this->template->standard_analysis_wc_print = $this->standard_analysis_wc_print;
$this->template->total_raw_word_count = $this->total_raw_word_count;
$this->template->total_raw_word_count_print = $this->total_raw_word_count_print;
$this->template->pname = $this->pname;
$this->template->pid = $this->pid;
$this->template->project_password = $this->ppassword;
$this->template->project_not_found = $this->project_not_found;
$this->template->fast_wc_time = $this->fast_wc_time;
$this->template->tm_wc_time = $this->tm_wc_time;
$this->template->tm_wc_unit = $this->tm_wc_unit;
$this->template->fast_wc_unit = $this->fast_wc_unit;
$this->template->standard_wc_unit = $this->standard_wc_unit;
$this->template->raw_wc_time = $this->raw_wc_time;
$this->template->standard_wc_time = $this->standard_wc_time;
$this->template->raw_wc_unit = $this->raw_wc_unit;
$this->template->project_status = $this->project_status;
$this->template->num_segments = $this->num_segments;
$this->template->num_segments_analyzed = $this->num_segments_analyzed;
$this->template->logged_user = $this->logged_user !== false ? $this->logged_user->shortName() : "";
$this->template->extended_user = $this->logged_user !== false ? trim($this->logged_user->fullName()) : "";
$this->template->build_number = INIT::$BUILD_NUMBER;
$this->template->enable_outsource = INIT::$ENABLE_OUTSOURCE;
$this->template->outsource_service_login = $this->_outsource_login_API;
$this->template->support_mail = INIT::$SUPPORT_MAIL;
$langDomains = Langs_LanguageDomains::getInstance();
$this->subject = $langDomains::getDisplayDomain($this->subject);
$this->template->subject = $this->subject;
$this->template->isLoggedIn = $this->isLoggedIn();
if (isset($_SESSION['_anonym_pid']) && !empty($_SESSION['_anonym_pid'])) {
$_SESSION['incomingUrl'] = INIT::$HTTPHOST . $_SERVER['REQUEST_URI'];
$_SESSION['_newProject'] = 1;
$this->template->showModalBoxLogin = true;
} else {
$this->template->showModalBoxLogin = false;
}
//url to which to send data in case of login
$client = OauthClient::getInstance()->getClient();
$this->template->oauthFormUrl = $client->createAuthUrl();
$this->template->incomingUrl = '/login?incomingUrl=' . $_SERVER['REQUEST_URI'];
//perform check on running daemons and send a mail randomly
$misconfiguration = Analysis_Manager::thereIsAMisconfiguration();
if ($misconfiguration && mt_rand(1, 3) == 1) {
$msg = "<strong>The analysis daemons seem not to be running despite server configuration.</strong><br />Change the application configuration or start analysis daemons.";
Utils::sendErrMailReport($msg, "Matecat Misconfiguration");
}
$this->template->daemon_misconfiguration = var_export($misconfiguration, true);
}
public static function fatalErrorHandler()
{
$errorType = array(E_CORE_ERROR => 'E_CORE_ERROR', E_COMPILE_ERROR => 'E_COMPILE_ERROR', E_ERROR => 'E_ERROR', E_USER_ERROR => 'E_USER_ERROR', E_RECOVERABLE_ERROR => 'E_RECOVERABLE_ERROR');
# Getting last error
$error = error_get_last();
# Checking if last error is a fatal error
switch ($error['type']) {
case E_CORE_ERROR:
case E_COMPILE_ERROR:
case E_ERROR:
case E_USER_ERROR:
case E_RECOVERABLE_ERROR:
ini_set('display_errors', 'Off');
if (!ob_get_level()) {
ob_start();
} else {
ob_end_clean();
ob_start();
}
debug_print_backtrace();
$output = ob_get_contents();
ob_end_clean();
# Here we handle the error, displaying HTML, logging, ...
$output .= "<pre>\n";
$output .= "[ {$errorType[$error['type']]} ]\n\t";
$output .= "{$error['message']}\n\t";
$output .= "Not Recoverable Error on line {$error['line']} in file " . $error['file'];
$output .= " - PHP " . PHP_VERSION . " (" . PHP_OS . ")\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::$fileName = 'fatal_errors.txt';
Log::doLog($output);
Utils::sendErrMailReport($output);
header("HTTP/1.1 200 OK");
if (isset($_SERVER['HTTP_X_REQUESTED_WITH']) && strtolower($_SERVER['HTTP_X_REQUESTED_WITH']) == 'xmlhttprequest' || $_SERVER['REQUEST_METHOD'] == 'POST') {
//json_rersponse
if (INIT::$EXCEPTION_DEBUG) {
echo json_encode(array("errors" => array(array("code" => -1000, "message" => $output)), "data" => array()));
} else {
echo json_encode(array("errors" => array(array("code" => -1000, "message" => "Oops we got an Error. Contact <a href='mailto:support@matecat.com'>support@matecat.com</a>")), "data" => array()));
}
} elseif (INIT::$EXCEPTION_DEBUG) {
echo $output;
}
break;
}
}
function main($args)
{
$db = Database::obtain();
$lastProcessedJob = (int) file_get_contents(self::$last_job_file_name);
do {
$queryMaxJob = "select min(id) as min, max(id) as max\n from jobs\n where completed = 1\n and id > %d";
$queryFirst = "select id, password, job_first_segment, job_last_segment\n from jobs\n where completed = 1\n and id >= %d and id <= %d";
$querySegments = "select suggestion,\n translation,\n raw_word_count,\n time_to_edit\n from segment_translations st\n join segments s on st.id_segment = s.id\n and s.id between %d and %d\n where status='translated'\n and id_job = %d\n and show_in_cattool = 1\n and id_segment > %d\n limit %d";
$queryUpdateJob = "update jobs\n set avg_post_editing_effort = %f,\n total_time_to_edit = %f\n where id = %d and password = '******'";
$minJobMaxJob = $db->query_first(sprintf($queryMaxJob, (int) $lastProcessedJob));
$maxJob = (int) $minJobMaxJob['max'];
$minJob = (int) $minJobMaxJob['min'];
$start = time();
for ($firstJob = $minJob; self::$RUNNING && $firstJob < $maxJob; $firstJob += self::NR_OF_JOBS) {
$jobs = $db->fetch_array(sprintf($queryFirst, $firstJob, $firstJob + self::NR_OF_JOBS));
//iterate over completed jobs, evaluate PEE and save it in the job row
for ($j = 0; self::$RUNNING && $j < count($jobs); $j++) {
$job = $jobs[$j];
//BEGIN TRANSACTION
$db->begin();
$_jid = $job['id'];
$_password = $job['password'];
$_job_first_segment = $job['job_first_segment'];
$_job_last_segment = $job['job_last_segment'];
Log::doLog("job {$_jid} -> " . ($_job_last_segment - $_job_first_segment) . " segments");
echo "job {$_jid} -> " . ($_job_last_segment - $_job_first_segment) . " segments\n";
$raw_post_editing_effort_job = 0;
$raw_wc_job = 0;
$time_to_edit_job = 0;
for ($firstSeg = $_job_first_segment; $firstSeg <= $_job_last_segment; $firstSeg += self::NR_OF_SEGS) {
if ($firstSeg > $_job_last_segment) {
$firstSeg = $_job_last_segment;
}
Log::doLog("starting from segment {$firstSeg}");
echo "starting from segment {$firstSeg}\n";
$segments = $db->fetch_array(sprintf($querySegments, $_job_first_segment, $_job_last_segment, $_jid, $firstSeg, self::NR_OF_SEGS));
foreach ($segments as $i => $segment) {
$post_editing_effort = round((1 - MyMemory::TMS_MATCH($segment['suggestion'], $segment['translation'])) * 100);
if ($post_editing_effort < 0) {
$post_editing_effort = 0;
} else {
if ($post_editing_effort > 100) {
$post_editing_effort = 100;
}
}
$raw_wc_job += $segment['raw_word_count'];
$time_to_edit_job += $segment['time_to_edit'];
$raw_post_editing_effort_job += $post_editing_effort * $segment['raw_word_count'];
}
//sleep 100 nanosecs
usleep(100);
}
$job_pee = round($raw_post_editing_effort_job / $raw_wc_job, 3);
Log::doLog("job pee: {$job_pee}\njob time to edit: {$time_to_edit_job}\nWriting into DB");
echo "job pee: {$job_pee}\njob time to edit: {$time_to_edit_job}\nWriting into DB\n";
$db->query(sprintf($queryUpdateJob, $job_pee, $time_to_edit_job, $_jid, $_password));
Log::doLog("done");
echo "done.\n";
if (!file_put_contents(self::$last_job_file_name, $_jid)) {
$db->rollback();
Utils::sendErrMailReport("", "[JobPostEditingEffortRunner] Failed to process job {$_jid}");
self::$RUNNING = false;
continue;
//exit;
}
//COMMIT TRANSACTION
$db->commit();
}
}
Log::doLog("took " . (time() - $start) / 60 . " seconds");
echo "took " . (time() - $start) / 60 . " seconds\n";
Log::doLog("sleeping for 1 month");
echo "sleeping for 1 month\n";
if (self::$RUNNING) {
sleep(self::$sleeptime);
}
} while (self::$RUNNING);
}
public function mergeALL(ArrayObject $projectStructure, $renewPassword = false)
{
$query_job = "SELECT *\n FROM jobs\n WHERE id = %u\n ORDER BY job_first_segment";
$query_job = sprintf($query_job, $projectStructure['job_to_merge']);
//$projectStructure[ 'job_to_split' ]
$rows = $this->dbHandler->fetch_array($query_job);
//get the min and
$first_job = reset($rows);
$job_first_segment = $first_job['job_first_segment'];
//the max segment from job list
$last_job = end($rows);
$job_last_segment = $last_job['job_last_segment'];
//change values of first job
$first_job['job_first_segment'] = $job_first_segment;
// redundant
$first_job['job_last_segment'] = $job_last_segment;
//merge TM keys: preserve only owner's keys
$tm_keys = array();
foreach ($rows as $chunk_info) {
$tm_keys[] = $chunk_info['tm_keys'];
}
try {
$owner_tm_keys = TmKeyManagement_TmKeyManagement::getOwnerKeys($tm_keys);
/**
* @var $owner_key TmKeyManagement_TmKeyStruct
*/
foreach ($owner_tm_keys as $i => $owner_key) {
$owner_tm_keys[$i] = $owner_key->toArray();
}
$first_job['tm_keys'] = json_encode($owner_tm_keys);
} catch (Exception $e) {
Log::doLog(__METHOD__ . " -> Merge Jobs error - TM key problem: " . $e->getMessage());
}
$oldPassword = $first_job['password'];
if ($renewPassword) {
$first_job['password'] = self::_generatePassword();
}
$_data = array();
foreach ($first_job as $field => $value) {
$_data[] = "`{$field}`='{$value}'";
}
//----------------------------------------------------
$queries = array();
$queries[] = "UPDATE jobs SET " . implode(", \n", $_data) . " WHERE id = {$first_job['id']} AND password = '******'";
//ose old password
//delete all old jobs
$queries[] = "DELETE FROM jobs WHERE id = {$first_job['id']} AND password != '{$first_job['password']}' ";
//use new password
foreach ($queries as $query) {
$res = $this->dbHandler->query($query);
if ($res !== true) {
$msg = "Failed to merge job " . $rows[0]['id'] . " from " . count($rows) . " chunks\n";
$msg .= "Tried to perform SQL: \n" . print_r($queries, true) . " \n\n";
$msg .= "Failed Statement is: \n" . print_r($query, true) . "\n";
$msg .= "Original Status for rebuild job and project was: \n" . print_r($rows, true) . "\n";
Utils::sendErrMailReport($msg);
throw new Exception('Failed to merge jobs, project damaged. Contact Matecat Support to rebuild project.', -8);
}
}
$wCountManager = new WordCount_Counter();
$wCountManager->initializeJobWordCount($first_job['id'], $first_job['password']);
Shop_Cart::getInstance('outsource_to_external_cache')->emptyCart();
}
/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return mixed
*/
function doAction()
{
try {
$this->streamFilePointer = $this->tmxHandler->downloadTMX();
} catch (Exception $e) {
$r = "<pre>";
$r .= print_r("User Email: " . $this->userMail, true);
$r .= print_r("User ID: " . $this->uid, true);
$r .= print_r($e->getMessage(), true);
$r .= print_r($e->getTraceAsString(), true);
$r .= "\n\n";
$r .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$r .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$r .= "\n\n\n";
$r .= "</pre>";
Log::$fileName = 'php_errors.txt';
Log::doLog($r);
Utils::sendErrMailReport($r, "Download TMX Error: " . $e->getMessage());
$this->unlockToken();
echo $e->getMessage();
exit;
}
}
