public function __construct()
{
//SESSION ENABLED
parent::sessionStart();
parent::__construct();
$filterArgs = array('page' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'step' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'project' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'filter' => array('filter' => FILTER_VALIDATE_BOOLEAN, 'options' => array(FILTER_NULL_ON_FAILURE)), 'pn' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_HIGH | FILTER_FLAG_STRIP_LOW), 'source' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_HIGH | FILTER_FLAG_STRIP_LOW), 'target' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_HIGH | FILTER_FLAG_STRIP_LOW), 'status' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_HIGH | FILTER_FLAG_STRIP_LOW), 'onlycompleted' => array('filter' => FILTER_VALIDATE_BOOLEAN, 'options' => array(FILTER_NULL_ON_FAILURE)));
$postInput = filter_input_array(INPUT_POST, $filterArgs);
// assigning default values
if (is_null($postInput['page']) || empty($postInput['page'])) {
$postInput['page'] = 1;
}
if (is_null($postInput['step']) || empty($postInput['step'])) {
$postInput['step'] = 25;
}
if (is_null($postInput['status']) || empty($postInput['status'])) {
$postInput['status'] = Constants_JobStatus::STATUS_ACTIVE;
}
$this->lang_handler = Langs_Languages::getInstance();
$this->page = (int) $postInput['page'];
$this->step = (int) $postInput['step'];
$this->project_id = $postInput['project'];
$this->filter_enabled = (int) $postInput['filter'];
$this->search_in_pname = (string) $postInput['pn'];
$this->search_source = (string) $postInput['source'];
$this->search_target = (string) $postInput['target'];
$this->search_status = (string) $postInput['status'];
$this->search_onlycompleted = $postInput['onlycompleted'];
}
public static function getInstance()
{
if (!self::$instance) {
self::$instance = new Langs_Languages();
}
return self::$instance;
}
/**
* Create a new FileFormatConverter, using old or new converters.
* This function looks for converters in the 'converters' table in the db.
* $converterVersion can be "legacy", "latest", or something like "1.0.0".
* In the first case legacy converters will be used; in the second case,
* the latest version of new converters will be used; in the third case,
* this function will look for the converters with the provided version,
* and if not found will use the converters with higher but closest version.
* Version check is done on the conversion_api_version field of the
* converters db table; new converters are expected to have a value like
* "open 1.0.0".
*/
public function __construct($converterVersion = null)
{
$this->converterVersion = $converterVersion;
$this->opt['httpheader'] = array("Content-Type:multipart/form-data;charset=UTF-8");
$this->lang_handler = Langs_Languages::getInstance();
$this->conversionObject = new ArrayObject(array('ip_machine' => null, 'ip_client' => null, 'path_name' => null, 'file_name' => null, 'path_backup' => null, 'file_size' => 0, 'direction' => null, 'error_message' => null, 'src_lang' => null, 'trg_lang' => null, 'status' => 'ok', 'conversion_time' => 0), ArrayObject::ARRAY_AS_PROPS);
// Get converters instances list from database,
$db = Database::obtain();
// The base query to obtain the converters
$baseQuery = 'SELECT ip_converter, cpu_weight, ip_storage, segmentation_rule' . ' FROM converters' . ' WHERE status_active = 1 AND status_offline = 0';
// Complete the $baseQuery according to the converter's version
if ($this->converterVersion == Constants_ConvertersVersions::LEGACY) {
// Retrieve only old converters
$query = $baseQuery . (INIT::$USE_ONLY_STABLE_CONVERTERS ? ' AND stable = 1' : '') . ' AND conversion_api_version NOT LIKE "open %"';
$converters = $db->fetch_array($query);
} else {
// Here we use new converters
if ($this->converterVersion == Constants_ConvertersVersions::LATEST) {
// Get the converters with the latest version
$query = $baseQuery . ' AND conversion_api_version = (' . 'SELECT MAX(conversion_api_version)' . ' FROM converters' . ' WHERE conversion_api_version LIKE "open %"' . (INIT::$USE_ONLY_STABLE_CONVERTERS ? ' AND stable = 1' : '') . ' AND status_active = 1 AND status_offline = 0' . ')';
$converters = $db->fetch_array($query);
} else {
$converters = self::__getSuitableConvertersByVersion($db, $this->converterVersion);
}
}
foreach ($converters as $converter_storage) {
$this->converters[$converter_storage['ip_converter']] = $converter_storage['cpu_weight'];
$this->storage_lookup_map[$converter_storage['ip_converter']] = $converter_storage['ip_storage'];
$this->converter2segmRule[$converter_storage['ip_converter']] = $converter_storage['segmentation_rule'];
}
// $this->converters = array('10.30.1.32' => 1);//for debugging purposes
// $this->storage_lookup_map = array('10.30.1.32' => '10.30.1.32');//for debugging purposes
}
public function __construct()
{
parent::__construct(false);
parent::makeTemplate("upload.html");
$this->guid = Utils::create_guid();
$this->lang_handler = Langs_Languages::getInstance();
$this->subject_handler = Langs_LanguageDomains::getInstance();
$this->subjectArray = $this->subject_handler->getEnabledDomains();
}
public function __construct()
{
parent::__construct(false);
parent::makeTemplate("upload.html");
$filterArgs = array('project_name' => array('filter' => FILTER_SANITIZE_STRING), 'private_tm_key' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH));
$__postInput = filter_input_array(INPUT_GET, $filterArgs);
$this->project_name = $__postInput["project_name"];
$this->private_tm_key = $__postInput["private_tm_key"];
$this->guid = Utils::create_guid();
$this->lang_handler = Langs_Languages::getInstance();
$this->subject_handler = Langs_LanguageDomains::getInstance();
$this->subjectArray = $this->subject_handler->getEnabledDomains();
}
public function __construct()
{
parent::__construct(true);
parent::makeTemplate("manage.html");
$filterArgs = array('page' => array('filter' => array(FILTER_SANITIZE_NUMBER_INT)), 'filter' => array('filter' => array(FILTER_VALIDATE_BOOLEAN), 'options' => array(FILTER_NULL_ON_FAILURE)));
$postInput = filter_input_array(INPUT_GET, $filterArgs);
$this->page = $postInput['page'];
if ($this->page == null || empty($this->page)) {
$this->page = 1;
}
$this->lang_handler = Langs_Languages::getInstance();
if ($postInput['filter'] !== null && $postInput['filter']) {
$this->filter_enabled = true;
} else {
$this->filter_enabled = false;
}
}
/**
* Create a new FileFormatConverter, using old or new converters.
* This function looks for converters in the 'converters' table in the db.
* $converterVersion can be "legacy", "latest", or something like "1.0.0".
* In the first case legacy converters will be used; in the second case,
* the latest version of new converters will be used; in the third case,
* this function will look for the converters with the provided version,
* and if not found will use the converters with higher but closest version.
* Version check is done on the conversion_api_version field of the
* converters db table; new converters are expected to have a value like
* "open 1.0.0".
*/
public function __construct($converterVersion = null)
{
$this->converterVersion = $converterVersion;
$this->opt['httpheader'] = array("Content-Type:multipart/form-data;charset=UTF-8");
$this->lang_handler = Langs_Languages::getInstance();
$this->conversionObject = new ArrayObject(array('ip_machine' => null, 'ip_client' => null, 'path_name' => null, 'file_name' => null, 'path_backup' => null, 'file_size' => 0, 'direction' => null, 'error_message' => null, 'src_lang' => null, 'trg_lang' => null, 'status' => 'ok', 'conversion_time' => 0), ArrayObject::ARRAY_AS_PROPS);
// Get converters instances list from database,
$db = Database::obtain();
// The base query to obtain the converters
$baseQuery = 'SELECT ip_converter, cpu_weight, ip_storage, segmentation_rule' . ' FROM converters' . ' WHERE status_active = 1 AND status_offline = 0';
// Complete the $baseQuery according to the converter's version
if ($this->converterVersion == Constants_ConvertersVersions::LEGACY) {
// Retrieve only old converters
$query = $baseQuery . (INIT::$USE_ONLY_STABLE_CONVERTERS ? ' AND stable = 1' : '') . ' AND conversion_api_version NOT LIKE "open %"';
} else {
// Here we use new converters
if ($this->converterVersion == Constants_ConvertersVersions::LATEST) {
// Get the converters with the latest version
$query = $baseQuery . ' AND conversion_api_version = (' . 'SELECT MAX(conversion_api_version)' . ' FROM converters' . ' WHERE conversion_api_version LIKE "open %"' . (INIT::$USE_ONLY_STABLE_CONVERTERS ? ' AND stable = 1' : '') . ' AND status_active = 1 AND status_offline = 0' . ')';
} else {
$closest_conversion_api_version = self::getClosestConversionApiVersion($this->converterVersion);
$query = $baseQuery . (INIT::$USE_ONLY_STABLE_CONVERTERS ? ' AND stable = 1' : '') . ' AND conversion_api_version = "' . $closest_conversion_api_version . '"';
}
}
$converters = $db->fetch_array($query);
// SUUUPER ugly, those variables should not be static at all! No way!
// But now the class works around these 3 static variables, and a
// refactoring is too risky. Enter this patch: I empty these 3 vars
// every time I create a new FileFormatConverter to be sure that new
// converters never mix with old converters
self::$converters = array();
self::$Storage_Lookup_IP_Map = array();
self::$converter2segmRule = array();
foreach ($converters as $converter_storage) {
self::$converters[$converter_storage['ip_converter']] = $converter_storage['cpu_weight'];
self::$Storage_Lookup_IP_Map[$converter_storage['ip_converter']] = $converter_storage['ip_storage'];
self::$converter2segmRule[$converter_storage['ip_converter']] = $converter_storage['segmentation_rule'];
}
// self::$converters = array('10.30.1.32' => 1);//for debugging purposes
// self::$Storage_Lookup_IP_Map = array('10.30.1.32' => '10.30.1.32');//for debugging purposes
$this->storage_lookup_map = self::$Storage_Lookup_IP_Map;
}
public function __construct()
{
if (!class_exists("INIT")) {
include_once "../../inc/config.inc.php";
INIT::obtain();
}
$this->opt['httpheader'] = array("Content-Type:multipart/form-data;charset=UTF-8");
$this->lang_handler = Langs_Languages::getInstance();
$this->conversionObject = new ArrayObject(array('ip_machine' => null, 'ip_client' => null, 'path_name' => null, 'file_name' => null, 'path_backup' => null, 'file_size' => 0, 'direction' => null, 'error_message' => null, 'src_lang' => null, 'trg_lang' => null, 'status' => 'ok', 'conversion_time' => 0), ArrayObject::ARRAY_AS_PROPS);
$db = Database::obtain();
$converters = $db->fetch_array("SELECT ip_converter, cpu_weight, ip_storage, segmentation_rule FROM converters WHERE status_active = 1 AND status_offline = 0");
foreach ($converters as $converter_storage) {
self::$converters[$converter_storage['ip_converter']] = $converter_storage['cpu_weight'];
self::$Storage_Lookup_IP_Map[$converter_storage['ip_converter']] = $converter_storage['ip_storage'];
self::$converter2segmRule[$converter_storage['ip_converter']] = $converter_storage['segmentation_rule'];
}
// self::$converters = array('10.11.0.98' => 1);//for debugging purposes
// self::$Storage_Lookup_IP_Map = array('10.11.0.98' => '10.11.0.99');//for debugging purposes
$this->storage_lookup_map = self::$Storage_Lookup_IP_Map;
}
function main($args)
{
$db = Database::obtain();
do {
//TODO: create DAO for this
$today = date("Y-m-d");
$queryJobs = "SELECT\n source,\n target,\n sum( total_time_to_edit ) as total_time_to_edit,\n sum(translated_words) + sum(approved_words) + sum(rejected_words) as total_words,\n sum( COALESCE (avg_post_editing_effort, 0) ) as total_post_editing_effort,\n count(*) as job_count\n FROM\n jobs j\n WHERE\n completed = 1\n AND source = '%s'\n GROUP BY target";
$queryInsert = "INSERT into language_stats\n (date, source, target, total_word_count, total_post_editing_effort, total_time_to_edit, job_count)\n VALUES %s\n ON DUPLICATE KEY UPDATE\n total_post_editing_effort = values( total_post_editing_effort ),\n total_time_to_edit = values( total_time_to_edit ),\n job_count = values( job_count )";
$updateTuplesTemplate = "( '%s', '%s', '%s', %f, %f, %f, %u )";
$langsObj = Langs_Languages::getInstance();
//getlanguage list
$languages = $langsObj->getEnabledLanguages();
$languages = Utils::array_column($languages, 'code');
foreach ($languages as $source_language) {
Log::doLog("Current source_language: {$source_language}");
echo "Current source_language: {$source_language}\n";
$languageStats = $db->fetch_array(sprintf($queryJobs, $source_language));
$languageTuples = array();
foreach ($languageStats as $languageCoupleStat) {
Log::doLog("Current language couple: " . $source_language . "-" . $languageCoupleStat['target']);
echo "Current language couple: " . $source_language . "-" . $languageCoupleStat['target'] . "\n";
$languageTuples[] = sprintf($updateTuplesTemplate, $today, $languageCoupleStat['source'], $languageCoupleStat['target'], round($languageCoupleStat['total_words'], 4), round($languageCoupleStat['total_post_editing_effort'], 4), round($languageCoupleStat['total_time_to_edit'], 4), $languageCoupleStat['job_count']);
}
if (count($languageTuples) > 0) {
Log::doLog("Found some stats. Saving in DB..");
echo "Found some stats. Saving in DB..\n";
$db->query(sprintf($queryInsert, implode(", ", $languageTuples)));
}
usleep(100);
}
//for the moment, this daemon is single-loop-execution
self::$RUNNING = false;
if (self::$RUNNING) {
sleep(self::$sleeptime);
}
} while (self::$RUNNING);
}
/**
* This function executes a language detection call to mymemory for an array of segments,
* located in projectStructure
*/
private function validateFilesLanguages()
{
/**
* @var $filesSegments RecursiveArrayObject
*/
$filesSegments = $this->projectStructure['segments'];
/**
* This is a map <file_name, check_result>, where check_result is one
* of these status strings:<br/>
* - ok --> the language detected for this file is the same of source language<br/>
* - warning --> the language detected for this file is different from the source language
*
* @var $filename2SourceLangCheck array
*/
$filename2SourceLangCheck = array();
//istantiate MyMemory analyzer and detect languages for each file uploaded
$mma = Engine::getInstance(1);
$res = $mma->detectLanguage($filesSegments, $this->projectStructure['lang_detect_files']);
//for each language detected, check if it's not equal to the source language
$langsDetected = $res['responseData']['translatedText'];
Log::dolog(__CLASS__ . " - DETECT LANG RES:", $langsDetected);
if ($res !== null && is_array($langsDetected) && count($langsDetected) == count($this->projectStructure['array_files'])) {
$counter = 0;
foreach ($langsDetected as $fileLang) {
$currFileName = $this->projectStructure['array_files'][$counter];
//get language code
if (strpos($fileLang, "-") === false) {
//PHP Strict: Only variables should be passed by reference
$_tmp = explode("-", $this->projectStructure['source_language']);
$sourceLang = array_shift($_tmp);
} else {
$sourceLang = $this->projectStructure['source_language'];
}
Log::dolog(__CLASS__ . " - DETECT LANG COMPARISON:", "{$fileLang}@@{$sourceLang}");
//get extended language name using google language code
$languageExtendedName = Langs_GoogleLanguageMapper::getLanguageCode($fileLang);
//get extended language name using standard language code
$langClass = Langs_Languages::getInstance();
$sourceLanguageExtendedName = strtolower($langClass->getLocalizedName($sourceLang));
Log::dolog(__CLASS__ . " - DETECT LANG NAME COMPARISON:", "{$sourceLanguageExtendedName}@@{$languageExtendedName}");
//Check job's detected language. In case of undefined language, mark it as valid
if ($fileLang !== 'und' && $fileLang != $sourceLang && $sourceLanguageExtendedName != $languageExtendedName) {
$filename2SourceLangCheck[$currFileName] = 'warning';
$languageExtendedName = ucfirst($languageExtendedName);
$this->projectStructure['result']['errors'][] = array("code" => -17, "message" => "The source language you selected seems " . "to be different from the source language in \"{$currFileName}\". Please check.");
} else {
$filename2SourceLangCheck[$currFileName] = 'ok';
}
$counter++;
}
if (in_array("warning", array_values($filename2SourceLangCheck))) {
$this->projectStructure['result']['lang_detect'] = $filename2SourceLangCheck;
}
} else {
//There are errors while parsing JSON.
//Noop
}
}
/**
* Post a file to myMemory
*
* Remove the first line from csv ( source and target )
* and rewrite the csv because MyMemory doesn't want the header line
*
* @param $file
* @param $key
* @param bool|false $name
*
* @return Engines_Results_MyMemory_TmxResponse
*/
public function glossaryImport($file, $key, $name = false)
{
try {
$origFile = new SplFileObject($file, 'r+');
$origFile->setFlags(SplFileObject::READ_CSV | SplFileObject::SKIP_EMPTY | SplFileObject::READ_AHEAD);
$tmpFileName = tempnam("/tmp", 'GLOS');
$newFile = new SplFileObject($tmpFileName, 'r+');
$newFile->setFlags(SplFileObject::READ_CSV | SplFileObject::SKIP_EMPTY | SplFileObject::READ_AHEAD);
foreach ($origFile as $line_num => $line) {
if (count($line) < 2) {
throw new RuntimeException("No valid glossary file provided. Field separator could be not valid.");
}
if ($line_num == 0) {
list($source_lang, $target_lang, ) = $line;
//eventually, remove BOM from source language
$bom = pack('H*', 'EFBBBF');
$source_lang = preg_replace("/^{$bom}/", "", $source_lang);
if (!Langs_Languages::isEnabled($source_lang)) {
throw new RuntimeException("The source language specified in the glossary is not supported: " . $source_lang);
}
if (!Langs_Languages::isEnabled($target_lang)) {
throw new RuntimeException("The target language specified in the glossary is not supported: " . $target_lang);
}
if (empty($source_lang) || empty($target_lang)) {
throw new RuntimeException("No language definition found in glossary file.");
}
continue;
}
//copy stream to stream
$newFile->fputcsv($line);
}
$newFile->fflush();
$origFile = null;
//close the file handle
$newFile = null;
//close the file handle
copy($tmpFileName, $file);
unlink($tmpFileName);
} catch (RuntimeException $e) {
$this->result = new Engines_Results_MyMemory_TmxResponse(array("responseStatus" => 406, "responseData" => null, "responseDetails" => $e->getMessage()));
return $this->result;
}
$postFields = array('glossary' => "@" . realpath($file), 'source_lang' => $source_lang, 'target_lang' => $target_lang, 'name' => $name);
$postFields['key'] = trim($key);
$this->call("glossary_import_relative_url", $postFields, true);
return $this->result;
}
public static function getEditingLogData($jid, $password, $use_ter_diff = false)
{
$data = getEditLog($jid, $password);
$slow_cut = 30;
$fast_cut = 0.25;
$stat_too_slow = array();
$stat_too_fast = array();
if (!$data) {
return false;
}
$stats['total-word-count'] = 0;
$stat_mt = array();
foreach ($data as &$seg) {
$seg['sm'] .= "%";
$seg['jid'] = $jid;
$tte = self::parse_time_to_edit($seg['tte']);
$seg['time_to_edit'] = "{$tte['1']}m:{$tte['2']}s";
$stat_rwc[] = $seg['rwc'];
// by definition we cannot have a 0 word sentence. It is probably a - or a tag, so we want to consider at least a word.
if ($seg['rwc'] < 1) {
$seg['rwc'] = 1;
}
$seg['secs-per-word'] = round($seg['tte'] / 1000 / $seg['rwc'], 1);
if ($seg['secs-per-word'] < $slow_cut and $seg['secs-per-word'] > $fast_cut) {
$seg['stats-valid'] = 'Yes';
$seg['stats-valid-color'] = '';
$seg['stats-valid-style'] = '';
$stat_valid_rwc[] = $seg['rwc'];
$stat_valid_tte[] = $seg['tte'];
$stat_spw[] = $seg['secs-per-word'];
} else {
$seg['stats-valid'] = 'No';
$seg['stats-valid-color'] = '#ee6633';
$seg['stats-valid-style'] = 'border:2px solid #EE6633';
}
// Stats
if ($seg['secs-per-word'] >= $slow_cut) {
$stat_too_slow[] = $seg['rwc'];
}
if ($seg['secs-per-word'] <= $fast_cut) {
$stat_too_fast[] = $seg['rwc'];
}
$seg['pe_effort_perc'] = round((1 - MyMemory::TMS_MATCH($seg['sug'], $seg['translation'])) * 100);
if ($seg['pe_effort_perc'] < 0) {
$seg['pe_effort_perc'] = 0;
}
if ($seg['pe_effort_perc'] > 100) {
$seg['pe_effort_perc'] = 100;
}
$stat_pee[] = $seg['pe_effort_perc'] * $seg['rwc'];
$seg['pe_effort_perc'] .= "%";
$lh = Langs_Languages::getInstance();
$lang = $lh->getIsoCode($lh->getLocalizedName($seg['target_lang']));
$sug_for_diff = self::placehold_xliff_tags($seg['sug']);
$tra_for_diff = self::placehold_xliff_tags($seg['translation']);
// possible patch
// $sug_for_diff = html_entity_decode($sug_for_diff, ENT_NOQUOTES, 'UTF-8');
// $tra_for_diff = html_entity_decode($tra_for_diff, ENT_NOQUOTES, 'UTF-8');
//with this patch we have warnings when accessing indexes
if ($use_ter_diff) {
$ter = MyMemory::diff_tercpp($sug_for_diff, $tra_for_diff, $lang);
} else {
$ter = array();
}
// Log::doLog( $sug_for_diff );
// Log::doLog( $tra_for_diff );
// Log::doLog( $ter );
$seg['ter'] = @$ter[1] * 100;
$stat_ter[] = $seg['ter'] * $seg['rwc'];
$seg['ter'] = round(@$ter[1] * 100) . "%";
$diff_ter = @$ter[0];
if ($seg['sug'] != $seg['translation']) {
//force use of third party ter diff
if ($use_ter_diff) {
$seg['diff'] = $diff_ter;
} else {
$diff_PE = MyMemory::diff_html($sug_for_diff, $tra_for_diff);
// we will use diff_PE until ter_diff will not work properly
$seg['diff'] = $diff_PE;
}
//$seg[ 'diff_ter' ] = $diff_ter;
} else {
$seg['diff'] = '';
//$seg[ 'diff_ter' ] = '';
}
$seg['diff'] = self::restore_xliff_tags_for_view($seg['diff']);
//$seg['diff_ter'] = self::restore_xliff_tags_for_view($seg['diff_ter']);
// BUG: While suggestions source is not correctly set
if ($seg['sm'] == "85%" or $seg['sm'] == "86%") {
$seg['ss'] = 'Machine Translation';
$stat_mt[] = $seg['rwc'];
} else {
$seg['ss'] = 'Translation Memory';
}
$seg['sug_view'] = trim(CatUtils::rawxliff2view($seg['sug']));
$seg['source'] = trim(CatUtils::rawxliff2view($seg['source']));
$seg['translation'] = trim(CatUtils::rawxliff2view($seg['translation']));
$array_patterns = array(rtrim(self::lfPlaceholderRegex, 'g'), rtrim(self::crPlaceholderRegex, 'g'), rtrim(self::crlfPlaceholderRegex, 'g'), rtrim(self::tabPlaceholderRegex, 'g'), rtrim(self::nbspPlaceholderRegex, 'g'));
$array_replacements_csv = array('\\n', '\\r', '\\r\\n', '\\t', Utils::unicode2chr(0xa0));
$seg['source_csv'] = preg_replace($array_patterns, $array_replacements_csv, $seg['source']);
$seg['translation_csv'] = preg_replace($array_patterns, $array_replacements_csv, $seg['translation']);
$seg['sug_csv'] = preg_replace($array_patterns, $array_replacements_csv, $seg['sug_view']);
$seg['diff_csv'] = preg_replace($array_patterns, $array_replacements_csv, $seg['diff']);
$array_replacements = array('<span class="_0A"></span><br />', '<span class="_0D"></span><br />', '<span class="_0D0A"></span><br />', '<span class="_tab">	</span>', '<span class="_nbsp"> </span>');
$seg['source'] = preg_replace($array_patterns, $array_replacements, $seg['source']);
$seg['translation'] = preg_replace($array_patterns, $array_replacements, $seg['translation']);
$seg['sug_view'] = preg_replace($array_patterns, $array_replacements, $seg['sug_view']);
$seg['diff'] = preg_replace($array_patterns, $array_replacements, $seg['diff']);
if ($seg['mt_qe'] == 0) {
$seg['mt_qe'] = 'N/A';
}
}
$stats['edited-word-count'] = array_sum($stat_rwc);
$stats['valid-word-count'] = array_sum($stat_valid_rwc);
if ($stats['edited-word-count'] > 0) {
$stats['too-slow-words'] = round(array_sum($stat_too_slow) / $stats['edited-word-count'], 2) * 100;
$stats['too-fast-words'] = round(array_sum($stat_too_fast) / $stats['edited-word-count'], 2) * 100;
$stats['avg-pee'] = round(array_sum($stat_pee) / array_sum($stat_rwc)) . "%";
$stats['avg-ter'] = round(array_sum($stat_ter) / array_sum($stat_rwc)) . "%";
}
// echo array_sum($stat_ter);
// echo "@@@";
// echo array_sum($stat_rwc);
// exit;
$stats['mt-words'] = round(array_sum($stat_mt) / $stats['edited-word-count'], 2) * 100;
$stats['tm-words'] = 100 - $stats['mt-words'];
$stats['total-valid-tte'] = round(array_sum($stat_valid_tte) / 1000);
// Non weighted...
// $stats['avg-secs-per-word'] = round(array_sum($stat_spw)/count($stat_spw),1);
// Weighted
$stats['avg-secs-per-word'] = round($stats['total-valid-tte'] / $stats['valid-word-count'], 1);
$stats['est-words-per-day'] = number_format(round(3600 * 8 / $stats['avg-secs-per-word']), 0, '.', ',');
// Last minute formatting (after calculations)
$temp = self::parse_time_to_edit(round(array_sum($stat_valid_tte)));
$stats['total-valid-tte'] = "{$temp['0']}h:{$temp['1']}m:{$temp['2']}s";
$stats['total-tte-seconds'] = $temp[0] * 3600 + $temp[1] * 60 + $temp[2];
return array($data, $stats);
}
public function doAction()
{
//get Job Infos
$job_data = getJobData($this->jid);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (!$pCheck->grantJobAccessByJobData($job_data, $this->password)) {
$this->result['errors'][] = array("code" => -10, "message" => "wrong password");
return;
}
$lang_handler = Langs_Languages::getInstance();
if ($this->ref_segment == '') {
$this->ref_segment = 0;
}
$data = getMoreSegments($this->jid, $this->password, $this->step, $this->ref_segment, $this->where);
$this->prepareNotes($data);
foreach ($data as $i => $seg) {
if ($this->where == 'before') {
if ((double) $seg['sid'] >= (double) $this->ref_segment) {
break;
}
}
if (empty($this->pname)) {
$this->pname = $seg['pname'];
}
if (empty($this->last_opened_segment)) {
$this->last_opened_segment = $seg['last_opened_segment'];
}
if (empty($this->cid)) {
$this->cid = $seg['cid'];
}
if (empty($this->pid)) {
$this->pid = $seg['pid'];
}
if (empty($this->tid)) {
$this->tid = $seg['tid'];
}
if (empty($this->create_date)) {
$this->create_date = $seg['create_date'];
}
if (empty($this->source_code)) {
$this->source_code = $seg['source'];
}
if (empty($this->target_code)) {
$this->target_code = $seg['target'];
}
if (empty($this->source)) {
$s = explode("-", $seg['source']);
$source = strtoupper($s[0]);
$this->source = $source;
}
if (empty($this->target)) {
$t = explode("-", $seg['target']);
$target = strtoupper($t[0]);
$this->target = $target;
}
if (empty($this->err)) {
$this->err = $seg['serialized_errors_list'];
}
$id_file = $seg['id_file'];
if (!isset($this->data["{$id_file}"])) {
$this->data["{$id_file}"]['jid'] = $seg['jid'];
$this->data["{$id_file}"]["filename"] = ZipArchiveExtended::getFileName($seg['filename']);
$this->data["{$id_file}"]["mime_type"] = $seg['mime_type'];
$this->data["{$id_file}"]['source'] = $lang_handler->getLocalizedName($seg['source']);
$this->data["{$id_file}"]['target'] = $lang_handler->getLocalizedName($seg['target']);
$this->data["{$id_file}"]['source_code'] = $seg['source'];
$this->data["{$id_file}"]['target_code'] = $seg['target'];
$this->data["{$id_file}"]['segments'] = array();
}
unset($seg['id_file']);
unset($seg['source']);
unset($seg['target']);
unset($seg['source_code']);
unset($seg['target_code']);
unset($seg['mime_type']);
unset($seg['filename']);
unset($seg['jid']);
unset($seg['pid']);
unset($seg['cid']);
unset($seg['tid']);
unset($seg['pname']);
unset($seg['create_date']);
unset($seg['id_segment_end']);
unset($seg['id_segment_start']);
unset($seg['serialized_errors_list']);
$seg['parsed_time_to_edit'] = CatUtils::parse_time_to_edit($seg['time_to_edit']);
$seg['source_chunk_lengths'] === null ? $seg['source_chunk_lengths'] = '[]' : null;
$seg['target_chunk_lengths'] === null ? $seg['target_chunk_lengths'] = '{"len":[0],"statuses":["DRAFT"]}' : null;
$seg['source_chunk_lengths'] = json_decode($seg['source_chunk_lengths'], true);
$seg['target_chunk_lengths'] = json_decode($seg['target_chunk_lengths'], true);
$seg['segment'] = CatUtils::rawxliff2view(CatUtils::reApplySegmentSplit($seg['segment'], $seg['source_chunk_lengths']));
$seg['translation'] = CatUtils::rawxliff2view(CatUtils::reApplySegmentSplit($seg['translation'], $seg['target_chunk_lengths']['len']));
$this->attachNotes($seg);
$this->data["{$id_file}"]['segments'][] = $seg;
}
$this->result['data']['files'] = $this->data;
$this->result['data']['where'] = $this->where;
}
private function getEditLogData($use_ter_diff = false)
{
$editLogDao = new EditLog_EditLogDao(Database::obtain());
$data = $editLogDao->getSegments($this->getJid(), $this->getPassword(), self::$start_id);
//get translation mismatches and convert the array in a hashmap
$translationMismatchList = $editLogDao->getTranslationMismatches($this->getJid());
foreach ($translationMismatchList as $idx => $translMismRow) {
$translMismRow[$translMismRow['segment_hash']] = (bool) $translMismRow['translation_mismatch'];
}
$__pagination_prev = PHP_INT_MAX;
$__pagination_next = -2147483648;
//PHP_INT_MIN
$stat_too_slow = array();
$stat_too_fast = array();
if (!$data) {
throw new Exception('There are no changes in this job', -1);
}
$stats['total-word-count'] = 0;
$stat_mt = array();
$stat_valid_rwc = array();
$stat_rwc = array();
$stat_valid_tte = array();
$stat_pee = array();
$stat_ter = array();
$output_data = array();
foreach ($data as $seg) {
//if the segment is before the current one
if ($seg->id < self::$start_id) {
if ($seg->id <= $__pagination_prev) {
$__pagination_prev = $seg->id;
}
continue;
}
if ($seg->id > $__pagination_next) {
$__pagination_next = $seg->id;
}
$displaySeg = new EditLog_EditLogSegmentClientStruct($seg->toArray());
$displaySeg->suggestion_match .= "%";
$displaySeg->job_id = $this->jid;
$tte = CatUtils::parse_time_to_edit($displaySeg->time_to_edit);
$displaySeg->display_time_to_edit = "{$tte['1']}m:{$tte['2']}s";
$stat_rwc[] = $seg->raw_word_count;
// by definition we cannot have a 0 word sentence. It is probably a - or a tag, so we want to consider at least a word.
if ($seg->raw_word_count < 1) {
$displaySeg->raw_word_count = 1;
}
//todo: remove this
$displaySeg->secs_per_word = $seg->getSecsPerWord();
if ($displaySeg->secs_per_word < self::EDIT_TIME_SLOW_CUT && $displaySeg->secs_per_word > self::EDIT_TIME_FAST_CUT) {
$displaySeg->stats_valid = true;
$stat_valid_rwc[] = $seg->raw_word_count;
$stat_spw[] = $displaySeg->secs_per_word;
} else {
$displaySeg->stats_valid = false;
}
// Stats
if ($displaySeg->secs_per_word >= self::EDIT_TIME_SLOW_CUT) {
$stat_too_slow[] = $seg->raw_word_count;
}
if ($displaySeg->secs_per_word <= self::EDIT_TIME_FAST_CUT) {
$stat_too_fast[] = $seg->raw_word_count;
}
$displaySeg->secs_per_word .= "s";
$displaySeg->pe_effort_perc = $displaySeg->getPeePerc();
if ($displaySeg->pe_effort_perc < 0) {
$displaySeg->pe_effort_perc = 0;
}
if ($displaySeg->pe_effort_perc > 100) {
$displaySeg->pe_effort_perc = 100;
}
$stat_pee[] = $displaySeg->pe_effort_perc * $seg->raw_word_count;
$displaySeg->pe_effort_perc .= "%";
$lh = Langs_Languages::getInstance();
$lang = $lh->getIsoCode($lh->getLocalizedName($seg->job_target));
$sug_for_diff = CatUtils::placehold_xliff_tags($seg->suggestion);
$tra_for_diff = CatUtils::placehold_xliff_tags($seg->translation);
//with this patch we have warnings when accessing indexes
if ($use_ter_diff) {
$ter = MyMemory::diff_tercpp($sug_for_diff, $tra_for_diff, $lang);
} else {
$ter = array();
}
$displaySeg->ter = @$ter[1] * 100;
$stat_ter[] = $displaySeg->ter * $seg->raw_word_count;
$displaySeg->ter = round(@$ter[1] * 100) . "%";
$diff_ter = @$ter[0];
if ($seg->suggestion != $seg->translation) {
//force use of third party ter diff
if ($use_ter_diff) {
$displaySeg->diff = $diff_ter;
} else {
$diff_PE = MyMemory::diff_html($sug_for_diff, $tra_for_diff);
// we will use diff_PE until ter_diff will not work properly
$displaySeg->diff = $diff_PE;
}
//$seg[ 'diff_ter' ] = $diff_ter;
} else {
$displaySeg->diff = '';
}
$displaySeg->diff = CatUtils::restore_xliff_tags_for_view($displaySeg->diff);
// BUG: While suggestions source is not correctly set
if ($displaySeg->suggestion_match == "85%" || $displaySeg->suggestion_match == "86%") {
$displaySeg->suggestion_source = 'Machine Translation';
$stat_mt[] = $seg->raw_word_count;
} else {
$displaySeg->suggestion_source = 'TM';
}
$array_patterns = array(rtrim(CatUtils::lfPlaceholderRegex, 'g'), rtrim(CatUtils::crPlaceholderRegex, 'g'), rtrim(CatUtils::crlfPlaceholderRegex, 'g'), rtrim(CatUtils::tabPlaceholderRegex, 'g'), rtrim(CatUtils::nbspPlaceholderRegex, 'g'));
$array_replacements_csv = array('\\n', '\\r', '\\r\\n', '\\t', Utils::unicode2chr(0xa0));
$displaySeg->source_csv = preg_replace($array_patterns, $array_replacements_csv, $seg->source);
$displaySeg->translation_csv = preg_replace($array_patterns, $array_replacements_csv, $seg->translation);
$displaySeg->sug_csv = preg_replace($array_patterns, $array_replacements_csv, $displaySeg->suggestion_view);
$displaySeg->diff_csv = preg_replace($array_patterns, $array_replacements_csv, $displaySeg->diff);
$array_replacements = array('<span class="_0A"></span><br />', '<span class="_0D"></span><br />', '<span class="_0D0A"></span><br />', '<span class="_tab">	</span>', '<span class="_nbsp"> </span>');
$displaySeg->source = preg_replace($array_patterns, $array_replacements, $seg->source);
$displaySeg->translation = preg_replace($array_patterns, $array_replacements, $seg->translation);
$displaySeg->suggestion_view = preg_replace($array_patterns, $array_replacements, $displaySeg->suggestion_view);
$displaySeg->diff = preg_replace($array_patterns, $array_replacements, $displaySeg->diff);
$displaySeg->source = trim(CatUtils::rawxliff2view($seg->source));
$displaySeg->suggestion_view = trim(CatUtils::rawxliff2view($seg->suggestion));
$displaySeg->translation = trim(CatUtils::rawxliff2view($seg->translation));
if ($seg->mt_qe == 0) {
$displaySeg->mt_qe = 'N/A';
}
$displaySeg->num_translation_mismatch = @(int) $translationMismatchList[$displaySeg->segment_hash];
$displaySeg->evaluateWarningString();
$output_data[] = $displaySeg;
}
$pagination = $this->evaluatePagination($__pagination_prev, $__pagination_next + 1);
$globalStats = $this->evaluateGlobalStats();
$stats['valid-word-count'] = $globalStats['raw_words'];
//TODO: this will not work anymore
$stats['edited-word-count'] = array_sum($stat_rwc);
if ($stats['edited-word-count'] > 0) {
$stats['too-slow-words'] = round(array_sum($stat_too_slow) / $stats['edited-word-count'], 2) * 100;
$stats['too-fast-words'] = round(array_sum($stat_too_fast) / $stats['edited-word-count'], 2) * 100;
$stats['avg-pee'] = round(array_sum($stat_pee) / array_sum($stat_rwc)) . "%";
$stats['avg-ter'] = round(array_sum($stat_ter) / array_sum($stat_rwc)) . "%";
}
$stats['mt-words'] = round(array_sum($stat_mt) / $stats['edited-word-count'], 2) * 100;
$stats['tm-words'] = 100 - $stats['mt-words'];
$stats['total-valid-tte'] = round($globalStats['tot_tte']);
// Non weighted...
// $stats['avg-secs-per-word'] = round(array_sum($stat_spw)/count($stat_spw),1);
// Weighted
$stats['avg-secs-per-word'] = round($globalStats['secs_per_word'] / 1000, 1);
$stats['est-words-per-day'] = number_format(round(3600 * 8 / $stats['avg-secs-per-word']), 0, '.', ',');
// Last minute formatting (after calculations)
$temp = CatUtils::parse_time_to_edit(round($stats['total-valid-tte']));
$stats['total-valid-tte'] = "{$temp['0']}h:{$temp['1']}m:{$temp['2']}s";
$stats['total-tte-seconds'] = $temp[0] * 3600 + $temp[1] * 60 + $temp[2];
$stats['avg-pee'] = round($globalStats['avg_pee'], 2);
$stats['avg-pee'] .= "%";
return array($output_data, $stats, $pagination);
}
public function doAction()
{
//get job language and data
//Fixed Bug: need a specific job, because we need The target Language
//Removed from within the foreach cycle, the job is always the same....
$jobData = $this->jobInfo = getJobData($this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($jobData) || !$pCheck->grantJobAccessByJobData($jobData, $this->password)) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
//get storage object
$fs = new FilesStorage();
$files_job = $fs->getFilesForJob($this->id_job, $this->id_file);
$nonew = 0;
$output_content = array();
/*
the procedure:
1)original xliff file is read directly from disk; a file handler is obtained
2)the file is read chunk by chunk by a stream parser: for each trans-unit that is encountered, target is replaced (or added) with the corresponding translation obtained from the DB
3)the parsed portion of xliff in the buffer is flushed on temporary file
4)the temporary file is sent to the converter and an original file is obtained
5)the temporary file is deleted
*/
// This array will contain all the files of $files_job split by
// converter version.
$files_job_by_converter_version = array();
// Detect the converter's version to use for each file, then store
// file info accordingly.
foreach ($files_job as $file) {
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
$files_job_by_converter_version[$fileType['converter_version']][] = $file;
}
// Process files according to the converters' versions, one version
// at a time
foreach ($files_job_by_converter_version as $converter_version => $files_job) {
//file array is chuncked. Each chunk will be used for a parallel conversion request.
$files_job = array_chunk($files_job, self::FILES_CHUNK_SIZE);
foreach ($files_job as $chunk) {
$converter = new FileFormatConverter($converter_version);
$files_to_be_converted = array();
foreach ($chunk as $file) {
$mime_type = $file['mime_type'];
$fileID = $file['id_file'];
$current_filename = $file['filename'];
//get path for the output file converted to know it's right extension
$_fileName = explode(DIRECTORY_SEPARATOR, $file['xliffFilePath']);
$outputPath = INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/' . $fileID . '/' . uniqid('', true) . "_.out." . array_pop($_fileName);
//make dir if doesn't exist
if (!file_exists(dirname($outputPath))) {
Log::doLog('Create Directory ' . escapeshellarg(dirname($outputPath)) . '');
mkdir(dirname($outputPath), 0775, true);
}
$data = getSegmentsDownload($this->id_job, $this->password, $fileID, $nonew);
//prepare regexp for nest step
$regexpEntity = '/&#x(0[0-8BCEF]|1[0-9A-F]|7F);/u';
$regexpAscii = '/([\\x{00}-\\x{1F}\\x{7F}]{1})/u';
foreach ($data as $i => $k) {
//create a secondary indexing mechanism on segments' array; this will be useful
//prepend a string so non-trans unit id ( ex: numerical ) are not overwritten
$data['matecat|' . $k['internal_id']][] = $i;
//FIXME: temporary patch
$data[$i]['translation'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['translation']);
$data[$i]['segment'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['segment']);
//remove binary chars in some xliff files
$sanitized_src = preg_replace($regexpAscii, '', $data[$i]['segment']);
$sanitized_trg = preg_replace($regexpAscii, '', $data[$i]['translation']);
//clean invalid xml entities ( charactes with ascii < 32 and different from 0A, 0D and 09
$sanitized_src = preg_replace($regexpEntity, '', $sanitized_src);
$sanitized_trg = preg_replace($regexpEntity, '', $sanitized_trg);
if ($sanitized_src != null) {
$data[$i]['segment'] = $sanitized_src;
}
if ($sanitized_trg != null) {
$data[$i]['translation'] = $sanitized_trg;
}
}
//instatiate parser
$xsp = new SdlXliffSAXTranslationReplacer($file['xliffFilePath'], $data, Langs_Languages::getInstance()->getLangRegionCode($jobData['target']), $outputPath);
if ($this->download_type == 'omegat') {
$xsp->setSourceInTarget(true);
}
//run parsing
Log::doLog("work on " . $fileID . " " . $current_filename);
$xsp->replaceTranslation();
//free memory
unset($xsp);
unset($data);
$output_content[$fileID]['document_content'] = file_get_contents($outputPath);
$output_content[$fileID]['output_filename'] = $current_filename;
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
if ($this->forceXliff) {
//clean the output filename by removing
// the unique hash identifier 55e5739b467109.05614837_.out.Test_English.doc.sdlxliff
$output_content[$fileID]['output_filename'] = preg_replace('#[0-9a-f]+\\.[0-9_]+\\.out\\.#i', '', FilesStorage::basename_fix($outputPath));
if ($fileType['proprietary_short_name'] === 'matecat_converter') {
// Set the XLIFF extension to .xlf
// Internally, MateCat continues using .sdlxliff as default
// extension for the XLIFF behind the projects.
// Changing this behavior requires a huge refactoring that
// it's scheduled for future versions.
// We quickly fixed the behaviour from the user standpoint
// using the following line of code, that changes the XLIFF's
// extension just a moment before it is downloaded by the user.
$output_content[$fileID]['output_filename'] = preg_replace("|\\.sdlxliff\$|i", ".xlf", $output_content[$fileID]['output_filename']);
}
}
/**
* Conversion Enforce
*/
$convertBackToOriginal = true;
//if it is a not converted file ( sdlxliff ) we have originalFile equals to xliffFile (it has just been copied)
$file['original_file'] = file_get_contents($file['originalFilePath']);
// When the 'proprietary' flag is set to false, the xliff
// is not passed to any converter, because is handled
// directly inside MateCAT.
$xliffWasNotConverted = $fileType['proprietary'] === false;
if (!INIT::$CONVERSION_ENABLED || ($file['originalFilePath'] == $file['xliffFilePath'] and $xliffWasNotConverted) or $this->forceXliff) {
$convertBackToOriginal = false;
Log::doLog("SDLXLIFF: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
} else {
//TODO: dos2unix ??? why??
//force unix type files
Log::doLog("NO SDLXLIFF, Conversion enforced: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
}
if ($convertBackToOriginal) {
$output_content[$fileID]['out_xliff_name'] = $outputPath;
$output_content[$fileID]['source'] = $jobData['source'];
$output_content[$fileID]['target'] = $jobData['target'];
$files_to_be_converted[$fileID] = $output_content[$fileID];
} elseif ($this->forceXliff) {
$this->cleanFilePath($output_content[$fileID]['document_content']);
}
}
$convertResult = $converter->multiConvertToOriginal($files_to_be_converted, $chosen_machine = false);
foreach (array_keys($files_to_be_converted) as $fileID) {
$output_content[$fileID]['document_content'] = $this->ifGlobalSightXliffRemoveTargetMarks($convertResult[$fileID]['document_content'], $files_to_be_converted[$fileID]['output_filename']);
//in case of .strings, they are required to be in UTF-16
//get extension to perform file detection
$extension = FilesStorage::pathinfo_fix($output_content[$fileID]['output_filename'], PATHINFO_EXTENSION);
if (strtoupper($extension) == 'STRINGS') {
//use this function to convert stuff
$encodingConvertedFile = CatUtils::convertEncoding('UTF-16', $output_content[$fileID]['document_content']);
//strip previously added BOM
$encodingConvertedFile[1] = $converter->stripBOM($encodingConvertedFile[1], 16);
//store new content
$output_content[$fileID]['document_content'] = $encodingConvertedFile[1];
//trash temporary data
unset($encodingConvertedFile);
}
}
unset($convertResult);
}
}
foreach ($output_content as $idFile => $fileInformations) {
$zipPathInfo = ZipArchiveExtended::zipPathInfo($output_content[$idFile]['output_filename']);
if (is_array($zipPathInfo)) {
$output_content[$idFile]['zipfilename'] = $zipPathInfo['zipfilename'];
$output_content[$idFile]['zipinternalPath'] = $zipPathInfo['dirname'];
$output_content[$idFile]['output_filename'] = $zipPathInfo['basename'];
}
}
//set the file Name
$pathinfo = FilesStorage::pathinfo_fix($this->fname);
$this->_filename = $pathinfo['filename'] . "_" . $jobData['target'] . "." . $pathinfo['extension'];
//qui prodest to check download type?
if ($this->download_type == 'omegat') {
$this->_filename .= ".zip";
$tmsService = new TMSService();
$tmsService->setOutputType('tm');
/**
* @var $tmFile SplTempFileObject
*/
$tmFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tmsService->setOutputType('mt');
/**
* @var $mtFile SplTempFileObject
*/
$mtFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tm_id = uniqid('tm');
$mt_id = uniqid('mt');
$output_content[$tm_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_TM . tmx");
foreach ($tmFile as $lineNumber => $content) {
$output_content[$tm_id]['document_content'] .= $content;
}
$output_content[$mt_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_MT . tmx");
foreach ($mtFile as $lineNumber => $content) {
$output_content[$mt_id]['document_content'] .= $content;
}
$this->createOmegaTZip($output_content, $jobData['source'], $jobData['target']);
//add zip archive content here;
} else {
try {
$output_content = $this->getOutputContentsWithZipFiles($output_content);
if (count($output_content) > 1) {
//cast $output_content elements to ZipContentObject
foreach ($output_content as $key => $__output_content_elem) {
$output_content[$key] = new ZipContentObject($__output_content_elem);
}
if ($pathinfo['extension'] != 'zip') {
if ($this->forceXliff) {
$this->_filename = $this->id_job . ".zip";
} else {
$this->_filename = $pathinfo['basename'] . ".zip";
}
}
$this->content = self::composeZip($output_content);
//add zip archive content here;
} else {
//always an array with 1 element, pop it, Ex: array( array() )
$output_content = array_pop($output_content);
$this->setContent($output_content);
}
} catch (Exception $e) {
$msg = "\n\n Error retrieving file content, Conversion failed??? \n\n Error: {$e->getMessage()} \n\n" . var_export($e->getTraceAsString(), true);
$msg .= "\n\n Request: " . var_export($_REQUEST, true);
Log::$fileName = 'fatal_errors.txt';
Log::doLog($msg);
Utils::sendErrMailReport($msg);
$this->unlockToken(array("code" => -110, "message" => "Download failed. Please contact " . INIT::$SUPPORT_MAIL));
throw $e;
// avoid sent Headers and empty file content with finalize method
}
}
try {
Utils::deleteDir(INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/');
} catch (Exception $e) {
Log::doLog('Failed to delete dir:' . $e->getMessage());
}
}
/**
* @param $start int
* @param $step int
* @param $search_in_pname string|null
* @param $search_source string|null
* @param $search_target string|null
* @param $search_status string|null
* @param $search_onlycompleted bool
* @param $filter_enabled bool
* @param $project_id int
*
* @return array
* @throws Exception
*/
public static function queryProjects($start, $step, $search_in_pname, $search_source, $search_target, $search_status, $search_onlycompleted, $filter_enabled, $project_id)
{
$data = getProjects($start, $step, $search_in_pname, $search_source, $search_target, $search_status, $search_onlycompleted, $filter_enabled, $project_id);
$projects = array();
$projectIDs = array();
$project2info = array();
//get project IDs from projects array
foreach ($data as $item) {
$projectIDs[] = $item['pid'];
}
if (empty($projectIDs)) {
return array();
}
//get job data using job IDs
$jobData = getJobsFromProjects($projectIDs, $search_source, $search_target, $search_status, $search_onlycompleted);
$lang_handler = Langs_Languages::getInstance();
//Prepare job data
$project2jobChunk = array();
foreach ($jobData as $job_array) {
$job = array();
/**
* Assign job extracted variables
*/
$job['id'] = $job_array['id'];
$job['pid'] = $job_array['id_project'];
$job['password'] = $job_array['password'];
$job['source'] = $job_array['source'];
$job['target'] = $job_array['target'];
$job['subject'] = $job_array['subject'];
$job['sourceTxt'] = $lang_handler->getLocalizedName($job['source']);
$job['targetTxt'] = $lang_handler->getLocalizedName($job['target']);
$job['create_date'] = $job_array['create_date'];
$job['formatted_create_date'] = self::formatJobDate($job_array['create_date']);
$job['job_first_segment'] = $job_array['job_first_segment'];
$job['job_last_segment'] = $job_array['job_last_segment'];
$job['mt_engine_name'] = $job_array['name'];
$job['id_tms'] = $job_array['id_tms'];
//generate and set job stats
$jobStats = new WordCount_Struct();
$jobStats->setDraftWords($job_array['DRAFT']);
$jobStats->setRejectedWords($job_array['REJECT']);
$jobStats->setTranslatedWords($job_array['TRANSLATED']);
$jobStats->setApprovedWords($job_array['APPROVED']);
//These would be redundant in response. Unset them.
unset($job_array['DRAFT']);
unset($job_array['REJECT']);
unset($job_array['TRANSLATED']);
unset($job_array['APPROVED']);
$job['stats'] = CatUtils::getFastStatsForJob($jobStats);
//generate and set job tm_keys
$tm_keys_json = $job_array['tm_keys'];
$tm_keys_json = TmKeyManagement_TmKeyManagement::getOwnerKeys(array($tm_keys_json));
$tm_keys = array();
foreach ($tm_keys_json as $tm_key_struct) {
/**
* @var $tm_key_struct TmKeyManagement_TmKeyStruct
*/
$tm_keys[] = array("key" => $tm_key_struct->key, "r" => $tm_key_struct->r ? 'Lookup' : ' ', "w" => $tm_key_struct->w ? 'Update' : '');
}
$job['private_tm_key'] = json_encode($tm_keys);
$job['disabled'] = $job_array['status_owner'] == Constants_JobStatus::STATUS_CANCELLED ? "disabled" : "";
$job['status'] = $job_array['status_owner'];
//These vars will be used in projects loop for some flag evaluation.
$project2info[$job['pid']]['status'][] = $job['status'];
$project2info[$job['pid']]['mt_engine_name'] = $job['mt_engine_name'];
$project2info[$job['pid']]['id_tms'] = $job['id_tms'];
$project2jobChunk[$job['pid']][$job['id']][$job['job_first_segment']] = $job;
}
//Prepare project data
foreach ($data as $item) {
$project = array();
$project['id'] = $item['pid'];
$project['name'] = $item['name'];
$project['jobs'] = array();
$project['no_active_jobs'] = true;
$project['has_cancelled'] = 0;
$project['has_archived'] = 0;
$project['password'] = $item['password'];
$project['tm_analysis'] = number_format($item['tm_analysis_wc'], 0, ".", ",");
$project['jobs'] = $project2jobChunk[$project['id']];
$project['no_active_jobs'] = $project['no_active_jobs'] ? ' allCancelled' : '';
$project2info[$project['id']]['status'] = array_unique($project2info[$project['id']]['status']);
$project['no_active_jobs'] = !in_array(Constants_JobStatus::STATUS_ACTIVE, $project2info[$project['id']]['status']) ? ' allCancelled' : '';
$project['has_cancelled'] = in_array(Constants_JobStatus::STATUS_CANCELLED, $project2info[$project['id']]['status']);
$project['has_archived'] = in_array(Constants_JobStatus::STATUS_ARCHIVED, $project2info[$project['id']]['status']);
$project['mt_engine_name'] = $project2info[$project['id']]['mt_engine_name'];
$project['id_tms'] = $project2info[$project['id']]['id_tms'];
$projects[] = $project;
}
return $projects;
}
public function doAction()
{
$project_by_jobs_data = getProjectData($this->pid, $this->ppassword, $this->jid, $this->jpassword);
$lang_handler = Langs_Languages::getInstance();
if (empty($project_by_jobs_data)) {
$this->project_not_found = true;
}
//pick the project subject from the first job
if (count($project_by_jobs_data) > 0) {
$this->subject = $project_by_jobs_data[0]['subject'];
}
foreach ($project_by_jobs_data as &$p_jdata) {
$p_jdata['filename'] = ZipArchiveExtended::getFileName($p_jdata['filename']);
//json_decode payable rates
$p_jdata['payable_rates'] = json_decode($p_jdata['payable_rates'], true);
$this->num_segments += $p_jdata['total_segments'];
if (empty($this->pname)) {
$this->pname = $p_jdata['name'];
}
if (empty($this->project_status)) {
$this->project_status = $p_jdata['status_analysis'];
if ($this->standard_analysis_wc == 0) {
$this->standard_analysis_wc = $p_jdata['standard_analysis_wc'];
}
}
//equivalent word count global
if ($this->tm_analysis_wc == 0) {
$this->tm_analysis_wc = $p_jdata['tm_analysis_wc'];
}
if ($this->tm_analysis_wc == 0) {
$this->tm_analysis_wc = $p_jdata['fast_analysis_wc'];
}
$this->tm_analysis_wc_print = number_format($this->tm_analysis_wc, 0, ".", ",");
if ($this->fast_analysis_wc == 0) {
$this->fast_analysis_wc = $p_jdata['fast_analysis_wc'];
$this->fast_analysis_wc_print = number_format($this->fast_analysis_wc, 0, ".", ",");
}
// if zero then print empty instead of 0
if ($this->standard_analysis_wc == 0) {
$this->standard_analysis_wc_print = "";
}
if ($this->fast_analysis_wc == 0) {
$this->fast_analysis_wc_print = "";
}
if ($this->tm_analysis_wc == 0) {
$this->tm_analysis_wc_print = "";
}
$this->total_raw_word_count += $p_jdata['file_raw_word_count'];
$source = $lang_handler->getLocalizedName($p_jdata['source']);
$target = $lang_handler->getLocalizedName($p_jdata['target']);
if (!isset($this->jobs[$p_jdata['jid']])) {
if (!isset($this->jobs[$p_jdata['jid']]['splitted'])) {
$this->jobs[$p_jdata['jid']]['splitted'] = '';
}
$this->jobs[$p_jdata['jid']]['jid'] = $p_jdata['jid'];
$this->jobs[$p_jdata['jid']]['source'] = $source;
$this->jobs[$p_jdata['jid']]['target'] = $target;
}
$source_short = $p_jdata['source'];
$target_short = $p_jdata['target'];
$password = $p_jdata['jpassword'];
unset($p_jdata['name']);
unset($p_jdata['source']);
unset($p_jdata['target']);
unset($p_jdata['jpassword']);
unset($p_jdata['fast_analysis_wc']);
unset($p_jdata['tm_analysis_wc']);
unset($p_jdata['standard_analysis_wc']);
if (!isset($this->jobs[$p_jdata['jid']]['chunks'][$password])) {
$this->jobs[$p_jdata['jid']]['chunks'][$password] = array();
$this->jobs[$p_jdata['jid']]['chunks'][$password]['jid'] = $p_jdata['jid'];
$this->jobs[$p_jdata['jid']]['chunks'][$password]['source'] = $source;
$this->jobs[$p_jdata['jid']]['chunks'][$password]['target'] = $target;
$this->jobs[$p_jdata['jid']]['chunks'][$password]['jpassword'] = $password;
$this->jobs[$p_jdata['jid']]['chunks'][$password]['source_short'] = $source_short;
$this->jobs[$p_jdata['jid']]['chunks'][$password]['target_short'] = $target_short;
$this->jobs[$p_jdata['jid']]['rates'] = $p_jdata['payable_rates'];
if (!array_key_exists("total_raw_word_count", $this->jobs[$p_jdata['jid']])) {
$this->jobs[$p_jdata['jid']]['chunks'][$password]['total_raw_word_count'] = 0;
}
if (!array_key_exists("total_eq_word_count", $this->jobs[$p_jdata['jid']])) {
$this->jobs[$p_jdata['jid']]['chunks'][$password]['total_eq_word_count'] = 0;
}
}
//calculate total word counts per job (summing different files)
$this->jobs[$p_jdata['jid']]['chunks'][$password]['total_raw_word_count'] += $p_jdata['file_raw_word_count'];
//format the total (yeah, it's ugly doing it every cycle)
$this->jobs[$p_jdata['jid']]['chunks'][$password]['total_raw_word_count_print'] = number_format($this->jobs[$p_jdata['jid']]['chunks'][$password]['total_raw_word_count'], 0, ".", ",");
$this->jobs[$p_jdata['jid']]['chunks'][$password]['total_eq_word_count'] += $p_jdata['file_eq_word_count'];
$this->jobs[$p_jdata['jid']]['chunks'][$password]['total_eq_word_count_print'] = number_format($this->jobs[$p_jdata['jid']]['chunks'][$password]['total_eq_word_count'], 0, ".", ",");
$p_jdata['file_eq_word_count'] = number_format($p_jdata['file_eq_word_count'], 0, ".", ",");
$p_jdata['file_raw_word_count'] = number_format($p_jdata['file_raw_word_count'], 0, ".", ",");
$this->jobs[$p_jdata['jid']]['chunks'][$password]['files'][$p_jdata['id_file']] = $p_jdata;
$this->jobs[$p_jdata['jid']]['splitted'] = count($this->jobs[$p_jdata['jid']]['chunks']) > 1 ? 'splitted' : '';
}
$raw_wc_time = $this->total_raw_word_count / INIT::$ANALYSIS_WORDS_PER_DAYS;
$tm_wc_time = $this->tm_analysis_wc / INIT::$ANALYSIS_WORDS_PER_DAYS;
$fast_wc_time = $this->fast_analysis_wc / INIT::$ANALYSIS_WORDS_PER_DAYS;
$raw_wc_unit = 'day';
$tm_wc_unit = 'day';
$fast_wc_unit = 'day';
if ($raw_wc_time > 0 and $raw_wc_time < 1) {
$raw_wc_time *= 8;
//convert to hours (1 work day = 8 hours)
$raw_wc_unit = 'hour';
}
if ($raw_wc_time > 0 and $raw_wc_time < 1) {
$raw_wc_time *= 60;
//convert to minutes
$raw_wc_unit = 'minute';
}
if ($raw_wc_time > 1) {
$raw_wc_unit .= 's';
}
if ($tm_wc_time > 0 and $tm_wc_time < 1) {
$tm_wc_time *= 8;
//convert to hours (1 work day = 8 hours)
$tm_wc_unit = 'hour';
}
if ($tm_wc_time > 0 and $tm_wc_time < 1) {
$tm_wc_time *= 60;
//convert to minutes
$tm_wc_unit = 'minute';
}
if ($tm_wc_time > 1) {
$tm_wc_unit .= 's';
}
if ($fast_wc_time > 0 and $fast_wc_time < 1) {
$fast_wc_time *= 8;
//convert to hours (1 work day = 8 hours)
$fast_wc_unit = 'hour';
}
if ($fast_wc_time > 0 and $fast_wc_time < 1) {
$fast_wc_time *= 60;
//convert to minutes
$fast_wc_unit = 'minute';
}
if ($fast_wc_time > 1) {
$fast_wc_unit .= 's';
}
$this->raw_wc_time = ceil($raw_wc_time);
$this->fast_wc_time = ceil($fast_wc_time);
$this->tm_wc_time = ceil($tm_wc_time);
$this->raw_wc_unit = $raw_wc_unit;
$this->tm_wc_unit = $tm_wc_unit;
$this->fast_wc_unit = $fast_wc_unit;
if ($this->raw_wc_time == 8 and $this->raw_wc_unit == "hours") {
$this->raw_wc_time = 1;
$this->raw_wc_unit = "day";
}
if ($this->raw_wc_time == 60 and $this->raw_wc_unit == "minutes") {
$this->raw_wc_time = 1;
$this->raw_wc_unit = "hour";
}
if ($this->fast_wc_time == 8 and $this->fast_wc_time == "hours") {
$this->fast_wc_time = 1;
$this->fast_wc_time = "day";
}
if ($this->tm_wc_time == 60 and $this->tm_wc_time == "minutes") {
$this->tm_wc_time = 1;
$this->tm_wc_time = "hour";
}
if ($this->total_raw_word_count == 0) {
$this->total_raw_word_count_print = "";
} else {
$this->total_raw_word_count_print = number_format($this->total_raw_word_count, 0, ".", ",");
}
// echo "<pre>" . print_r ( $this->jobs, true ) . "</pre>"; exit;
}
public function doAction()
{
$files_found = array();
$lang_handler = Langs_Languages::getInstance();
$data = getSegmentsInfo($this->jid, $this->password);
if (empty($data) or $data < 0) {
$this->job_not_found = true;
//stop execution
return;
}
//retrieve job owner. It will be useful also if the job is archived or cancelled
$this->job_owner = $data[0]['job_owner'] != "" ? $data[0]['job_owner'] : "*****@*****.**";
if ($data[0]['status'] == Constants_JobStatus::STATUS_CANCELLED) {
$this->job_cancelled = true;
//stop execution
return;
}
if ($data[0]['status'] == Constants_JobStatus::STATUS_ARCHIVED) {
$this->job_archived = true;
//stop execution
return;
}
/*
* I prefer to use a programmatic approach to the check for the archive date instead of a pure query
* because the query to check "Utils::getArchivableJobs($this->jid)" should be
* executed every time a job is loaded ( F5 or CTRL+R on browser ) and it cost some milliseconds ( ~0.1s )
* and it is little heavy for the database.
* We use the data we already have from last query and perform
* the check on the last translation only if the job is older than 30 days
*
*/
$lastUpdate = new DateTime($data[0]['last_update']);
$oneMonthAgo = new DateTime();
$oneMonthAgo->modify('-' . INIT::JOB_ARCHIVABILITY_THRESHOLD . ' days');
if ($lastUpdate < $oneMonthAgo && !$this->job_cancelled) {
$lastTranslationInJob = new Datetime(getLastTranslationDate($this->jid));
if ($lastTranslationInJob < $oneMonthAgo) {
$res = "job";
$new_status = Constants_JobStatus::STATUS_ARCHIVED;
updateJobsStatus($res, $this->jid, $new_status, null, null, $this->password);
$this->job_archived = true;
}
}
foreach ($data as $i => $job) {
$this->project_status = $job;
// get one row values for the project are the same for every row
if (empty($this->pname)) {
$this->pname = $job['pname'];
$this->downloadFileName = $job['pname'] . ".zip";
// will be overwritten below in case of one file job
}
if (empty($this->last_opened_segment)) {
$this->last_opened_segment = $job['last_opened_segment'];
}
if (empty($this->cid)) {
$this->cid = $job['cid'];
}
if (empty($this->pid)) {
$this->pid = $job['pid'];
}
if (empty($this->create_date)) {
$this->create_date = $job['create_date'];
}
if (empty($this->source_code)) {
$this->source_code = $job['source'];
}
if (empty($this->target_code)) {
$this->target_code = $job['target'];
}
if (empty($this->source)) {
$s = explode("-", $job['source']);
$source = strtoupper($s[0]);
$this->source = $source;
$this->source_rtl = $lang_handler->isRTL(strtolower($this->source)) ? ' rtl-source' : '';
}
if (empty($this->target)) {
$t = explode("-", $job['target']);
$target = strtoupper($t[0]);
$this->target = $target;
$this->target_rtl = $lang_handler->isRTL(strtolower($this->target)) ? ' rtl-target' : '';
}
//check if language belongs to supported right-to-left languages
if ($job['status'] == Constants_JobStatus::STATUS_ARCHIVED) {
$this->job_archived = true;
$this->job_owner = $data[0]['job_owner'];
}
$id_file = $job['id_file'];
if (!isset($this->data["{$id_file}"])) {
$files_found[] = $job['filename'];
}
$wStruct = new WordCount_Struct();
$wStruct->setIdJob($this->jid);
$wStruct->setJobPassword($this->password);
$wStruct->setNewWords($job['new_words']);
$wStruct->setDraftWords($job['draft_words']);
$wStruct->setTranslatedWords($job['translated_words']);
$wStruct->setApprovedWords($job['approved_words']);
$wStruct->setRejectedWords($job['rejected_words']);
unset($job['id_file']);
unset($job['source']);
unset($job['target']);
unset($job['source_code']);
unset($job['target_code']);
unset($job['mime_type']);
unset($job['filename']);
unset($job['jid']);
unset($job['pid']);
unset($job['cid']);
unset($job['tid']);
unset($job['pname']);
unset($job['create_date']);
unset($job['owner']);
unset($job['last_opened_segment']);
unset($job['new_words']);
unset($job['draft_words']);
unset($job['translated_words']);
unset($job['approved_words']);
unset($job['rejected_words']);
//For projects created with No tm analysis enabled
if ($wStruct->getTotal() == 0 && ($job['status_analysis'] == Constants_ProjectStatus::STATUS_DONE || $job['status_analysis'] == Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE)) {
$wCounter = new WordCount_Counter();
$wStruct = $wCounter->initializeJobWordCount($this->jid, $this->password);
Log::doLog("BackWard compatibility set Counter.");
}
$this->job_stats = CatUtils::getFastStatsForJob($wStruct);
}
//Needed because a just created job has last_opened segment NULL
if (empty($this->last_opened_segment)) {
$this->last_opened_segment = getFirstSegmentId($this->jid, $this->password);
}
$this->first_job_segment = $this->project_status['job_first_segment'];
$this->last_job_segment = $this->project_status['job_last_segment'];
if (count($files_found) == 1) {
$this->downloadFileName = $files_found[0];
}
/**
* get first segment of every file
*/
$fileInfo = getFirstSegmentOfFilesInJob($this->jid);
$TotalPayable = array();
foreach ($fileInfo as &$file) {
$file['file_name'] = ZipArchiveExtended::getFileName($file['file_name']);
$TotalPayable[$file['id_file']]['TOTAL_FORMATTED'] = $file['TOTAL_FORMATTED'];
}
$this->firstSegmentOfFiles = json_encode($fileInfo);
$this->fileCounter = json_encode($TotalPayable);
list($uid, $user_email) = $this->getLoginUserParams();
if (self::isRevision()) {
$this->userRole = TmKeyManagement_Filter::ROLE_REVISOR;
} elseif ($user_email == $data[0]['job_owner']) {
$this->userRole = TmKeyManagement_Filter::OWNER;
} else {
$this->userRole = TmKeyManagement_Filter::ROLE_TRANSLATOR;
}
/*
* Take the keys of the user
*/
try {
$_keyList = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => $uid));
$keyList = $_keyList->read($dh);
} catch (Exception $e) {
$keyList = array();
Log::doLog($e->getMessage());
}
$reverse_lookup_user_personal_keys = array('pos' => array(), 'elements' => array());
/**
* Set these keys as editable for the client
*
* @var $keyList TmKeyManagement_MemoryKeyStruct[]
*/
foreach ($keyList as $_j => $key) {
/**
* @var $_client_tm_key TmKeyManagement_TmKeyStruct
*/
//create a reverse lookup
$reverse_lookup_user_personal_keys['pos'][$_j] = $key->tm_key->key;
$reverse_lookup_user_personal_keys['elements'][$_j] = $key;
$this->_keyList['totals'][$_j] = new TmKeyManagement_ClientTmKeyStruct($key->tm_key);
}
/*
* Now take the JOB keys
*/
$job_keyList = json_decode($data[0]['tm_keys'], true);
$this->tid = count($job_keyList) > 0;
/**
* Start this N^2 cycle from keys of the job,
* these should be statistically lesser than the keys of the user
*
* @var $keyList array
*/
foreach ($job_keyList as $jobKey) {
$jobKey = new TmKeyManagement_ClientTmKeyStruct($jobKey);
if ($this->isLoggedIn() && count($reverse_lookup_user_personal_keys['pos'])) {
/*
* If user has some personal keys, check for the job keys if they are present, and obfuscate
* when they are not
*/
$_index_position = array_search($jobKey->key, $reverse_lookup_user_personal_keys['pos']);
if ($_index_position !== false) {
//i found a key in the job that is present in my database
//i'm owner?? and the key is an owner type key?
if (!$jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) {
$jobKey->r = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['r']};
$jobKey->w = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['w']};
$jobKey = $jobKey->hideKey($uid);
} else {
if ($jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) {
// I'm not the job owner, but i know the key because it is in my keyring
// so, i can upload and download TMX, but i don't want it to be removed from job
// in tm.html relaxed the control to "key.edit" to enable buttons
// $jobKey = $jobKey->hideKey( $uid ); // enable editing
} else {
if ($jobKey->owner && $this->userRole == TmKeyManagement_Filter::OWNER) {
//do Nothing
}
}
}
unset($this->_keyList['totals'][$_index_position]);
} else {
/*
* This is not a key of that user, set right and obfuscate
*/
$jobKey->r = true;
$jobKey->w = true;
$jobKey = $jobKey->hideKey(-1);
}
$this->_keyList['job_keys'][] = $jobKey;
} else {
/*
* This user is anonymous or it has no keys in its keyring, obfuscate all
*/
$jobKey->r = true;
$jobKey->w = true;
$this->_keyList['job_keys'][] = $jobKey->hideKey(-1);
}
}
//clean unordered keys
$this->_keyList['totals'] = array_values($this->_keyList['totals']);
/**
* Retrieve information about job errors
* ( Note: these information are fed by the revision process )
* @see setRevisionController
*/
$jobQA = new Revise_JobQA($this->jid, $this->password, $wStruct->getTotal());
$jobQA->retrieveJobErrorTotals();
$jobVote = $jobQA->evalJobVote();
$this->qa_data = json_encode($jobQA->getQaData());
$this->qa_overall = $jobVote['minText'];
$engine = new EnginesModel_EngineDAO(Database::obtain());
//this gets all engines of the user
if ($this->isLoggedIn()) {
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->type = 'MT';
$engineQuery->uid = $uid;
$engineQuery->active = 1;
$mt_engines = $engine->read($engineQuery);
} else {
$mt_engines = array();
}
// this gets MyMemory
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->type = 'TM';
$engineQuery->active = 1;
$tms_engine = $engine->setCacheTTL(3600 * 24 * 30)->read($engineQuery);
//this gets MT engine active for the job
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->id = $this->project_status['id_mt_engine'];
$engineQuery->active = 1;
$active_mt_engine = $engine->setCacheTTL(60 * 10)->read($engineQuery);
/*
* array_unique cast EnginesModel_EngineStruct to string
*
* EnginesModel_EngineStruct implements __toString method
*
*/
$this->translation_engines = array_unique(array_merge($active_mt_engine, $tms_engine, $mt_engines));
}
public function doAction()
{
$debug = array();
$debug['total'][] = time();
//get job language and data
//Fixed Bug: need a specific job, because we need The target Language
//Removed from within the foreach cycle, the job is always the same....
$jobData = $this->jobInfo = getJobData($this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($jobData) || !$pCheck->grantJobAccessByJobData($jobData, $this->password)) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
$debug['get_file'][] = time();
$files_job = getFilesForJob($this->id_job, $this->id_file);
$debug['get_file'][] = time();
$nonew = 0;
$output_content = array();
/*
* the procedure is now as follows:
* 1)original file is loaded from DB into RAM and the flushed in a temp file on disk; a file handler is obtained
* 2)RAM gets freed from original content
* 3)the file is read chunk by chunk by a stream parser: for each tran-unit that is encountered,
* target is replaced (or added) with the corresponding translation among segments
* the current string in the buffer is flushed on standard output
* 4)the temporary file is deleted by another process after some time
*
*/
//file array is chuncked. Each chunk will be used for a parallel conversion request.
$files_job = array_chunk($files_job, self::FILES_CHUNK_SIZE);
foreach ($files_job as $chunk) {
$converter = new FileFormatConverter();
$files_buffer = array();
foreach ($chunk as $file) {
$mime_type = $file['mime_type'];
$fileID = $file['id_file'];
$current_filename = $file['filename'];
$original_xliff = $file['xliff_file'];
//get path
$path = INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/' . $fileID . '/' . $current_filename . "_" . uniqid('', true) . '.sdlxliff';
//make dir if doesn't exist
if (!file_exists(dirname($path))) {
Log::doLog('exec ("chmod 666 ' . escapeshellarg($path) . '");');
mkdir(dirname($path), 0777, true);
exec("chmod 666 " . escapeshellarg($path));
}
//create file
$fp = fopen($path, 'w+');
//flush file to disk
fwrite($fp, $original_xliff);
//free memory, as we can work with file on disk now
unset($original_xliff);
$debug['get_segments'][] = time();
$data = getSegmentsDownload($this->id_job, $this->password, $fileID, $nonew);
$debug['get_segments'][] = time();
//create a secondary indexing mechanism on segments' array; this will be useful
//prepend a string so non-trans unit id ( ex: numerical ) are not overwritten
//clean also not valid xml entities ( charactes with ascii < 32 and different from 0A, 0D and 09
$regexpEntity = '/&#x(0[0-8BCEF]|1[0-9A-F]|7F);/u';
//remove binary chars in some xliff files
$regexpAscii = '/([\\x{00}-\\x{1F}\\x{7F}]{1})/u';
foreach ($data as $i => $k) {
$data['matecat|' . $k['internal_id']][] = $i;
//FIXME: temporary patch
$data[$i]['translation'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['translation']);
$data[$i]['segment'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['segment']);
$sanitized_src = preg_replace($regexpAscii, '', $data[$i]['segment']);
$sanitized_trg = preg_replace($regexpAscii, '', $data[$i]['translation']);
$sanitized_src = preg_replace($regexpEntity, '', $sanitized_src);
$sanitized_trg = preg_replace($regexpEntity, '', $sanitized_trg);
if ($sanitized_src != null) {
$data[$i]['segment'] = $sanitized_src;
}
if ($sanitized_trg != null) {
$data[$i]['translation'] = $sanitized_trg;
}
}
$debug['replace'][] = time();
//instatiate parser
$xsp = new XliffSAXTranslationReplacer($path, $data, Langs_Languages::getInstance()->getLangRegionCode($jobData['target']), $fp);
if ($this->download_type == 'omegat') {
$xsp->setSourceInTarget(true);
}
//run parsing
Log::doLog("work on " . $fileID . " " . $current_filename);
$xsp->replaceTranslation();
fclose($fp);
unset($xsp);
$debug['replace'][] = time();
$output_xliff = file_get_contents($path . '.out.sdlxliff');
$output_content[$fileID]['documentContent'] = $output_xliff;
$output_content[$fileID]['filename'] = $current_filename;
unset($output_xliff);
if ($this->forceXliff) {
$file_info_details = pathinfo($output_content[$fileID]['filename']);
$output_content[$fileID]['filename'] = $file_info_details['filename'] . ".out.sdlxliff";
}
//TODO set a flag in database when file uploaded to know if this file is a proprietary xlf converted
//TODO so we can load from database the original file blob ONLY when needed
/**
* Conversion Enforce
*/
$convertBackToOriginal = true;
try {
//if it is a not converted file ( sdlxliff ) we have an empty field original_file
//so we can simplify all the logic with:
// is empty original_file? if it is, we don't need conversion back because
// we already have an sdlxliff or an accepted file
$file['original_file'] = @gzinflate($file['original_file']);
if (!INIT::$CONVERSION_ENABLED || empty($file['original_file']) && $mime_type == 'sdlxliff' || $this->forceXliff) {
$convertBackToOriginal = false;
Log::doLog("SDLXLIFF: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
} else {
//TODO: dos2unix ??? why??
//force unix type files
Log::doLog("NO SDLXLIFF, Conversion enforced: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
if ($convertBackToOriginal) {
$output_content[$fileID]['out_xliff_name'] = $path . '.out.sdlxliff';
$output_content[$fileID]['source'] = $jobData['source'];
$output_content[$fileID]['target'] = $jobData['target'];
$files_buffer[$fileID] = $output_content[$fileID];
} elseif ($this->forceXliff) {
$this->cleanFilePath($output_content[$fileID]['documentContent']);
}
}
$debug['do_conversion'][] = time();
$convertResult = $converter->multiConvertToOriginal($files_buffer, $chosen_machine = false);
foreach (array_keys($files_buffer) as $fileID) {
$output_content[$fileID]['documentContent'] = $this->removeTargetMarks($convertResult[$fileID]['documentContent'], $files_buffer[$fileID]['filename']);
//in case of .strings, they are required to be in UTF-16
//get extension to perform file detection
$extension = pathinfo($output_content[$fileID]['filename'], PATHINFO_EXTENSION);
if (strtoupper($extension) == 'STRINGS') {
//use this function to convert stuff
$encodingConvertedFile = CatUtils::convertEncoding('UTF-16', $output_content[$fileID]['documentContent']);
//strip previously added BOM
$encodingConvertedFile[1] = $converter->stripBOM($encodingConvertedFile[1], 16);
//store new content
$output_content[$fileID]['documentContent'] = $encodingConvertedFile[1];
//trash temporary data
unset($encodingConvertedFile);
}
}
// $output_content[ $fileID ][ 'documentContent' ] = $convertResult[ 'documentContent' ];
unset($convertResult);
$debug['do_conversion'][] = time();
}
//set the file Name
$pathinfo = pathinfo($this->fname);
$this->filename = $pathinfo['filename'] . "_" . $jobData['target'] . "." . $pathinfo['extension'];
//qui prodest to check download type?
if ($this->download_type == 'omegat') {
$this->filename .= ".zip";
$tmsService = new TMSService();
$tmsService->setOutputType('tm');
/**
* @var $tmFile SplTempFileObject
*/
$tmFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tmsService->setOutputType('mt');
/**
* @var $mtFile SplTempFileObject
*/
$mtFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tm_id = uniqid('tm');
$mt_id = uniqid('mt');
$output_content[$tm_id] = array('documentContent' => '', 'filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_TM . tmx");
foreach ($tmFile as $lineNumber => $content) {
$output_content[$tm_id]['documentContent'] .= $content;
}
$output_content[$mt_id] = array('documentContent' => '', 'filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_MT . tmx");
foreach ($mtFile as $lineNumber => $content) {
$output_content[$mt_id]['documentContent'] .= $content;
}
$this->createOmegaTZip($output_content, $jobData['source'], $jobData['target']);
//add zip archive content here;
} else {
if (count($output_content) > 1) {
if ($pathinfo['extension'] != 'zip') {
if ($this->forceXliff) {
$this->filename = $this->id_job . ".zip";
} else {
$this->filename = $pathinfo['basename'] . ".zip";
}
}
$this->composeZip($output_content, $jobData['source']);
//add zip archive content here;
} else {
//always an array with 1 element, pop it, Ex: array( array() )
$output_content = array_pop($output_content);
$this->setContent($output_content);
}
}
$debug['total'][] = time();
Utils::deleteDir(INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/');
}
