public function doAction()
{
// Just add the XLIFF extension, the DetectProprietaryXliff class needs it
$file_path = $_FILES['xliff']['tmp_name'] . '.xlf';
move_uploaded_file($_FILES['xliff']['tmp_name'], $file_path);
// Detect XLIFF type
$fileInfo = DetectProprietaryXliff::getInfo($file_path);
$converterVersion = $fileInfo['converter_version'];
$converter = new FileFormatConverter($converterVersion);
$conversion = $converter->multiConvertToOriginal(array(1 => array('document_content' => file_get_contents($file_path))));
if ($conversion[1]['isSuccess'] !== true) {
$this->error = true;
$this->errorMessage = $conversion[1]['errorMessage'];
} else {
$this->content = json_encode(array("fileName" => $conversion[1]['filename'], "fileContent" => base64_encode($conversion[1]['document_content']), "size" => filesize($file_path), "type" => mime_content_type($file_path), "message" => "File downloaded! Check your download folder"));
}
}
protected static function _reset()
{
self::$fileType = array('info' => array(), 'proprietary' => false, 'proprietary_name' => null, 'proprietary_short_name' => null);
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_customer'] = 'translated_user';
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid.", -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//found TMX, enable language checking routines
$this->checkTMX = 1;
array_unshift($sortedFiles, $fileName);
} else {
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$file = new stdClass();
$file->file_path = "{$uploadDir}/{$fileName}";
$this->tmxServiceWrapper->setName($fileName);
$this->tmxServiceWrapper->setFile(array($file));
try {
$this->tmxServiceWrapper->addTmxInMyMemory();
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//in any case, skip the rest of the loop, go to the next file
continue;
}
/**
* Conversion Enforce
*
* we have to know if a file can be found in _converted directory
*
* Check Extension no more sufficient, we want check content
* if this is an idiom xlf file type, conversion are enforced
* $enforcedConversion = true; //( if conversion is enabled )
*/
$isAConvertedFile = true;
try {
$fileType = DetectProprietaryXliff::getInfo(INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'] . DIRECTORY_SEPARATOR . $fileName);
if (DetectProprietaryXliff::isXliffExtension()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//ONLY IDIOM is forced to be converted
//if file is not proprietary like idiom AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
} else {
//if conversion enforce is active
//we force all xliff files but not files produced by SDL Studio because we can handle them
if ($fileType['proprietary_short_name'] == 'trados') {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
/**
* Application misconfiguration.
* upload should not be happened, but if we are here, raise an error.
* @see upload.class.php
* */
$this->projectStructure['result']['errors'][] = array("code" => -8, "message" => "Proprietary xlf format detected. Not able to import this XLIFF file. ({$fileName})");
setcookie("upload_session", "", time() - 10000);
return -1;
//stop execution
} elseif (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
$mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
$original_content = "";
/*
if it's not one of the listed formats (or it is, but you had to convert it anyway),
and conversion is enabled in first place
*/
if ($isAConvertedFile) {
//converted file is inside "_converted" directory
$fileDir = $uploadDir . '_converted';
$original_content = file_get_contents("{$uploadDir}/{$fileName}");
$sha1_original = sha1($original_content);
$original_content = gzdeflate($original_content, 5);
//file name is a xliff converted like: 'a_word_document.doc.sdlxliff'
$real_fileName = $fileName . '.sdlxliff';
} else {
//filename is already an xliff and it is in a canonical normal directory
$sha1_original = "";
$fileDir = $uploadDir;
$real_fileName = $fileName;
}
$filePathName = $fileDir . DIRECTORY_SEPARATOR . $real_fileName;
if (!file_exists($filePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = pathinfo($filePathName);
if ($info['extension'] == 'xliff' || $info['extension'] == 'sdlxliff' || $info['extension'] == 'xlf') {
$contents = file_get_contents($filePathName);
} else {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$fid = insertFile($this->projectStructure, $fileName, $mimeType, $contents, $sha1_original, $original_content);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments($contents, $fid);
unset($contents);
//free memory
//Log::doLog( $this->projectStructure['segments'] );
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File is Too large. ( {$fileName} )");
}
Log::doLog($e->getMessage());
}
//exit;
}
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
/****************/
//loop again through files to check to check for TMX loading
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$this->tmxServiceWrapper->setName($fileName);
$result = array();
//is the TM loaded?
//wait until current TMX is loaded
while (true) {
try {
$result = $this->tmxServiceWrapper->tmxUploadStatus();
if ($result['completed']) {
//"$fileName" has been loaded into MyMemory"
//exit the loop
break;
}
//"waiting for "$fileName" to be loaded into MyMemory"
sleep(3);
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
//exit project creation
return false;
}
}
//once the language is loaded, check if language is compliant (unless something useful has already been found)
if (1 == $this->checkTMX) {
//get localized target languages of TM (in case it's a multilingual TM)
$tmTargets = explode(';', $result['data']['target_lang']);
//indicates if something has been found for current memory
$found = false;
//compare localized target languages array (in case it's a multilingual project) to the TM supplied
//if nothing matches, then the TM supplied can't have matches for this project
//create an empty var and add the source language too
$project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
foreach ($project_languages as $projectTarget) {
if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
$found = true;
break;
}
}
//if this TM matches the project lagpair and something has been found
if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
//the TMX is good to go
$this->checkTMX = 0;
} elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
/*
* This means that the TMX has a srclang as specification in the header. Warn the user.
* Ex:
* <header creationtool="SDL Language Platform"
* creationtoolversion="8.0"
* datatype="rtf"
* segtype="sentence"
* adminlang="DE-DE"
* srclang="DE-DE" />
*/
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
$this->checkTMX = 0;
Log::doLog($this->projectStructure['result']);
}
}
}
}
if (1 == $this->checkTMX) {
//this means that noone of uploaded TMX were usable for this project. Warn the user.
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
Log::doLog($this->projectStructure['result']);
return false;
}
if (!empty($this->projectStructure['result']['errors'])) {
Log::doLog("Project Creation Failed. Sent to Output all errors.");
Log::doLog($this->projectStructure['result']['errors']);
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n FROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
$rows = $this->dbHandler->fetch_array($query_visible_segments);
if (!$rows) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($uploadDir);
if (is_dir($uploadDir . '_converted')) {
Utils::deleteDir($uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
/**
* TODO: remove after queue implementation
*/
// if( 0 && $project_summary[0]['project_segments'] > 50000 ) {
// $this->projectStructure[ 'status' ] = Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
//
// $msg = "
// WARNING: a project with more than 50.000 segments was created. ( " . $project_summary[0]['project_segments'] . " )\n" .
// var_export( $this->projectStructure[ 'result' ], true ) . "\n\n" .
// " " .
// var_export( $project_summary[0] , true ) . "\n";
//
// Utils::sendErrMailReport( $msg, "Alert: Project Creation Abort. - " );
//
// }
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
// var_dump($this->projectStructure);
// exit;
}
/**
* Backwards compatibility method and forward
*
* Works by Reference variable
*
* @param $fileMetaData [
* "id_file",
* "filename",
* "source",
* "mime_type",
* "sha1_original_file"
* ]
*/
private function migrateFileDB2FS(&$fileMetaData)
{
//create temporary storage to place stuff
$tempDir = "/tmp" . DIRECTORY_SEPARATOR . uniqid("", true);
mkdir($tempDir, 0755);
//fetch xliff from the files database
$xliffContent = $this->getXliffFromDB($fileMetaData['id_file']);
//try pulling the original content too (if it's empty it means that it was an unconverted xliff)
$fileContent = $this->getOriginalFromDB($fileMetaData['id_file']);
if (!empty($fileContent)) {
//it's a converted file
//i'd like to know it's real extension....
//create temporary file with appropriately modified name
$result = DetectProprietaryXliff::getInfoByStringData($xliffContent);
if ($result['proprietary_short_name'] == 'trados') {
$tempXliff = $tempDir . DIRECTORY_SEPARATOR . $fileMetaData['filename'] . ".sdlxliff";
} else {
$tempXliff = $tempDir . DIRECTORY_SEPARATOR . $fileMetaData['filename'] . ".xlf";
}
//create file
$tempOriginal = $tempDir . DIRECTORY_SEPARATOR . $fileMetaData['filename'];
//flush content
file_put_contents($tempOriginal, $fileContent);
//get hash, based on original
$sha1 = sha1($fileContent);
//free memory
unset($fileContent);
} else {
//if it's a unconverted xliff
//create temporary file with original name
$tempXliff = $tempDir . DIRECTORY_SEPARATOR . $fileMetaData['filename'];
// set original to empty
$tempOriginal = false;
//get hash
$sha1 = sha1($xliffContent);
}
//flush xliff file content
file_put_contents($tempXliff, $xliffContent);
//free memory
unset($xliffContent);
if (stripos($fileMetaData['sha1_original_file'], DIRECTORY_SEPARATOR) === false) {
$query = "select create_date from projects where id = {$fileMetaData['id_project']}";
$db = Database::obtain();
$results = $db->fetch_array($query);
$dateHashPath = date_create($results[0]['create_date'])->format('Ymd') . DIRECTORY_SEPARATOR . $sha1;
$db->update('files', array("sha1_original_file" => $dateHashPath), 'id = ' . $fileMetaData['id_file']);
//update Reference
$fileMetaData['sha1_original_file'] = $dateHashPath;
}
//build a cache package
$this->makeCachePackage($sha1, $fileMetaData['source'], $tempOriginal, $tempXliff);
//build a file package
$this->moveFromCacheToFileDir($fileMetaData['sha1_original_file'], $fileMetaData['source'], $fileMetaData['id_file']);
//clean temporary stuff
Utils::deleteDir($tempDir);
}
/**
* Remove the tag mrk if the file is an xlif and if the file is a globalsight file
*
* Also, check for encoding and transform utf16 to utf8 and back
*
* @param $documentContent
* @param $path
*
* @return string
*/
public function ifGlobalSightXliffRemoveTargetMarks($documentContent, $path)
{
$extension = FilesStorage::pathinfo_fix($path);
if (!DetectProprietaryXliff::isXliffExtension($extension)) {
return $documentContent;
}
$is_utf8 = true;
$original_charset = 'utf-8';
//not used, useful only to avoid IDE warning for not used variable
//The file is UTF-16 Encoded
if (stripos(substr($documentContent, 0, 100), "<?xml ") === false) {
$is_utf8 = false;
list($original_charset, $documentContent) = CatUtils::convertEncoding('UTF-8', $documentContent);
}
//avoid in memory copy of very large files if possible
$detect_result = DetectProprietaryXliff::getInfoByStringData(substr($documentContent, 0, 1024));
//clean mrk tags for GlobalSight application compatibility
//this should be a sax parser instead of in memory copy for every trans-unit
if ($detect_result['proprietary_short_name'] == 'globalsight') {
// Getting Trans-units
$trans_units = explode('<trans-unit', $documentContent);
foreach ($trans_units as $pos => $trans_unit) {
// First element in the XLIFF split is the header, not the first file
if ($pos > 0) {
//remove seg-source tags
$trans_unit = preg_replace('|<seg-source.*?</seg-source>|si', '', $trans_unit);
//take the target content
$trans_unit = preg_replace('#<mrk[^>]+>|</mrk>#si', '', $trans_unit);
$trans_units[$pos] = $trans_unit;
}
}
// End of trans-units
$documentContent = implode('<trans-unit', $trans_units);
}
if (!$is_utf8) {
list($__utf8, $documentContent) = CatUtils::convertEncoding($original_charset, $documentContent);
}
return $documentContent;
}
public function doAction()
{
$this->file_name = html_entity_decode($this->file_name, ENT_QUOTES);
$file_path = $this->intDir . DIRECTORY_SEPARATOR . $this->file_name;
if (!file_exists($file_path)) {
$this->result['code'] = -6;
// No Good, Default
$this->result['errors'][] = array("code" => -6, "message" => "Error during upload. Please retry.", 'debug' => FilesStorage::basename_fix($this->file_name));
return -1;
}
//XLIFF Conversion management
//cyclomatic complexity 9999999 ..... but it works, for now.
try {
$fileType = DetectProprietaryXliff::getInfo($file_path);
if (DetectProprietaryXliff::isXliffExtension() || DetectProprietaryXliff::getMemoryFileType()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//if file is not proprietary AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary'] || DetectProprietaryXliff::getMemoryFileType()) {
$this->result['code'] = 1;
// OK for client
//This file has to be linked to cache!
return 0;
//ok don't convert a standard sdlxliff
}
} else {
// if conversion enforce is active
// we force all xliff files but not files produced by
// SDL Studio or by the MateCAT converters, because we
// can handle them
if ($fileType['proprietary_short_name'] == 'matecat_converter' || $fileType['proprietary_short_name'] == 'trados' || DetectProprietaryXliff::getMemoryFileType()) {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
unlink($file_path);
$this->result['code'] = -7;
// No Good, Default
$this->result['errors'][] = array("code" => -7, "message" => 'Matecat Open-Source does not support ' . ucwords($fileType['proprietary_name']) . '. Use MatecatPro.', 'debug' => FilesStorage::basename_fix($this->file_name));
return -1;
} elseif (!$fileType['proprietary']) {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} catch (Exception $e) {
//try catch not used because of exception no more raised
$this->result['code'] = -8;
// No Good, Default
$this->result['errors'][] = array("code" => -8, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return -1;
}
//compute hash to locate the file in the cache
$sha1 = sha1_file($file_path);
//initialize path variable
$cachedXliffPath = false;
//get storage object
$fs = new FilesStorage();
//TODO: REMOVE SET ENVIRONMENT FOR LEGACY CONVERSION INSTANCES
if (INIT::$LEGACY_CONVERSION !== false) {
INIT::$SAVE_SHASUM_FOR_FILES_LOADED = false;
}
//if already present in database cache get the converted without convert it again
if (INIT::$SAVE_SHASUM_FOR_FILES_LOADED) {
//move the file in the right directory from the packages to the file dir
$cachedXliffPath = $fs->getXliffFromCache($sha1, $this->source_lang);
if (!$cachedXliffPath) {
Log::doLog("Failed to fetch xliff for {$sha1} from disk cache (is file there?)");
}
}
//if invalid or no cached version
if (!isset($cachedXliffPath) or empty($cachedXliffPath)) {
//we have to convert it
// By default, use always the new converters...
$converterVersion = Constants_ConvertersVersions::LATEST;
if ($this->segmentation_rule !== null) {
// ...but new converters don't support custom segmentation rules.
// if $this->segmentation_rule is set use the old ones.
$converterVersion = Constants_ConvertersVersions::LEGACY;
}
//TODO: REMOVE SET ENVIRONMENT FOR LEGACY CONVERSION INSTANCES
if (INIT::$LEGACY_CONVERSION !== false) {
$converterVersion = Constants_ConvertersVersions::LEGACY;
}
$converter = new FileFormatConverter($converterVersion);
if (strpos($this->target_lang, ',') !== false) {
$single_language = explode(',', $this->target_lang);
$single_language = $single_language[0];
} else {
$single_language = $this->target_lang;
}
$convertResult = $converter->convertToSdlxliff($file_path, $this->source_lang, $single_language, false, $this->segmentation_rule);
if ($convertResult['isSuccess'] == 1) {
/* try to back convert the file */
$output_content = array();
$output_content['out_xliff_name'] = $file_path . '.out.sdlxliff';
$output_content['source'] = $this->source_lang;
$output_content['target'] = $single_language;
$output_content['content'] = $convertResult['xliffContent'];
$output_content['filename'] = $this->file_name;
$back_convertResult = $converter->convertToOriginal($output_content);
/* try to back convert the file */
if ($back_convertResult['isSuccess'] == false) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: there is a problem with this file, it cannot be converted back to the original one.";
$this->result['code'] = -110;
$this->result['errors'][] = array("code" => -110, "message" => $convertResult['errorMessage'], 'debug' => FilesStorage::basename_fix($this->file_name));
return false;
}
//store converted content on a temporary path on disk (and off RAM)
$cachedXliffPath = tempnam("/tmp", "MAT_XLF");
file_put_contents($cachedXliffPath, $convertResult['xliffContent']);
unset($convertResult['xliffContent']);
/*
store the converted file in the cache
put a reference in the upload dir to the cache dir, so that from the UUID we can reach the converted file in the cache
(this is independent by the "save xliff for caching" options, since we always end up storing original and xliff on disk)
*/
//save in cache
$res_insert = $fs->makeCachePackage($sha1, $this->source_lang, $file_path, $cachedXliffPath);
if (!$res_insert) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: File upload failed because you have MateCat running in multiple tabs. Please close all other MateCat tabs in your browser.";
$this->result['code'] = -103;
$this->result['errors'][] = array("code" => -103, "message" => $convertResult['errorMessage'], 'debug' => FilesStorage::basename_fix($this->file_name));
unset($cachedXliffPath);
return false;
}
} else {
$file = FilesStorage::pathinfo_fix($this->file_name);
switch ($file['extension']) {
case 'docx':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
break;
case 'doc':
case 'rtf':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOCX.";
break;
case 'inx':
$defaultError = "Importing Error. Try to commit changes in InDesign before importing.";
break;
case 'idml':
$defaultError = "Importing Error. MateCat does not support this version of InDesign, try converting it to a previous one.";
break;
default:
$defaultError = "Importing error. Try opening and saving the document with a new name.";
break;
}
if (stripos($convertResult['errorMessage'], "failed to create SDLXLIFF.") !== false || stripos($convertResult['errorMessage'], "COM target does not implement IDispatch") !== false) {
$convertResult['errorMessage'] = "Error: failed importing file.";
} elseif (stripos($convertResult['errorMessage'], "Unable to open Excel file - it may be password protected") !== false) {
$convertResult['errorMessage'] = $convertResult['errorMessage'] . " Try to remove protection using the Unprotect Sheet command on Windows Excel.";
} elseif (stripos($convertResult['errorMessage'], "The document contains unaccepted changes") !== false) {
$convertResult['errorMessage'] = "The document contains track changes. Accept all changes before uploading it.";
} elseif (stripos($convertResult['errorMessage'], "Error: Could not find file") !== false || stripos($convertResult['errorMessage'], "tw4winMark") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "Attempted to read or write protected memory") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "The document was created in Microsoft Word 97 or earlier")) {
$convertResult['errorMessage'] = $defaultError;
} elseif ($file['extension'] == 'csv' && empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "This CSV file is not eligible to be imported due internal wrong format. Try to convert in TXT using UTF8 encoding";
} elseif (empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "Failed to convert file. Internal error. Please Try again.";
} elseif (stripos($convertResult['errorMessage'], "DocumentFormat.OpenXml.dll") !== false) {
//this error is triggered on DOCX when converter's parser can't decode some regions of the file
$convertResult['errorMessage'] = "Conversion error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
} elseif ($file['extension'] == 'idml') {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "Error: The source language of the file") !== false) {
//Error: The source language of the file (English (United States)) is different from the project source language.
//we take the error, is good
} else {
$convertResult['errorMessage'] = "Import error. Try converting it to a compatible file format (e.g. doc > docx, xlsx > xls)";
}
//custom error message passed directly to javascript client and displayed as is
$this->result['code'] = -100;
$this->result['errors'][] = array("code" => -100, "message" => $convertResult['errorMessage'], "debug" => $file['basename']);
}
}
//if everything went well and we've obtained a path toward a valid package (original+xliff), either via cache or conversion
if (isset($cachedXliffPath) and !empty($cachedXliffPath)) {
//FILE Found in cache, destroy the already present shasum for other languages ( if user swapped languages )
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->cookieDir;
$fs->deleteHashFromUploadDir($uploadDir, $sha1 . "|" . $this->source_lang);
//put reference to cache in upload dir to link cache to session
$fs->linkSessionToCache($sha1, $this->source_lang, $this->cookieDir, FilesStorage::basename_fix($file_path));
//a usable package is available, give positive feedback
$this->result['code'] = 1;
}
return 0;
}
private function isConversionToEnforce($fileName)
{
$isAConvertedFile = true;
$fullPath = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'] . DIRECTORY_SEPARATOR . $fileName;
try {
$isAConvertedFile = DetectProprietaryXliff::isConversionToEnforce($fullPath);
if (-1 === $isAConvertedFile) {
$this->projectStructure['result']['errors'][] = array("code" => -8, "message" => "Proprietary xlf format detected. Not able to import this XLIFF file. ({$fileName})");
setcookie("upload_session", "", time() - 10000);
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
return $isAConvertedFile;
}
public function doAction()
{
$this->result['code'] = 0;
// No Good, Default
if (empty($this->file_name)) {
$this->result['code'] = -1;
// No Good, Default
$this->result['errors'][] = array("code" => -1, "message" => "Error: missing file name.");
return false;
}
$this->file_name = html_entity_decode($this->file_name, ENT_QUOTES);
$file_path = $this->intDir . DIRECTORY_SEPARATOR . $this->file_name;
if (!file_exists($file_path)) {
$this->result['code'] = -6;
// No Good, Default
$this->result['errors'][] = array("code" => -6, "message" => "Error during upload. Please retry.");
return -1;
}
//get uploaded file from disk
$original_content = file_get_contents($file_path);
$sha1 = sha1($original_content);
//if already present in database cache get the converted without convert it again
if (INIT::$SAVE_SHASUM_FOR_FILES_LOADED) {
$xliffContent = getXliffBySHA1($sha1, $this->source_lang, $this->target_lang, $this->cache_days, $this->segmentation_rule);
}
//XLIFF Conversion management
//cyclomatic complexity 9999999 ..... but it works, for now.
try {
$fileType = DetectProprietaryXliff::getInfo($file_path);
if (DetectProprietaryXliff::isXliffExtension()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//ONLY IDIOM is forced to be converted
//if file is not proprietary like idiom AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary'] || $fileType['info']['extension'] == 'tmx') {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
} else {
//if conversion enforce is active
//we force all xliff files but not files produced by SDL Studio because we can handle them
if ($fileType['proprietary_short_name'] == 'trados' || $fileType['info']['extension'] == 'tmx') {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
unlink($file_path);
$this->result['code'] = -7;
// No Good, Default
$this->result['errors'][] = array("code" => -7, "message" => 'Matecat Open-Source does not support ' . ucwords($fileType['proprietary_name']) . '. Use MatecatPro.', 'debug' => basename($this->file_name));
return -1;
} elseif (!$fileType['proprietary']) {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} catch (Exception $e) {
//try catch not used because of exception no more raised
$this->result['code'] = -8;
// No Good, Default
$this->result['errors'][] = array("code" => -8, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return -1;
}
//there is a cached copy of conversion? inflate
if (isset($xliffContent) && !empty($xliffContent)) {
$xliffContent = gzinflate($xliffContent);
$res = $this->put_xliff_on_file($xliffContent, $this->intDir);
if (!$res) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: failed to save converted file from cache to disk";
$this->result['code'] = -101;
$this->result['errors'][] = array("code" => -101, "message" => $convertResult['errorMessage'], 'debug' => basename($this->file_name));
}
//else whe have to convert it
} else {
$original_content_zipped = gzdeflate($original_content, 5);
unset($original_content);
$converter = new FileFormatConverter($this->segmentation_rule);
if (strpos($this->target_lang, ',') !== false) {
$single_language = explode(',', $this->target_lang);
$single_language = $single_language[0];
} else {
$single_language = $this->target_lang;
}
$convertResult = $converter->convertToSdlxliff($file_path, $this->source_lang, $single_language, false, $this->segmentation_rule);
if ($convertResult['isSuccess'] == 1) {
/* try to back convert the file */
$output_content = array();
$output_content['out_xliff_name'] = $file_path . '.out.sdlxliff';
$output_content['source'] = $this->source_lang;
$output_content['target'] = $single_language;
$output_content['content'] = $convertResult['xliffContent'];
$output_content['filename'] = $this->file_name;
$back_convertResult = $converter->convertToOriginal($output_content);
/* try to back convert the file */
if ($back_convertResult['isSuccess'] == false) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: there is a problem with this file, it cannot be converted back to the original one.";
$this->result['code'] = -110;
$this->result['errors'][] = array("code" => -110, "message" => $convertResult['errorMessage'], 'debug' => basename($this->file_name));
return false;
}
//$uid = $convertResult['uid']; // va inserito nel database
$xliffContent = $convertResult['xliffContent'];
$xliffContentZipped = gzdeflate($xliffContent, 5);
//cache the converted file
if (INIT::$SAVE_SHASUM_FOR_FILES_LOADED) {
$res_insert = insertFileIntoMap($sha1, $this->source_lang, $this->target_lang, $original_content_zipped, $xliffContentZipped, $this->segmentation_rule);
if ($res_insert < 0) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: File too large";
$this->result['code'] = -102;
$this->result['errors'][] = array("code" => -102, "message" => $convertResult['errorMessage'], 'debug' => basename($this->file_name));
return;
}
}
unset($xliffContentZipped);
$res = $this->put_xliff_on_file($xliffContent, $this->intDir);
if (!$res) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: failed to save file on disk";
$this->result['code'] = -103;
$this->result['errors'][] = array("code" => -103, "message" => $convertResult['errorMessage'], 'debug' => basename($this->file_name));
return false;
}
} else {
$file = pathinfo($this->file_name);
switch ($file['extension']) {
case 'docx':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
break;
case 'doc':
case 'rtf':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOCX.";
break;
case 'inx':
$defaultError = "Importing Error. Try to commit changes in InDesign before importing.";
break;
case 'idml':
$defaultError = "Importing Error. MateCat does not support this version of InDesign, try converting it to a previous one.";
break;
default:
$defaultError = "Importing error. Try opening and saving the document with a new name.";
break;
}
if (stripos($convertResult['errorMessage'], "failed to create SDLXLIFF.") !== false || stripos($convertResult['errorMessage'], "COM target does not implement IDispatch") !== false) {
$convertResult['errorMessage'] = "Error: failed importing file.";
} elseif (stripos($convertResult['errorMessage'], "Unable to open Excel file - it may be password protected") !== false) {
$convertResult['errorMessage'] = $convertResult['errorMessage'] . " Try to remove protection using the Unprotect Sheet command on Windows Excel.";
} elseif (stripos($convertResult['errorMessage'], "The document contains unaccepted changes") !== false) {
$convertResult['errorMessage'] = "The document contains track changes. Accept all changes before uploading it.";
} elseif (stripos($convertResult['errorMessage'], "Error: Could not find file") !== false || stripos($convertResult['errorMessage'], "tw4winMark") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "Attempted to read or write protected memory") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "The document was created in Microsoft Word 97 or earlier")) {
$convertResult['errorMessage'] = $defaultError;
} elseif ($file['extension'] == 'csv' && empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "This CSV file is not eligible to be imported due internal wrong format. Try to convert in TXT using UTF8 encoding";
} elseif (empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "Failed to convert file. Internal error. Please Try again.";
} elseif (stripos($convertResult['errorMessage'], "DocumentFormat.OpenXml.dll") !== false) {
//this error is triggered on DOCX when converter's parser can't decode some regions of the file
$convertResult['errorMessage'] = "Conversion error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
} elseif ($file['extension'] == 'idml') {
$convertResult['errorMessage'] = $defaultError;
} else {
$convertResult['errorMessage'] = "Import error. Try converting it to a compatible file format (e.g. doc > docx, xlsx > xls)";
}
//custom error message passed directly to javascript client and displayed as is
$this->result['code'] = -100;
$this->result['errors'][] = array("code" => -100, "message" => $convertResult['errorMessage'], "debug" => $file['basename']);
}
}
}
public static function isJobBasedOnMateCatFilters($jobId)
{
$fs = new FilesStorage();
$files = $fs->getFilesForJob($jobId, null);
foreach ($files as $file) {
$fileType = DetectProprietaryXliff::getInfo($files[0]['xliffFilePath']);
if ($fileType['proprietary_short_name'] !== 'matecat_converter') {
// If only one XLIFF is not created with MateCat Filters, we can't say
// that the project is entirely based on new Filters
return false;
}
}
// If the flow arrives here, all the files' XLIFFs are based on new Filters
return true;
}
public static function isConversionToEnforce($fullPath)
{
$isAConvertedFile = true;
$fileType = self::getInfo($fullPath);
if (self::isXliffExtension()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//if file is not proprietary AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
} else {
//if conversion enforce is active
//we force all xliff files but not files produced by SDL Studio because we can handle them
if ($fileType['proprietary_short_name'] == 'matecat_converter' || $fileType['proprietary_short_name'] == 'trados' || DetectProprietaryXliff::getMemoryFileType()) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
/**
* Application misconfiguration.
* upload should not be happened, but if we are here, raise an error.
* @see upload.class.php
* */
$isAConvertedFile = -1;
//stop execution
} elseif (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
return $isAConvertedFile;
}
