public function moveFromCacheToFileDir($dateHashPath, $lang, $idFile, $newFileName = null)
{
list($datePath, $hash) = explode(DIRECTORY_SEPARATOR, $dateHashPath);
$cacheTree = implode(DIRECTORY_SEPARATOR, $this->_composeCachePath($hash));
//destination dir
$fileDir = $this->filesDir . DIRECTORY_SEPARATOR . $datePath . DIRECTORY_SEPARATOR . $idFile;
$cacheDir = $this->cacheDir . DIRECTORY_SEPARATOR . $cacheTree . "|" . $lang . DIRECTORY_SEPARATOR . "package";
$res = true;
//check if doesn't exist
if (!is_dir($fileDir)) {
//make files' directory structure
$res &= mkdir($fileDir, 0755, true);
$res &= mkdir($fileDir . DIRECTORY_SEPARATOR . "package");
$res &= mkdir($fileDir . DIRECTORY_SEPARATOR . "package" . DIRECTORY_SEPARATOR . "orig");
$res &= mkdir($fileDir . DIRECTORY_SEPARATOR . "package" . DIRECTORY_SEPARATOR . "work");
$res &= mkdir($fileDir . DIRECTORY_SEPARATOR . "orig");
$res &= mkdir($fileDir . DIRECTORY_SEPARATOR . "xliff");
}
//make links from cache to files
//BUG: this stuff may not work if FILES and CACHES are on different filesystems
//orig, suppress error because of xliff files have not original one
$origFilePath = $this->getSingleFileInPath($cacheDir . DIRECTORY_SEPARATOR . "orig");
$tmpOrigFileName = $origFilePath;
if (is_file($origFilePath)) {
/*
* Force the new filename if it is provided
*/
if (!empty($newFileName)) {
$tmpOrigFileName = $newFileName;
}
$res &= $this->link($origFilePath, $fileDir . DIRECTORY_SEPARATOR . "orig" . DIRECTORY_SEPARATOR . FilesStorage::basename_fix($tmpOrigFileName));
}
//work
/*
* Force the new filename if it is provided
*/
$convertedFilePath = $this->getSingleFileInPath($cacheDir . DIRECTORY_SEPARATOR . "work");
$tmpConvertedFilePath = $convertedFilePath;
if (!empty($newFileName)) {
if (!DetectProprietaryXliff::isXliffExtension(FilesStorage::pathinfo_fix($newFileName))) {
$convertedExtension = FilesStorage::pathinfo_fix($convertedFilePath, PATHINFO_EXTENSION);
$tmpConvertedFilePath = $newFileName . "." . $convertedExtension;
}
}
$res &= $this->link($convertedFilePath, $fileDir . DIRECTORY_SEPARATOR . "xliff" . DIRECTORY_SEPARATOR . FilesStorage::basename_fix($tmpConvertedFilePath));
if (!$res) {
throw new UnexpectedValueException('Internal Error: Failed to create/copy the file on disk from cache.', -13);
}
}
public function doAction()
{
//get job language and data
//Fixed Bug: need a specific job, because we need The target Language
//Removed from within the foreach cycle, the job is always the same....
$jobData = $this->jobInfo = getJobData($this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($jobData) || !$pCheck->grantJobAccessByJobData($jobData, $this->password)) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
//get storage object
$fs = new FilesStorage();
$files_job = $fs->getFilesForJob($this->id_job, $this->id_file);
$nonew = 0;
$output_content = array();
/*
the procedure:
1)original xliff file is read directly from disk; a file handler is obtained
2)the file is read chunk by chunk by a stream parser: for each trans-unit that is encountered, target is replaced (or added) with the corresponding translation obtained from the DB
3)the parsed portion of xliff in the buffer is flushed on temporary file
4)the temporary file is sent to the converter and an original file is obtained
5)the temporary file is deleted
*/
// This array will contain all the files of $files_job split by
// converter version.
$files_job_by_converter_version = array();
// Detect the converter's version to use for each file, then store
// file info accordingly.
foreach ($files_job as $file) {
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
$files_job_by_converter_version[$fileType['converter_version']][] = $file;
}
// Process files according to the converters' versions, one version
// at a time
foreach ($files_job_by_converter_version as $converter_version => $files_job) {
//file array is chuncked. Each chunk will be used for a parallel conversion request.
$files_job = array_chunk($files_job, self::FILES_CHUNK_SIZE);
foreach ($files_job as $chunk) {
$converter = new FileFormatConverter($converter_version);
$files_to_be_converted = array();
foreach ($chunk as $file) {
$mime_type = $file['mime_type'];
$fileID = $file['id_file'];
$current_filename = $file['filename'];
//get path for the output file converted to know it's right extension
$_fileName = explode(DIRECTORY_SEPARATOR, $file['xliffFilePath']);
$outputPath = INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/' . $fileID . '/' . uniqid('', true) . "_.out." . array_pop($_fileName);
//make dir if doesn't exist
if (!file_exists(dirname($outputPath))) {
Log::doLog('Create Directory ' . escapeshellarg(dirname($outputPath)) . '');
mkdir(dirname($outputPath), 0775, true);
}
$data = getSegmentsDownload($this->id_job, $this->password, $fileID, $nonew);
//prepare regexp for nest step
$regexpEntity = '/&#x(0[0-8BCEF]|1[0-9A-F]|7F);/u';
$regexpAscii = '/([\\x{00}-\\x{1F}\\x{7F}]{1})/u';
foreach ($data as $i => $k) {
//create a secondary indexing mechanism on segments' array; this will be useful
//prepend a string so non-trans unit id ( ex: numerical ) are not overwritten
$data['matecat|' . $k['internal_id']][] = $i;
//FIXME: temporary patch
$data[$i]['translation'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['translation']);
$data[$i]['segment'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['segment']);
//remove binary chars in some xliff files
$sanitized_src = preg_replace($regexpAscii, '', $data[$i]['segment']);
$sanitized_trg = preg_replace($regexpAscii, '', $data[$i]['translation']);
//clean invalid xml entities ( charactes with ascii < 32 and different from 0A, 0D and 09
$sanitized_src = preg_replace($regexpEntity, '', $sanitized_src);
$sanitized_trg = preg_replace($regexpEntity, '', $sanitized_trg);
if ($sanitized_src != null) {
$data[$i]['segment'] = $sanitized_src;
}
if ($sanitized_trg != null) {
$data[$i]['translation'] = $sanitized_trg;
}
}
//instatiate parser
$xsp = new SdlXliffSAXTranslationReplacer($file['xliffFilePath'], $data, Langs_Languages::getInstance()->getLangRegionCode($jobData['target']), $outputPath);
if ($this->download_type == 'omegat') {
$xsp->setSourceInTarget(true);
}
//run parsing
Log::doLog("work on " . $fileID . " " . $current_filename);
$xsp->replaceTranslation();
//free memory
unset($xsp);
unset($data);
$output_content[$fileID]['document_content'] = file_get_contents($outputPath);
$output_content[$fileID]['output_filename'] = $current_filename;
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
if ($this->forceXliff) {
//clean the output filename by removing
// the unique hash identifier 55e5739b467109.05614837_.out.Test_English.doc.sdlxliff
$output_content[$fileID]['output_filename'] = preg_replace('#[0-9a-f]+\\.[0-9_]+\\.out\\.#i', '', FilesStorage::basename_fix($outputPath));
if ($fileType['proprietary_short_name'] === 'matecat_converter') {
// Set the XLIFF extension to .xlf
// Internally, MateCat continues using .sdlxliff as default
// extension for the XLIFF behind the projects.
// Changing this behavior requires a huge refactoring that
// it's scheduled for future versions.
// We quickly fixed the behaviour from the user standpoint
// using the following line of code, that changes the XLIFF's
// extension just a moment before it is downloaded by the user.
$output_content[$fileID]['output_filename'] = preg_replace("|\\.sdlxliff\$|i", ".xlf", $output_content[$fileID]['output_filename']);
}
}
/**
* Conversion Enforce
*/
$convertBackToOriginal = true;
//if it is a not converted file ( sdlxliff ) we have originalFile equals to xliffFile (it has just been copied)
$file['original_file'] = file_get_contents($file['originalFilePath']);
// When the 'proprietary' flag is set to false, the xliff
// is not passed to any converter, because is handled
// directly inside MateCAT.
$xliffWasNotConverted = $fileType['proprietary'] === false;
if (!INIT::$CONVERSION_ENABLED || ($file['originalFilePath'] == $file['xliffFilePath'] and $xliffWasNotConverted) or $this->forceXliff) {
$convertBackToOriginal = false;
Log::doLog("SDLXLIFF: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
} else {
//TODO: dos2unix ??? why??
//force unix type files
Log::doLog("NO SDLXLIFF, Conversion enforced: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
}
if ($convertBackToOriginal) {
$output_content[$fileID]['out_xliff_name'] = $outputPath;
$output_content[$fileID]['source'] = $jobData['source'];
$output_content[$fileID]['target'] = $jobData['target'];
$files_to_be_converted[$fileID] = $output_content[$fileID];
} elseif ($this->forceXliff) {
$this->cleanFilePath($output_content[$fileID]['document_content']);
}
}
$convertResult = $converter->multiConvertToOriginal($files_to_be_converted, $chosen_machine = false);
foreach (array_keys($files_to_be_converted) as $fileID) {
$output_content[$fileID]['document_content'] = $this->ifGlobalSightXliffRemoveTargetMarks($convertResult[$fileID]['document_content'], $files_to_be_converted[$fileID]['output_filename']);
//in case of .strings, they are required to be in UTF-16
//get extension to perform file detection
$extension = FilesStorage::pathinfo_fix($output_content[$fileID]['output_filename'], PATHINFO_EXTENSION);
if (strtoupper($extension) == 'STRINGS') {
//use this function to convert stuff
$encodingConvertedFile = CatUtils::convertEncoding('UTF-16', $output_content[$fileID]['document_content']);
//strip previously added BOM
$encodingConvertedFile[1] = $converter->stripBOM($encodingConvertedFile[1], 16);
//store new content
$output_content[$fileID]['document_content'] = $encodingConvertedFile[1];
//trash temporary data
unset($encodingConvertedFile);
}
}
unset($convertResult);
}
}
foreach ($output_content as $idFile => $fileInformations) {
$zipPathInfo = ZipArchiveExtended::zipPathInfo($output_content[$idFile]['output_filename']);
if (is_array($zipPathInfo)) {
$output_content[$idFile]['zipfilename'] = $zipPathInfo['zipfilename'];
$output_content[$idFile]['zipinternalPath'] = $zipPathInfo['dirname'];
$output_content[$idFile]['output_filename'] = $zipPathInfo['basename'];
}
}
//set the file Name
$pathinfo = FilesStorage::pathinfo_fix($this->fname);
$this->_filename = $pathinfo['filename'] . "_" . $jobData['target'] . "." . $pathinfo['extension'];
//qui prodest to check download type?
if ($this->download_type == 'omegat') {
$this->_filename .= ".zip";
$tmsService = new TMSService();
$tmsService->setOutputType('tm');
/**
* @var $tmFile SplTempFileObject
*/
$tmFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tmsService->setOutputType('mt');
/**
* @var $mtFile SplTempFileObject
*/
$mtFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tm_id = uniqid('tm');
$mt_id = uniqid('mt');
$output_content[$tm_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_TM . tmx");
foreach ($tmFile as $lineNumber => $content) {
$output_content[$tm_id]['document_content'] .= $content;
}
$output_content[$mt_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_MT . tmx");
foreach ($mtFile as $lineNumber => $content) {
$output_content[$mt_id]['document_content'] .= $content;
}
$this->createOmegaTZip($output_content, $jobData['source'], $jobData['target']);
//add zip archive content here;
} else {
try {
$output_content = $this->getOutputContentsWithZipFiles($output_content);
if (count($output_content) > 1) {
//cast $output_content elements to ZipContentObject
foreach ($output_content as $key => $__output_content_elem) {
$output_content[$key] = new ZipContentObject($__output_content_elem);
}
if ($pathinfo['extension'] != 'zip') {
if ($this->forceXliff) {
$this->_filename = $this->id_job . ".zip";
} else {
$this->_filename = $pathinfo['basename'] . ".zip";
}
}
$this->content = self::composeZip($output_content);
//add zip archive content here;
} else {
//always an array with 1 element, pop it, Ex: array( array() )
$output_content = array_pop($output_content);
$this->setContent($output_content);
}
} catch (Exception $e) {
$msg = "\n\n Error retrieving file content, Conversion failed??? \n\n Error: {$e->getMessage()} \n\n" . var_export($e->getTraceAsString(), true);
$msg .= "\n\n Request: " . var_export($_REQUEST, true);
Log::$fileName = 'fatal_errors.txt';
Log::doLog($msg);
Utils::sendErrMailReport($msg);
$this->unlockToken(array("code" => -110, "message" => "Download failed. Please contact " . INIT::$SUPPORT_MAIL));
throw $e;
// avoid sent Headers and empty file content with finalize method
}
}
try {
Utils::deleteDir(INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/');
} catch (Exception $e) {
Log::doLog('Failed to delete dir:' . $e->getMessage());
}
}
public function doAction()
{
$this->file_name = html_entity_decode($this->file_name, ENT_QUOTES);
$file_path = $this->intDir . DIRECTORY_SEPARATOR . $this->file_name;
if (!file_exists($file_path)) {
$this->result['code'] = -6;
// No Good, Default
$this->result['errors'][] = array("code" => -6, "message" => "Error during upload. Please retry.", 'debug' => FilesStorage::basename_fix($this->file_name));
return -1;
}
//XLIFF Conversion management
//cyclomatic complexity 9999999 ..... but it works, for now.
try {
$fileType = DetectProprietaryXliff::getInfo($file_path);
if (DetectProprietaryXliff::isXliffExtension() || DetectProprietaryXliff::getMemoryFileType()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//if file is not proprietary AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary'] || DetectProprietaryXliff::getMemoryFileType()) {
$this->result['code'] = 1;
// OK for client
//This file has to be linked to cache!
return 0;
//ok don't convert a standard sdlxliff
}
} else {
// if conversion enforce is active
// we force all xliff files but not files produced by
// SDL Studio or by the MateCAT converters, because we
// can handle them
if ($fileType['proprietary_short_name'] == 'matecat_converter' || $fileType['proprietary_short_name'] == 'trados' || DetectProprietaryXliff::getMemoryFileType()) {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
unlink($file_path);
$this->result['code'] = -7;
// No Good, Default
$this->result['errors'][] = array("code" => -7, "message" => 'Matecat Open-Source does not support ' . ucwords($fileType['proprietary_name']) . '. Use MatecatPro.', 'debug' => FilesStorage::basename_fix($this->file_name));
return -1;
} elseif (!$fileType['proprietary']) {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} catch (Exception $e) {
//try catch not used because of exception no more raised
$this->result['code'] = -8;
// No Good, Default
$this->result['errors'][] = array("code" => -8, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return -1;
}
//compute hash to locate the file in the cache
$sha1 = sha1_file($file_path);
//initialize path variable
$cachedXliffPath = false;
//get storage object
$fs = new FilesStorage();
//TODO: REMOVE SET ENVIRONMENT FOR LEGACY CONVERSION INSTANCES
if (INIT::$LEGACY_CONVERSION !== false) {
INIT::$SAVE_SHASUM_FOR_FILES_LOADED = false;
}
//if already present in database cache get the converted without convert it again
if (INIT::$SAVE_SHASUM_FOR_FILES_LOADED) {
//move the file in the right directory from the packages to the file dir
$cachedXliffPath = $fs->getXliffFromCache($sha1, $this->source_lang);
if (!$cachedXliffPath) {
Log::doLog("Failed to fetch xliff for {$sha1} from disk cache (is file there?)");
}
}
//if invalid or no cached version
if (!isset($cachedXliffPath) or empty($cachedXliffPath)) {
//we have to convert it
// By default, use always the new converters...
$converterVersion = Constants_ConvertersVersions::LATEST;
if ($this->segmentation_rule !== null) {
// ...but new converters don't support custom segmentation rules.
// if $this->segmentation_rule is set use the old ones.
$converterVersion = Constants_ConvertersVersions::LEGACY;
}
//TODO: REMOVE SET ENVIRONMENT FOR LEGACY CONVERSION INSTANCES
if (INIT::$LEGACY_CONVERSION !== false) {
$converterVersion = Constants_ConvertersVersions::LEGACY;
}
$converter = new FileFormatConverter($converterVersion);
if (strpos($this->target_lang, ',') !== false) {
$single_language = explode(',', $this->target_lang);
$single_language = $single_language[0];
} else {
$single_language = $this->target_lang;
}
$convertResult = $converter->convertToSdlxliff($file_path, $this->source_lang, $single_language, false, $this->segmentation_rule);
if ($convertResult['isSuccess'] == 1) {
/* try to back convert the file */
$output_content = array();
$output_content['out_xliff_name'] = $file_path . '.out.sdlxliff';
$output_content['source'] = $this->source_lang;
$output_content['target'] = $single_language;
$output_content['content'] = $convertResult['xliffContent'];
$output_content['filename'] = $this->file_name;
$back_convertResult = $converter->convertToOriginal($output_content);
/* try to back convert the file */
if ($back_convertResult['isSuccess'] == false) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: there is a problem with this file, it cannot be converted back to the original one.";
$this->result['code'] = -110;
$this->result['errors'][] = array("code" => -110, "message" => $convertResult['errorMessage'], 'debug' => FilesStorage::basename_fix($this->file_name));
return false;
}
//store converted content on a temporary path on disk (and off RAM)
$cachedXliffPath = tempnam("/tmp", "MAT_XLF");
file_put_contents($cachedXliffPath, $convertResult['xliffContent']);
unset($convertResult['xliffContent']);
/*
store the converted file in the cache
put a reference in the upload dir to the cache dir, so that from the UUID we can reach the converted file in the cache
(this is independent by the "save xliff for caching" options, since we always end up storing original and xliff on disk)
*/
//save in cache
$res_insert = $fs->makeCachePackage($sha1, $this->source_lang, $file_path, $cachedXliffPath);
if (!$res_insert) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: File upload failed because you have MateCat running in multiple tabs. Please close all other MateCat tabs in your browser.";
$this->result['code'] = -103;
$this->result['errors'][] = array("code" => -103, "message" => $convertResult['errorMessage'], 'debug' => FilesStorage::basename_fix($this->file_name));
unset($cachedXliffPath);
return false;
}
} else {
$file = FilesStorage::pathinfo_fix($this->file_name);
switch ($file['extension']) {
case 'docx':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
break;
case 'doc':
case 'rtf':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOCX.";
break;
case 'inx':
$defaultError = "Importing Error. Try to commit changes in InDesign before importing.";
break;
case 'idml':
$defaultError = "Importing Error. MateCat does not support this version of InDesign, try converting it to a previous one.";
break;
default:
$defaultError = "Importing error. Try opening and saving the document with a new name.";
break;
}
if (stripos($convertResult['errorMessage'], "failed to create SDLXLIFF.") !== false || stripos($convertResult['errorMessage'], "COM target does not implement IDispatch") !== false) {
$convertResult['errorMessage'] = "Error: failed importing file.";
} elseif (stripos($convertResult['errorMessage'], "Unable to open Excel file - it may be password protected") !== false) {
$convertResult['errorMessage'] = $convertResult['errorMessage'] . " Try to remove protection using the Unprotect Sheet command on Windows Excel.";
} elseif (stripos($convertResult['errorMessage'], "The document contains unaccepted changes") !== false) {
$convertResult['errorMessage'] = "The document contains track changes. Accept all changes before uploading it.";
} elseif (stripos($convertResult['errorMessage'], "Error: Could not find file") !== false || stripos($convertResult['errorMessage'], "tw4winMark") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "Attempted to read or write protected memory") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "The document was created in Microsoft Word 97 or earlier")) {
$convertResult['errorMessage'] = $defaultError;
} elseif ($file['extension'] == 'csv' && empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "This CSV file is not eligible to be imported due internal wrong format. Try to convert in TXT using UTF8 encoding";
} elseif (empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "Failed to convert file. Internal error. Please Try again.";
} elseif (stripos($convertResult['errorMessage'], "DocumentFormat.OpenXml.dll") !== false) {
//this error is triggered on DOCX when converter's parser can't decode some regions of the file
$convertResult['errorMessage'] = "Conversion error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
} elseif ($file['extension'] == 'idml') {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "Error: The source language of the file") !== false) {
//Error: The source language of the file (English (United States)) is different from the project source language.
//we take the error, is good
} else {
$convertResult['errorMessage'] = "Import error. Try converting it to a compatible file format (e.g. doc > docx, xlsx > xls)";
}
//custom error message passed directly to javascript client and displayed as is
$this->result['code'] = -100;
$this->result['errors'][] = array("code" => -100, "message" => $convertResult['errorMessage'], "debug" => $file['basename']);
}
}
//if everything went well and we've obtained a path toward a valid package (original+xliff), either via cache or conversion
if (isset($cachedXliffPath) and !empty($cachedXliffPath)) {
//FILE Found in cache, destroy the already present shasum for other languages ( if user swapped languages )
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->cookieDir;
$fs->deleteHashFromUploadDir($uploadDir, $sha1 . "|" . $this->source_lang);
//put reference to cache in upload dir to link cache to session
$fs->linkSessionToCache($sha1, $this->source_lang, $this->cookieDir, FilesStorage::basename_fix($file_path));
//a usable package is available, give positive feedback
$this->result['code'] = 1;
}
return 0;
}
/**
* Creates a message from an HTML string, making modifications for inline images and backgrounds
* and creates a plain-text version by converting the HTML
* Overwrites any existing values in $this->Body and $this->AltBody
* @access public
* @param string $message HTML message string
* @param string $basedir baseline directory for path
* @param bool $advanced Whether to use the advanced HTML to text converter
* @return string $message
*/
public function MsgHTML($message, $basedir = '', $advanced = false)
{
preg_match_all("/(src|background)=[\"'](.*)[\"']/Ui", $message, $images);
if (isset($images[2])) {
foreach ($images[2] as $i => $url) {
// do not change urls for absolute images (thanks to corvuscorax)
if (!preg_match('#^[A-z]+://#', $url)) {
$filename = FilesStorage::basename_fix($url);
$directory = dirname($url);
if ($directory == '.') {
$directory = '';
}
$cid = md5($url) . '@phpmailer.0';
//RFC2392 S 2
if (strlen($basedir) > 1 && substr($basedir, -1) != '/') {
$basedir .= '/';
}
if (strlen($directory) > 1 && substr($directory, -1) != '/') {
$directory .= '/';
}
if ($this->AddEmbeddedImage($basedir . $directory . $filename, $cid, $filename, 'base64', self::_mime_types(self::mb_pathinfo($filename, PATHINFO_EXTENSION)))) {
$message = preg_replace("/" . $images[1][$i] . "=[\"']" . preg_quote($url, '/') . "[\"']/Ui", $images[1][$i] . "=\"cid:" . $cid . "\"", $message);
}
}
}
}
$this->IsHTML(true);
if (empty($this->AltBody)) {
$this->AltBody = 'To view this email message, open it in a program that understands HTML!' . "\n\n";
}
//Convert all message body line breaks to CRLF, makes quoted-printable encoding work much better
$this->Body = $this->NormalizeBreaks($message);
$this->AltBody = $this->NormalizeBreaks($this->html2text($message, $advanced));
return $this->Body;
}
/**
* Avoid race conditions by javascript multiple calls
*
* @param $file_path
*/
private function deleteSha($file_path)
{
$sha1 = sha1_file($file_path);
if (!$sha1) {
return;
}
//can be present more than one file with the same sha
//so in the sha1 file there could be more than one row
$file_sha = glob($this->options['upload_dir'] . $sha1 . "*");
//delete sha1 also
$fp = fopen($file_sha[0], "r+");
// no file found
if (!$fp) {
return;
}
$i = 0;
while (!flock($fp, LOCK_EX | LOCK_NB)) {
// acquire an exclusive lock
$i++;
if ($i == 40) {
return;
}
//exit the loop after 2 seconds, can not acquire the lock
usleep(50000);
continue;
}
$file_content = fread($fp, filesize($file_sha[0]));
$file_content_array = explode("\n", $file_content);
//remove the last line ( is an empty string )
array_pop($file_content_array);
$fileName = FilesStorage::basename_fix($file_path);
$key = array_search($fileName, $file_content_array);
unset($file_content_array[$key]);
if (!empty($file_content_array)) {
fseek($fp, 0);
//rewind
ftruncate($fp, 0);
//truncate to zero bytes length
fwrite($fp, implode("\n", $file_content_array) . "\n");
fflush($fp);
flock($fp, LOCK_UN);
// release the lock
fclose($fp);
} else {
flock($fp, LOCK_UN);
// release the lock
fclose($fp);
@unlink(@$file_sha[0]);
}
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project?
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid: " . $_tmKey['key'], -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
//check for glossary files and tmx and put them in front of the list
$infoFile = DetectProprietaryXliff::getInfo($fileName);
if (DetectProprietaryXliff::getMemoryFileType()) {
//found TMX, enable language checking routines
if (DetectProprietaryXliff::isTMXFile()) {
$this->checkTMX = 1;
}
//not used at moment but needed if we want to do a poll for status
if (DetectProprietaryXliff::isGlossaryFile()) {
$this->checkGlossary = 1;
}
//prepend in front of the list
array_unshift($sortedFiles, $fileName);
} else {
//append at the end of the list
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = $this->uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
//we are going to access the storage, get model object to manipulate it
$this->fileStorage = new FilesStorage();
$linkFiles = $this->fileStorage->getHashesFromDir($this->uploadDir);
/*
loop through all input files to
1) upload TMX and Glossaries
*/
try {
$this->_pushTMXToMyMemory();
} catch (Exception $e) {
//exit project creation
return false;
}
//TMX Management
/*
loop through all input files to
2)convert, in case, non standard XLIFF files to a format that Matecat understands
Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
*/
foreach ($this->projectStructure['array_files'] as $fileName) {
/*
Conversion Enforce
Checking Extension is no more sufficient, we want check content
$enforcedConversion = true; //( if conversion is enabled )
*/
$isAFileToConvert = $this->isConversionToEnforce($fileName);
//if it's one of the listed formats or conversion is not enabled in first place
if (!$isAFileToConvert) {
/*
filename is already an xliff and it's in upload directory
we have to make a cache package from it to avoid altering the original path
*/
//get file
$filePathName = "{$this->uploadDir}/{$fileName}";
//calculate hash + add the fileName, if i load 3 equal files with the same content
// they will be squashed to the last one
$sha1 = sha1_file($filePathName);
//make a cache package (with work/ only, emtpy orig/)
$this->fileStorage->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
//put reference to cache in upload dir to link cache to session
$this->fileStorage->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken'], $fileName);
//add newly created link to list
$linkFiles['conversionHashes']['sha'][] = $sha1 . "|" . $this->projectStructure['source_language'];
$linkFiles['conversionHashes']['fileName'][$sha1 . "|" . $this->projectStructure['source_language']][] = $fileName;
//when the same sdlxliff is uploaded more than once with different names
$linkFiles['conversionHashes']['sha'] = array_unique($linkFiles['conversionHashes']['sha']);
unset($sha1);
}
}
//now, upload dir contains only hash-links
//we start copying files to "file" dir, inserting metadata in db and extracting segments
foreach ($linkFiles['conversionHashes']['sha'] as $linkFile) {
//converted file is inside cache directory
//get hash from file name inside UUID dir
$hashFile = FilesStorage::basename_fix($linkFile);
$hashFile = explode('|', $hashFile);
//use hash and lang to fetch file from package
$cachedXliffFilePathName = $this->fileStorage->getXliffFromCache($hashFile[0], $hashFile[1]);
//get sha
$sha1_original = $hashFile[0];
//associate the hash to the right file in upload directory
//get original file name, to insert into DB and cp in storage
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
$_originalFileName = $linkFiles['conversionHashes']['fileName'][$linkFile];
unset($hashFile);
if (!file_exists($cachedXliffFilePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = FilesStorage::pathinfo_fix($cachedXliffFilePathName);
if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$yearMonthPath = date_create($this->projectStructure['create_date'])->format('Ymd');
$fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
foreach ($_originalFileName as $originalFileName) {
$mimeType = FilesStorage::pathinfo_fix($originalFileName, PATHINFO_EXTENSION);
$fid = insertFile($this->projectStructure, $originalFileName, $mimeType, $fileDateSha1Path);
//move the file in the right directory from the packages to the file dir
$this->fileStorage->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid, $originalFileName);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments(file_get_contents($cachedXliffFilePathName), $fid);
}
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$originalFileName}.");
$this->fileStorage->deleteHashFromUploadDir($this->uploadDir, $linkFile);
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$originalFileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$originalFileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$originalFileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} elseif ($e->getCode() == -13) {
$this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return null;
// SEVERE EXCEPTION we can not write to disk!! Break project creation
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$originalFileName}. Please try again.");
}
Log::doLog($e->getMessage());
Log::doLog($e->getTraceAsString());
}
}
//end of conversion hash-link loop
try {
$this->_zipFileHandling($linkFiles);
} catch (Exception $e) {
//exit project creation
return false;
}
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
if (!$this->_doCheckForErrors()) {
//exit project creation
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
try {
$rows = $this->dbHandler->fetch_array($query_visible_segments);
} catch (PDOException $e) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: {$e->getMessage()} - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($this->uploadDir);
if (is_dir($this->uploadDir . '_converted')) {
Utils::deleteDir($this->uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->projectStructure['dqf_key']);
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
}
