public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project?
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid: " . $_tmKey['key'], -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
//check for glossary files and tmx and put them in front of the list
$infoFile = DetectProprietaryXliff::getInfo($fileName);
if (DetectProprietaryXliff::getMemoryFileType()) {
//found TMX, enable language checking routines
if (DetectProprietaryXliff::isTMXFile()) {
$this->checkTMX = 1;
}
//not used at moment but needed if we want to do a poll for status
if (DetectProprietaryXliff::isGlossaryFile()) {
$this->checkGlossary = 1;
}
//prepend in front of the list
array_unshift($sortedFiles, $fileName);
} else {
//append at the end of the list
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = $this->uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
//we are going to access the storage, get model object to manipulate it
$this->fileStorage = new FilesStorage();
$linkFiles = $this->fileStorage->getHashesFromDir($this->uploadDir);
/*
loop through all input files to
1) upload TMX and Glossaries
*/
try {
$this->_pushTMXToMyMemory();
} catch (Exception $e) {
//exit project creation
return false;
}
//TMX Management
/*
loop through all input files to
2)convert, in case, non standard XLIFF files to a format that Matecat understands
Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
*/
foreach ($this->projectStructure['array_files'] as $fileName) {
/*
Conversion Enforce
Checking Extension is no more sufficient, we want check content
$enforcedConversion = true; //( if conversion is enabled )
*/
$isAFileToConvert = $this->isConversionToEnforce($fileName);
//if it's one of the listed formats or conversion is not enabled in first place
if (!$isAFileToConvert) {
/*
filename is already an xliff and it's in upload directory
we have to make a cache package from it to avoid altering the original path
*/
//get file
$filePathName = "{$this->uploadDir}/{$fileName}";
//calculate hash + add the fileName, if i load 3 equal files with the same content
// they will be squashed to the last one
$sha1 = sha1_file($filePathName);
//make a cache package (with work/ only, emtpy orig/)
$this->fileStorage->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
//put reference to cache in upload dir to link cache to session
$this->fileStorage->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken'], $fileName);
//add newly created link to list
$linkFiles['conversionHashes']['sha'][] = $sha1 . "|" . $this->projectStructure['source_language'];
$linkFiles['conversionHashes']['fileName'][$sha1 . "|" . $this->projectStructure['source_language']][] = $fileName;
//when the same sdlxliff is uploaded more than once with different names
$linkFiles['conversionHashes']['sha'] = array_unique($linkFiles['conversionHashes']['sha']);
unset($sha1);
}
}
//now, upload dir contains only hash-links
//we start copying files to "file" dir, inserting metadata in db and extracting segments
foreach ($linkFiles['conversionHashes']['sha'] as $linkFile) {
//converted file is inside cache directory
//get hash from file name inside UUID dir
$hashFile = FilesStorage::basename_fix($linkFile);
$hashFile = explode('|', $hashFile);
//use hash and lang to fetch file from package
$cachedXliffFilePathName = $this->fileStorage->getXliffFromCache($hashFile[0], $hashFile[1]);
//get sha
$sha1_original = $hashFile[0];
//associate the hash to the right file in upload directory
//get original file name, to insert into DB and cp in storage
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
$_originalFileName = $linkFiles['conversionHashes']['fileName'][$linkFile];
unset($hashFile);
if (!file_exists($cachedXliffFilePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = FilesStorage::pathinfo_fix($cachedXliffFilePathName);
if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$yearMonthPath = date_create($this->projectStructure['create_date'])->format('Ymd');
$fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
foreach ($_originalFileName as $originalFileName) {
$mimeType = FilesStorage::pathinfo_fix($originalFileName, PATHINFO_EXTENSION);
$fid = insertFile($this->projectStructure, $originalFileName, $mimeType, $fileDateSha1Path);
//move the file in the right directory from the packages to the file dir
$this->fileStorage->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid, $originalFileName);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments(file_get_contents($cachedXliffFilePathName), $fid);
}
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$originalFileName}.");
$this->fileStorage->deleteHashFromUploadDir($this->uploadDir, $linkFile);
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$originalFileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$originalFileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$originalFileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} elseif ($e->getCode() == -13) {
$this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return null;
// SEVERE EXCEPTION we can not write to disk!! Break project creation
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$originalFileName}. Please try again.");
}
Log::doLog($e->getMessage());
Log::doLog($e->getTraceAsString());
}
}
//end of conversion hash-link loop
try {
$this->_zipFileHandling($linkFiles);
} catch (Exception $e) {
//exit project creation
return false;
}
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
if (!$this->_doCheckForErrors()) {
//exit project creation
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
try {
$rows = $this->dbHandler->fetch_array($query_visible_segments);
} catch (PDOException $e) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: {$e->getMessage()} - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($this->uploadDir);
if (is_dir($this->uploadDir . '_converted')) {
Utils::deleteDir($this->uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->projectStructure['dqf_key']);
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
}
public function doAction()
{
$this->file_name = html_entity_decode($this->file_name, ENT_QUOTES);
$file_path = $this->intDir . DIRECTORY_SEPARATOR . $this->file_name;
if (!file_exists($file_path)) {
$this->result['code'] = -6;
// No Good, Default
$this->result['errors'][] = array("code" => -6, "message" => "Error during upload. Please retry.", 'debug' => FilesStorage::basename_fix($this->file_name));
return -1;
}
//XLIFF Conversion management
//cyclomatic complexity 9999999 ..... but it works, for now.
try {
$fileType = DetectProprietaryXliff::getInfo($file_path);
if (DetectProprietaryXliff::isXliffExtension() || DetectProprietaryXliff::getMemoryFileType()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//if file is not proprietary AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary'] || DetectProprietaryXliff::getMemoryFileType()) {
$this->result['code'] = 1;
// OK for client
//This file has to be linked to cache!
return 0;
//ok don't convert a standard sdlxliff
}
} else {
// if conversion enforce is active
// we force all xliff files but not files produced by
// SDL Studio or by the MateCAT converters, because we
// can handle them
if ($fileType['proprietary_short_name'] == 'matecat_converter' || $fileType['proprietary_short_name'] == 'trados' || DetectProprietaryXliff::getMemoryFileType()) {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
unlink($file_path);
$this->result['code'] = -7;
// No Good, Default
$this->result['errors'][] = array("code" => -7, "message" => 'Matecat Open-Source does not support ' . ucwords($fileType['proprietary_name']) . '. Use MatecatPro.', 'debug' => FilesStorage::basename_fix($this->file_name));
return -1;
} elseif (!$fileType['proprietary']) {
$this->result['code'] = 1;
// OK for client
$this->result['errors'][] = array("code" => 0, "message" => "OK");
return 0;
//ok don't convert a standard sdlxliff
}
}
} catch (Exception $e) {
//try catch not used because of exception no more raised
$this->result['code'] = -8;
// No Good, Default
$this->result['errors'][] = array("code" => -8, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return -1;
}
//compute hash to locate the file in the cache
$sha1 = sha1_file($file_path);
//initialize path variable
$cachedXliffPath = false;
//get storage object
$fs = new FilesStorage();
//TODO: REMOVE SET ENVIRONMENT FOR LEGACY CONVERSION INSTANCES
if (INIT::$LEGACY_CONVERSION !== false) {
INIT::$SAVE_SHASUM_FOR_FILES_LOADED = false;
}
//if already present in database cache get the converted without convert it again
if (INIT::$SAVE_SHASUM_FOR_FILES_LOADED) {
//move the file in the right directory from the packages to the file dir
$cachedXliffPath = $fs->getXliffFromCache($sha1, $this->source_lang);
if (!$cachedXliffPath) {
Log::doLog("Failed to fetch xliff for {$sha1} from disk cache (is file there?)");
}
}
//if invalid or no cached version
if (!isset($cachedXliffPath) or empty($cachedXliffPath)) {
//we have to convert it
// By default, use always the new converters...
$converterVersion = Constants_ConvertersVersions::LATEST;
if ($this->segmentation_rule !== null) {
// ...but new converters don't support custom segmentation rules.
// if $this->segmentation_rule is set use the old ones.
$converterVersion = Constants_ConvertersVersions::LEGACY;
}
//TODO: REMOVE SET ENVIRONMENT FOR LEGACY CONVERSION INSTANCES
if (INIT::$LEGACY_CONVERSION !== false) {
$converterVersion = Constants_ConvertersVersions::LEGACY;
}
$converter = new FileFormatConverter($converterVersion);
if (strpos($this->target_lang, ',') !== false) {
$single_language = explode(',', $this->target_lang);
$single_language = $single_language[0];
} else {
$single_language = $this->target_lang;
}
$convertResult = $converter->convertToSdlxliff($file_path, $this->source_lang, $single_language, false, $this->segmentation_rule);
if ($convertResult['isSuccess'] == 1) {
/* try to back convert the file */
$output_content = array();
$output_content['out_xliff_name'] = $file_path . '.out.sdlxliff';
$output_content['source'] = $this->source_lang;
$output_content['target'] = $single_language;
$output_content['content'] = $convertResult['xliffContent'];
$output_content['filename'] = $this->file_name;
$back_convertResult = $converter->convertToOriginal($output_content);
/* try to back convert the file */
if ($back_convertResult['isSuccess'] == false) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: there is a problem with this file, it cannot be converted back to the original one.";
$this->result['code'] = -110;
$this->result['errors'][] = array("code" => -110, "message" => $convertResult['errorMessage'], 'debug' => FilesStorage::basename_fix($this->file_name));
return false;
}
//store converted content on a temporary path on disk (and off RAM)
$cachedXliffPath = tempnam("/tmp", "MAT_XLF");
file_put_contents($cachedXliffPath, $convertResult['xliffContent']);
unset($convertResult['xliffContent']);
/*
store the converted file in the cache
put a reference in the upload dir to the cache dir, so that from the UUID we can reach the converted file in the cache
(this is independent by the "save xliff for caching" options, since we always end up storing original and xliff on disk)
*/
//save in cache
$res_insert = $fs->makeCachePackage($sha1, $this->source_lang, $file_path, $cachedXliffPath);
if (!$res_insert) {
//custom error message passed directly to javascript client and displayed as is
$convertResult['errorMessage'] = "Error: File upload failed because you have MateCat running in multiple tabs. Please close all other MateCat tabs in your browser.";
$this->result['code'] = -103;
$this->result['errors'][] = array("code" => -103, "message" => $convertResult['errorMessage'], 'debug' => FilesStorage::basename_fix($this->file_name));
unset($cachedXliffPath);
return false;
}
} else {
$file = FilesStorage::pathinfo_fix($this->file_name);
switch ($file['extension']) {
case 'docx':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
break;
case 'doc':
case 'rtf':
$defaultError = "Importing error. Try opening and saving the document with a new name. If this does not work, try converting to DOCX.";
break;
case 'inx':
$defaultError = "Importing Error. Try to commit changes in InDesign before importing.";
break;
case 'idml':
$defaultError = "Importing Error. MateCat does not support this version of InDesign, try converting it to a previous one.";
break;
default:
$defaultError = "Importing error. Try opening and saving the document with a new name.";
break;
}
if (stripos($convertResult['errorMessage'], "failed to create SDLXLIFF.") !== false || stripos($convertResult['errorMessage'], "COM target does not implement IDispatch") !== false) {
$convertResult['errorMessage'] = "Error: failed importing file.";
} elseif (stripos($convertResult['errorMessage'], "Unable to open Excel file - it may be password protected") !== false) {
$convertResult['errorMessage'] = $convertResult['errorMessage'] . " Try to remove protection using the Unprotect Sheet command on Windows Excel.";
} elseif (stripos($convertResult['errorMessage'], "The document contains unaccepted changes") !== false) {
$convertResult['errorMessage'] = "The document contains track changes. Accept all changes before uploading it.";
} elseif (stripos($convertResult['errorMessage'], "Error: Could not find file") !== false || stripos($convertResult['errorMessage'], "tw4winMark") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "Attempted to read or write protected memory") !== false) {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "The document was created in Microsoft Word 97 or earlier")) {
$convertResult['errorMessage'] = $defaultError;
} elseif ($file['extension'] == 'csv' && empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "This CSV file is not eligible to be imported due internal wrong format. Try to convert in TXT using UTF8 encoding";
} elseif (empty($convertResult['errorMessage'])) {
$convertResult['errorMessage'] = "Failed to convert file. Internal error. Please Try again.";
} elseif (stripos($convertResult['errorMessage'], "DocumentFormat.OpenXml.dll") !== false) {
//this error is triggered on DOCX when converter's parser can't decode some regions of the file
$convertResult['errorMessage'] = "Conversion error. Try opening and saving the document with a new name. If this does not work, try converting to DOC.";
} elseif ($file['extension'] == 'idml') {
$convertResult['errorMessage'] = $defaultError;
} elseif (stripos($convertResult['errorMessage'], "Error: The source language of the file") !== false) {
//Error: The source language of the file (English (United States)) is different from the project source language.
//we take the error, is good
} else {
$convertResult['errorMessage'] = "Import error. Try converting it to a compatible file format (e.g. doc > docx, xlsx > xls)";
}
//custom error message passed directly to javascript client and displayed as is
$this->result['code'] = -100;
$this->result['errors'][] = array("code" => -100, "message" => $convertResult['errorMessage'], "debug" => $file['basename']);
}
}
//if everything went well and we've obtained a path toward a valid package (original+xliff), either via cache or conversion
if (isset($cachedXliffPath) and !empty($cachedXliffPath)) {
//FILE Found in cache, destroy the already present shasum for other languages ( if user swapped languages )
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->cookieDir;
$fs->deleteHashFromUploadDir($uploadDir, $sha1 . "|" . $this->source_lang);
//put reference to cache in upload dir to link cache to session
$fs->linkSessionToCache($sha1, $this->source_lang, $this->cookieDir, FilesStorage::basename_fix($file_path));
//a usable package is available, give positive feedback
$this->result['code'] = 1;
}
return 0;
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_customer'] = 'translated_user';
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid.", -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//found TMX, enable language checking routines
$this->checkTMX = 1;
array_unshift($sortedFiles, $fileName);
} else {
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
//fetch cache links, created by converter, from upload directory
$linkFiles = scandir($uploadDir);
//remove dir hardlinks, as uninteresting, as weel as regular files; only hash-links
foreach ($linkFiles as $k => $linkFile) {
if (strpos($linkFile, '.') !== false or strpos($linkFile, '|') === false) {
unset($linkFiles[$k]);
}
}
/*
loop through all input files to
1)upload TMX
2)convert, in case, non standard XLIFF files to a format that Matecat understands
Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
*/
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//load it into MyMemory; we'll check later on how it went
$file = new stdClass();
$file->file_path = "{$uploadDir}/{$fileName}";
$this->tmxServiceWrapper->setName($fileName);
$this->tmxServiceWrapper->setFile(array($file));
try {
$this->tmxServiceWrapper->addTmxInMyMemory();
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//in any case, skip the rest of the loop, go to the next file
continue;
}
/*
Conversion Enforce
Checking Extension is no more sufficient, we want check content if this is an idiom xlf file type, conversion are enforced
$enforcedConversion = true; //( if conversion is enabled )
*/
$isAnXliffToConvert = $this->isConversionToEnforce($fileName);
//we are going to access the storage, get model object to manipulate it
$fs = new FilesStorage();
//if it's one of the listed formats or conversion is not enabled in first place
if (!$isAnXliffToConvert) {
/*
filename is already an xliff and it's in upload directory
we have to make a cache package from it to avoid altering the original path
*/
//get file
$filePathName = "{$uploadDir}/{$fileName}";
//calculate hash + add the fileName, if i load 3 equal files with the same content
// they will be squashed to the last one
$sha1 = sha1(file_get_contents($filePathName) . $filePathName);
//make a cache package (with work/ only, emtpy orig/)
$fs->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
//put reference to cache in upload dir to link cache to session
$fs->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken']);
//add newly created link to list
$linkFiles[] = $sha1 . "|" . $this->projectStructure['source_language'];
unset($sha1);
}
}
//now, upload dir contains only hash-links
//we start copying files to "file" dir, inserting metadata in db and extracting segments
foreach ($linkFiles as $linkFile) {
//converted file is inside cache directory
//get hash from file name inside UUID dir
$hashFile = basename($linkFile);
$hashFile = explode('|', $hashFile);
//use hash and lang to fetch file from package
$xliffFilePathName = $fs->getXliffFromCache($hashFile[0], $hashFile[1]);
//get sha
$sha1_original = $hashFile[0];
//get original file name
$originalFilePathName = $fs->getOriginalFromCache($hashFile[0], $hashFile[1]);
$raw_originalFilePathName = explode(DIRECTORY_SEPARATOR, $originalFilePathName);
$fileName = array_pop($raw_originalFilePathName);
unset($hashFile);
if (!file_exists($xliffFilePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = pathinfo($xliffFilePathName);
if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
$yearMonthPath = date_create()->format("Ymd");
$fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
$fid = insertFile($this->projectStructure, $fileName, $mimeType, $fileDateSha1Path);
//move the file in the right directory from the packages to the file dir
$fs->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments(file_get_contents($xliffFilePathName), $fid);
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
$fs->deleteHashFromUploadDir($uploadDir, $linkFile);
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} elseif ($e->getCode() == -13) {
$this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return null;
// SEVERE EXCEPTION we can not write to disk!! Break project creation
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$fileName}. Please try again.");
}
Log::doLog($e->getMessage());
}
}
//end of hash-link loop
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
/****************/
//loop again through files to check to check for TMX loading
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$this->tmxServiceWrapper->setName($fileName);
$result = array();
//is the TM loaded?
//wait until current TMX is loaded
while (true) {
try {
$result = $this->tmxServiceWrapper->tmxUploadStatus();
if ($result['completed']) {
//"$fileName" has been loaded into MyMemory"
//exit the loop
break;
}
//"waiting for "$fileName" to be loaded into MyMemory"
sleep(3);
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
//exit project creation
return false;
}
}
//once the language is loaded, check if language is compliant (unless something useful has already been found)
if (1 == $this->checkTMX) {
//get localized target languages of TM (in case it's a multilingual TM)
$tmTargets = explode(';', $result['data']['target_lang']);
//indicates if something has been found for current memory
$found = false;
//compare localized target languages array (in case it's a multilingual project) to the TM supplied
//if nothing matches, then the TM supplied can't have matches for this project
//create an empty var and add the source language too
$project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
foreach ($project_languages as $projectTarget) {
if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
$found = true;
break;
}
}
//if this TM matches the project lagpair and something has been found
if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
//the TMX is good to go
$this->checkTMX = 0;
} elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
/*
* This means that the TMX has a srclang as specification in the header. Warn the user.
* Ex:
* <header creationtool="SDL Language Platform"
* creationtoolversion="8.0"
* datatype="rtf"
* segtype="sentence"
* adminlang="DE-DE"
* srclang="DE-DE" />
*/
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
$this->checkTMX = 0;
Log::doLog($this->projectStructure['result']);
}
}
}
}
if (1 == $this->checkTMX) {
//this means that noone of uploaded TMX were usable for this project. Warn the user.
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
Log::doLog($this->projectStructure['result']);
return false;
}
if (!empty($this->projectStructure['result']['errors'])) {
Log::doLog("Project Creation Failed. Sent to Output all errors.");
Log::doLog($this->projectStructure['result']['errors']);
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
$rows = $this->dbHandler->fetch_array($query_visible_segments);
if (!$rows) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($uploadDir);
if (is_dir($uploadDir . '_converted')) {
Utils::deleteDir($uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
}
