/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return mixed
*/
public function doAction()
{
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->DQF_PMANAGER_KEY);
$this->result['data'] = 'OK';
} catch (Exception $e) {
$this->result['data'] = 'KO';
$this->result['errors'][] = array('code' => -1, 'message' => $e->getMessage());
}
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_customer'] = 'translated_user';
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid.", -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//found TMX, enable language checking routines
$this->checkTMX = 1;
array_unshift($sortedFiles, $fileName);
} else {
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$file = new stdClass();
$file->file_path = "{$uploadDir}/{$fileName}";
$this->tmxServiceWrapper->setName($fileName);
$this->tmxServiceWrapper->setFile(array($file));
try {
$this->tmxServiceWrapper->addTmxInMyMemory();
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//in any case, skip the rest of the loop, go to the next file
continue;
}
/**
* Conversion Enforce
*
* we have to know if a file can be found in _converted directory
*
* Check Extension no more sufficient, we want check content
* if this is an idiom xlf file type, conversion are enforced
* $enforcedConversion = true; //( if conversion is enabled )
*/
$isAConvertedFile = true;
try {
$fileType = DetectProprietaryXliff::getInfo(INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'] . DIRECTORY_SEPARATOR . $fileName);
if (DetectProprietaryXliff::isXliffExtension()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//ONLY IDIOM is forced to be converted
//if file is not proprietary like idiom AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
} else {
//if conversion enforce is active
//we force all xliff files but not files produced by SDL Studio because we can handle them
if ($fileType['proprietary_short_name'] == 'trados') {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
/**
* Application misconfiguration.
* upload should not be happened, but if we are here, raise an error.
* @see upload.class.php
* */
$this->projectStructure['result']['errors'][] = array("code" => -8, "message" => "Proprietary xlf format detected. Not able to import this XLIFF file. ({$fileName})");
setcookie("upload_session", "", time() - 10000);
return -1;
//stop execution
} elseif (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
$mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
$original_content = "";
/*
if it's not one of the listed formats (or it is, but you had to convert it anyway),
and conversion is enabled in first place
*/
if ($isAConvertedFile) {
//converted file is inside "_converted" directory
$fileDir = $uploadDir . '_converted';
$original_content = file_get_contents("{$uploadDir}/{$fileName}");
$sha1_original = sha1($original_content);
$original_content = gzdeflate($original_content, 5);
//file name is a xliff converted like: 'a_word_document.doc.sdlxliff'
$real_fileName = $fileName . '.sdlxliff';
} else {
//filename is already an xliff and it is in a canonical normal directory
$sha1_original = "";
$fileDir = $uploadDir;
$real_fileName = $fileName;
}
$filePathName = $fileDir . DIRECTORY_SEPARATOR . $real_fileName;
if (!file_exists($filePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = pathinfo($filePathName);
if ($info['extension'] == 'xliff' || $info['extension'] == 'sdlxliff' || $info['extension'] == 'xlf') {
$contents = file_get_contents($filePathName);
} else {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$fid = insertFile($this->projectStructure, $fileName, $mimeType, $contents, $sha1_original, $original_content);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments($contents, $fid);
unset($contents);
//free memory
//Log::doLog( $this->projectStructure['segments'] );
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File is Too large. ( {$fileName} )");
}
Log::doLog($e->getMessage());
}
//exit;
}
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
/****************/
//loop again through files to check to check for TMX loading
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$this->tmxServiceWrapper->setName($fileName);
$result = array();
//is the TM loaded?
//wait until current TMX is loaded
while (true) {
try {
$result = $this->tmxServiceWrapper->tmxUploadStatus();
if ($result['completed']) {
//"$fileName" has been loaded into MyMemory"
//exit the loop
break;
}
//"waiting for "$fileName" to be loaded into MyMemory"
sleep(3);
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
//exit project creation
return false;
}
}
//once the language is loaded, check if language is compliant (unless something useful has already been found)
if (1 == $this->checkTMX) {
//get localized target languages of TM (in case it's a multilingual TM)
$tmTargets = explode(';', $result['data']['target_lang']);
//indicates if something has been found for current memory
$found = false;
//compare localized target languages array (in case it's a multilingual project) to the TM supplied
//if nothing matches, then the TM supplied can't have matches for this project
//create an empty var and add the source language too
$project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
foreach ($project_languages as $projectTarget) {
if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
$found = true;
break;
}
}
//if this TM matches the project lagpair and something has been found
if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
//the TMX is good to go
$this->checkTMX = 0;
} elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
/*
* This means that the TMX has a srclang as specification in the header. Warn the user.
* Ex:
* <header creationtool="SDL Language Platform"
* creationtoolversion="8.0"
* datatype="rtf"
* segtype="sentence"
* adminlang="DE-DE"
* srclang="DE-DE" />
*/
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
$this->checkTMX = 0;
Log::doLog($this->projectStructure['result']);
}
}
}
}
if (1 == $this->checkTMX) {
//this means that noone of uploaded TMX were usable for this project. Warn the user.
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
Log::doLog($this->projectStructure['result']);
return false;
}
if (!empty($this->projectStructure['result']['errors'])) {
Log::doLog("Project Creation Failed. Sent to Output all errors.");
Log::doLog($this->projectStructure['result']['errors']);
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n FROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
$rows = $this->dbHandler->fetch_array($query_visible_segments);
if (!$rows) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($uploadDir);
if (is_dir($uploadDir . '_converted')) {
Utils::deleteDir($uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
/**
* TODO: remove after queue implementation
*/
// if( 0 && $project_summary[0]['project_segments'] > 50000 ) {
// $this->projectStructure[ 'status' ] = Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
//
// $msg = "
// WARNING: a project with more than 50.000 segments was created. ( " . $project_summary[0]['project_segments'] . " )\n" .
// var_export( $this->projectStructure[ 'result' ], true ) . "\n\n" .
// " " .
// var_export( $project_summary[0] , true ) . "\n";
//
// Utils::sendErrMailReport( $msg, "Alert: Project Creation Abort. - " );
//
// }
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
// var_dump($this->projectStructure);
// exit;
}
public function doAction()
{
try {
$this->_checkData();
} catch (Exception $e) {
if ($e->getCode() == -1) {
Utils::sendErrMailReport($e->getMessage());
}
Log::doLog($e->getMessage());
return $e->getCode();
}
//check tag mismatch
//get original source segment, first
$segment = getSegment($this->id_segment);
//compare segment-translation and get results
$check = new QA($segment['segment'], $this->translation);
$check->performConsistencyCheck();
if ($check->thereAreWarnings()) {
$err_json = $check->getWarningsJSON();
$translation = $this->translation;
} else {
$err_json = '';
$translation = $check->getTrgNormalized();
}
/*
* begin stats counter
*
* It works good with default InnoDB Isolation level
*
* REPEATABLE-READ offering a row level lock for this id_segment
*
*/
$db = Database::obtain();
$db->begin();
$old_translation = getCurrentTranslation($this->id_job, $this->id_segment);
// $old_version = ( empty( $old_translation['translation_date'] ) ? '0' : date_create( $old_translation['translation_date'] )->getTimestamp() );
// if( $this->client_target_version != $old_version ){
// $this->result[ 'errors' ][ ] = array( "code" => -102, "message" => "Translation version mismatch" );
// $db->rollback();
// return false;
// }
//if volume analysis is not enabled and no translation rows exists
//create the row
if (!INIT::$VOLUME_ANALYSIS_ENABLED && empty($old_translation['status'])) {
$_Translation = array();
$_Translation['id_segment'] = (int) $this->id_segment;
$_Translation['id_job'] = (int) $this->id_job;
$_Translation['status'] = Constants_TranslationStatus::STATUS_NEW;
$_Translation['segment_hash'] = $segment['segment_hash'];
$_Translation['translation'] = $segment['segment'];
$_Translation['standard_word_count'] = $segment['raw_word_count'];
$_Translation['serialized_errors_list'] = '';
$_Translation['suggestion_position'] = 0;
$_Translation['warning'] = false;
$_Translation['translation_date'] = date("Y-m-d H:i:s");
$res = addTranslation($_Translation);
if ($res < 0) {
$this->result['errors'][] = array("code" => -101, "message" => "database errors");
$db->rollback();
return $res;
}
/*
* begin stats counter
*
*/
$old_translation = $_Translation;
}
$_Translation = array();
$_Translation['id_segment'] = $this->id_segment;
$_Translation['id_job'] = $this->id_job;
$_Translation['status'] = $this->status;
$_Translation['time_to_edit'] = $this->time_to_edit;
$_Translation['translation'] = preg_replace('/[ \\t\\n\\r\\0\\x0A\\xA0]+$/u', '', $translation);
$_Translation['serialized_errors_list'] = $err_json;
$_Translation['suggestion_position'] = $this->chosen_suggestion_index;
$_Translation['warning'] = $check->thereAreWarnings();
$_Translation['translation_date'] = date("Y-m-d H:i:s");
/**
* when the status of the translation changes, the auto propagation flag
* must be removed
*/
if ($_Translation['translation'] != $old_translation['translation'] || $this->status == Constants_TranslationStatus::STATUS_TRANSLATED || $this->status == Constants_TranslationStatus::STATUS_APPROVED) {
$_Translation['autopropagated_from'] = 'NULL';
}
$res = CatUtils::addSegmentTranslation($_Translation);
if (!empty($res['errors'])) {
$this->result['errors'] = $res['errors'];
$msg = "\n\n Error addSegmentTranslation \n\n Database Error \n\n " . var_export(array_merge($this->result, $_POST), true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
$db->rollback();
return -1;
}
if (INIT::$DQF_ENABLED && !empty($this->jobData['dqf_key']) && $_Translation['status'] == Constants_TranslationStatus::STATUS_TRANSLATED) {
$dqfSegmentStruct = DQF_DqfSegmentStruct::getStruct();
if ($old_translation['suggestion'] == null) {
$dqfSegmentStruct->target_segment = "";
$dqfSegmentStruct->tm_match = 0;
} else {
$dqfSegmentStruct->target_segment = $old_translation['suggestion'];
$dqfSegmentStruct->tm_match = $old_translation['suggestion_match'];
}
$dqfSegmentStruct->task_id = $this->id_job;
$dqfSegmentStruct->segment_id = $this->id_segment;
$dqfSegmentStruct->source_segment = $segment['segment'];
$dqfSegmentStruct->new_target_segment = $_Translation['translation'];
$dqfSegmentStruct->time = $_Translation['time_to_edit'];
// $dqfSegmentStruct->mtengine = $this->jobData['id_mt_engine'];
$dqfSegmentStruct->mt_engine_version = 1;
try {
$dqfQueueHandler = new Analysis_DqfQueueHandler();
$dqfQueueHandler->createSegment($dqfSegmentStruct);
} catch (Exception $exn) {
$msg = $exn->getMessage() . "\n\n" . $exn->getTraceAsString();
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
$propagateToTranslated = true;
//for the moment, this is set explicitely
if ($this->propagate == false) {
$propagateToTranslated = false;
}
//propagate translations
$TPropagation = array();
//Warning: this value will NOT be used to update values,
//but to exclude current segment from auto-propagation
$_idSegment = $this->id_segment;
$propagateToTranslated ? $TPropagation['status'] = $this->status : null;
$TPropagation['id_job'] = $this->id_job;
$TPropagation['translation'] = $translation;
$TPropagation['autopropagated_from'] = $this->id_segment;
$TPropagation['serialized_errors_list'] = $err_json;
$TPropagation['warning'] = $check->thereAreWarnings();
// $TPropagation[ 'translation_date' ] = date( "Y-m-d H:i:s" );
$TPropagation['segment_hash'] = $old_translation['segment_hash'];
$propagationTotal = array();
if ($propagateToTranslated && in_array($this->status, array(Constants_TranslationStatus::STATUS_TRANSLATED, Constants_TranslationStatus::STATUS_APPROVED, Constants_TranslationStatus::STATUS_REJECTED))) {
try {
$propagationTotal = propagateTranslation($TPropagation, $this->jobData, $_idSegment, $propagateToTranslated);
} catch (Exception $e) {
$msg = $e->getMessage() . "\n\n" . $e->getTraceAsString();
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
//Recount Job Totals
$old_wStruct = new WordCount_Struct();
$old_wStruct->setIdJob($this->id_job);
$old_wStruct->setJobPassword($this->password);
$old_wStruct->setNewWords($this->jobData['new_words']);
$old_wStruct->setDraftWords($this->jobData['draft_words']);
$old_wStruct->setTranslatedWords($this->jobData['translated_words']);
$old_wStruct->setApprovedWords($this->jobData['approved_words']);
$old_wStruct->setRejectedWords($this->jobData['rejected_words']);
$old_wStruct->setIdSegment($this->id_segment);
//redundant, this is made into WordCount_Counter::updateDB
$old_wStruct->setOldStatus($old_translation['status']);
$old_wStruct->setNewStatus($this->status);
//redundant because the update is made only where status = old status
if ($this->status != $old_translation['status']) {
//cambiato status, sposta i conteggi
$old_count = !is_null($old_translation['eq_word_count']) ? $old_translation['eq_word_count'] : $segment['raw_word_count'];
//if there is not a row in segment_translations because volume analysis is disabled
//search for a just created row
$old_status = !empty($old_translation['status']) ? $old_translation['status'] : Constants_TranslationStatus::STATUS_NEW;
$counter = new WordCount_Counter($old_wStruct);
$counter->setOldStatus($old_status);
$counter->setNewStatus($this->status);
$newValues = array();
$newValues[] = $counter->getUpdatedValues($old_count);
foreach ($propagationTotal as $__pos => $old_value) {
$counter->setOldStatus($old_value['status']);
$counter->setNewStatus($this->status);
$newValues[] = $counter->getUpdatedValues($old_value['total']);
}
try {
$newTotals = $counter->updateDB($newValues);
} catch (Exception $e) {
$this->result['errors'][] = array("code" => -101, "message" => "database errors");
// Log::doLog("Lock: Transaction Aborted. " . $e->getMessage() );
// $x1 = explode( "\n" , var_export( $old_translation, true) );
// Log::doLog("Lock: Translation status was " . implode( " ", $x1 ) );
$db->rollback();
return $e->getCode();
}
} else {
$newTotals = $old_wStruct;
}
$job_stats = CatUtils::getFastStatsForJob($newTotals);
$project = getProject($this->jobData['id_project']);
$project = array_pop($project);
$job_stats['ANALYSIS_COMPLETE'] = $project['status_analysis'] == Constants_ProjectStatus::STATUS_DONE || $project['status_analysis'] == Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE ? true : false;
$file_stats = array();
$this->result['stats'] = $job_stats;
$this->result['file_stats'] = $file_stats;
$this->result['code'] = 1;
$this->result['data'] = "OK";
$this->result['version'] = date_create($_Translation['translation_date'])->getTimestamp();
/* FIXME: added for code compatibility with front-end. Remove. */
$_warn = $check->getWarnings();
$warning = $_warn[0];
/* */
$this->result['warning']['cod'] = $warning->outcome;
if ($warning->outcome > 0) {
$this->result['warning']['id'] = $this->id_segment;
} else {
$this->result['warning']['id'] = 0;
}
//strtoupper transforms null to "" so check for the first element to be an empty string
if (!empty($this->split_statuses[0]) && !empty($this->split_num)) {
/* put the split inside the transaction if they are present */
$translationStruct = TranslationsSplit_SplitStruct::getStruct();
$translationStruct->id_segment = $this->id_segment;
$translationStruct->id_job = $this->id_job;
$translationStruct->target_chunk_lengths = array('len' => $this->split_chunk_lengths, 'statuses' => $this->split_statuses);
$translationDao = new TranslationsSplit_SplitDAO(Database::obtain());
$result = $translationDao->update($translationStruct);
}
$db->commit();
if ($job_stats['TRANSLATED_PERC'] == '100') {
$update_completed = setJobCompleteness($this->id_job, 1);
}
if (@$update_completed < 0) {
$msg = "\n\n Error setJobCompleteness \n\n " . var_export($_POST, true);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project?
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid: " . $_tmKey['key'], -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
//check for glossary files and tmx and put them in front of the list
$infoFile = DetectProprietaryXliff::getInfo($fileName);
if (DetectProprietaryXliff::getMemoryFileType()) {
//found TMX, enable language checking routines
if (DetectProprietaryXliff::isTMXFile()) {
$this->checkTMX = 1;
}
//not used at moment but needed if we want to do a poll for status
if (DetectProprietaryXliff::isGlossaryFile()) {
$this->checkGlossary = 1;
}
//prepend in front of the list
array_unshift($sortedFiles, $fileName);
} else {
//append at the end of the list
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = $this->uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
//we are going to access the storage, get model object to manipulate it
$this->fileStorage = new FilesStorage();
$linkFiles = $this->fileStorage->getHashesFromDir($this->uploadDir);
/*
loop through all input files to
1) upload TMX and Glossaries
*/
try {
$this->_pushTMXToMyMemory();
} catch (Exception $e) {
//exit project creation
return false;
}
//TMX Management
/*
loop through all input files to
2)convert, in case, non standard XLIFF files to a format that Matecat understands
Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
*/
foreach ($this->projectStructure['array_files'] as $fileName) {
/*
Conversion Enforce
Checking Extension is no more sufficient, we want check content
$enforcedConversion = true; //( if conversion is enabled )
*/
$isAFileToConvert = $this->isConversionToEnforce($fileName);
//if it's one of the listed formats or conversion is not enabled in first place
if (!$isAFileToConvert) {
/*
filename is already an xliff and it's in upload directory
we have to make a cache package from it to avoid altering the original path
*/
//get file
$filePathName = "{$this->uploadDir}/{$fileName}";
//calculate hash + add the fileName, if i load 3 equal files with the same content
// they will be squashed to the last one
$sha1 = sha1_file($filePathName);
//make a cache package (with work/ only, emtpy orig/)
$this->fileStorage->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
//put reference to cache in upload dir to link cache to session
$this->fileStorage->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken'], $fileName);
//add newly created link to list
$linkFiles['conversionHashes']['sha'][] = $sha1 . "|" . $this->projectStructure['source_language'];
$linkFiles['conversionHashes']['fileName'][$sha1 . "|" . $this->projectStructure['source_language']][] = $fileName;
//when the same sdlxliff is uploaded more than once with different names
$linkFiles['conversionHashes']['sha'] = array_unique($linkFiles['conversionHashes']['sha']);
unset($sha1);
}
}
//now, upload dir contains only hash-links
//we start copying files to "file" dir, inserting metadata in db and extracting segments
foreach ($linkFiles['conversionHashes']['sha'] as $linkFile) {
//converted file is inside cache directory
//get hash from file name inside UUID dir
$hashFile = FilesStorage::basename_fix($linkFile);
$hashFile = explode('|', $hashFile);
//use hash and lang to fetch file from package
$cachedXliffFilePathName = $this->fileStorage->getXliffFromCache($hashFile[0], $hashFile[1]);
//get sha
$sha1_original = $hashFile[0];
//associate the hash to the right file in upload directory
//get original file name, to insert into DB and cp in storage
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
$_originalFileName = $linkFiles['conversionHashes']['fileName'][$linkFile];
unset($hashFile);
if (!file_exists($cachedXliffFilePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = FilesStorage::pathinfo_fix($cachedXliffFilePathName);
if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$yearMonthPath = date_create($this->projectStructure['create_date'])->format('Ymd');
$fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
foreach ($_originalFileName as $originalFileName) {
$mimeType = FilesStorage::pathinfo_fix($originalFileName, PATHINFO_EXTENSION);
$fid = insertFile($this->projectStructure, $originalFileName, $mimeType, $fileDateSha1Path);
//move the file in the right directory from the packages to the file dir
$this->fileStorage->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid, $originalFileName);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments(file_get_contents($cachedXliffFilePathName), $fid);
}
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$originalFileName}.");
$this->fileStorage->deleteHashFromUploadDir($this->uploadDir, $linkFile);
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$originalFileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$originalFileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$originalFileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} elseif ($e->getCode() == -13) {
$this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return null;
// SEVERE EXCEPTION we can not write to disk!! Break project creation
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$originalFileName}. Please try again.");
}
Log::doLog($e->getMessage());
Log::doLog($e->getTraceAsString());
}
}
//end of conversion hash-link loop
try {
$this->_zipFileHandling($linkFiles);
} catch (Exception $e) {
//exit project creation
return false;
}
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
if (!$this->_doCheckForErrors()) {
//exit project creation
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
try {
$rows = $this->dbHandler->fetch_array($query_visible_segments);
} catch (PDOException $e) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: {$e->getMessage()} - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($this->uploadDir);
if (is_dir($this->uploadDir . '_converted')) {
Utils::deleteDir($this->uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->projectStructure['dqf_key']);
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_customer'] = 'translated_user';
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid.", -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//found TMX, enable language checking routines
$this->checkTMX = 1;
array_unshift($sortedFiles, $fileName);
} else {
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
//fetch cache links, created by converter, from upload directory
$linkFiles = scandir($uploadDir);
//remove dir hardlinks, as uninteresting, as weel as regular files; only hash-links
foreach ($linkFiles as $k => $linkFile) {
if (strpos($linkFile, '.') !== false or strpos($linkFile, '|') === false) {
unset($linkFiles[$k]);
}
}
/*
loop through all input files to
1)upload TMX
2)convert, in case, non standard XLIFF files to a format that Matecat understands
Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
*/
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//load it into MyMemory; we'll check later on how it went
$file = new stdClass();
$file->file_path = "{$uploadDir}/{$fileName}";
$this->tmxServiceWrapper->setName($fileName);
$this->tmxServiceWrapper->setFile(array($file));
try {
$this->tmxServiceWrapper->addTmxInMyMemory();
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//in any case, skip the rest of the loop, go to the next file
continue;
}
/*
Conversion Enforce
Checking Extension is no more sufficient, we want check content if this is an idiom xlf file type, conversion are enforced
$enforcedConversion = true; //( if conversion is enabled )
*/
$isAnXliffToConvert = $this->isConversionToEnforce($fileName);
//we are going to access the storage, get model object to manipulate it
$fs = new FilesStorage();
//if it's one of the listed formats or conversion is not enabled in first place
if (!$isAnXliffToConvert) {
/*
filename is already an xliff and it's in upload directory
we have to make a cache package from it to avoid altering the original path
*/
//get file
$filePathName = "{$uploadDir}/{$fileName}";
//calculate hash + add the fileName, if i load 3 equal files with the same content
// they will be squashed to the last one
$sha1 = sha1(file_get_contents($filePathName) . $filePathName);
//make a cache package (with work/ only, emtpy orig/)
$fs->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
//put reference to cache in upload dir to link cache to session
$fs->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken']);
//add newly created link to list
$linkFiles[] = $sha1 . "|" . $this->projectStructure['source_language'];
unset($sha1);
}
}
//now, upload dir contains only hash-links
//we start copying files to "file" dir, inserting metadata in db and extracting segments
foreach ($linkFiles as $linkFile) {
//converted file is inside cache directory
//get hash from file name inside UUID dir
$hashFile = basename($linkFile);
$hashFile = explode('|', $hashFile);
//use hash and lang to fetch file from package
$xliffFilePathName = $fs->getXliffFromCache($hashFile[0], $hashFile[1]);
//get sha
$sha1_original = $hashFile[0];
//get original file name
$originalFilePathName = $fs->getOriginalFromCache($hashFile[0], $hashFile[1]);
$raw_originalFilePathName = explode(DIRECTORY_SEPARATOR, $originalFilePathName);
$fileName = array_pop($raw_originalFilePathName);
unset($hashFile);
if (!file_exists($xliffFilePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = pathinfo($xliffFilePathName);
if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
$yearMonthPath = date_create()->format("Ymd");
$fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
$fid = insertFile($this->projectStructure, $fileName, $mimeType, $fileDateSha1Path);
//move the file in the right directory from the packages to the file dir
$fs->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments(file_get_contents($xliffFilePathName), $fid);
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
$fs->deleteHashFromUploadDir($uploadDir, $linkFile);
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} elseif ($e->getCode() == -13) {
$this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return null;
// SEVERE EXCEPTION we can not write to disk!! Break project creation
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$fileName}. Please try again.");
}
Log::doLog($e->getMessage());
}
}
//end of hash-link loop
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
/****************/
//loop again through files to check to check for TMX loading
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$this->tmxServiceWrapper->setName($fileName);
$result = array();
//is the TM loaded?
//wait until current TMX is loaded
while (true) {
try {
$result = $this->tmxServiceWrapper->tmxUploadStatus();
if ($result['completed']) {
//"$fileName" has been loaded into MyMemory"
//exit the loop
break;
}
//"waiting for "$fileName" to be loaded into MyMemory"
sleep(3);
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
//exit project creation
return false;
}
}
//once the language is loaded, check if language is compliant (unless something useful has already been found)
if (1 == $this->checkTMX) {
//get localized target languages of TM (in case it's a multilingual TM)
$tmTargets = explode(';', $result['data']['target_lang']);
//indicates if something has been found for current memory
$found = false;
//compare localized target languages array (in case it's a multilingual project) to the TM supplied
//if nothing matches, then the TM supplied can't have matches for this project
//create an empty var and add the source language too
$project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
foreach ($project_languages as $projectTarget) {
if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
$found = true;
break;
}
}
//if this TM matches the project lagpair and something has been found
if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
//the TMX is good to go
$this->checkTMX = 0;
} elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
/*
* This means that the TMX has a srclang as specification in the header. Warn the user.
* Ex:
* <header creationtool="SDL Language Platform"
* creationtoolversion="8.0"
* datatype="rtf"
* segtype="sentence"
* adminlang="DE-DE"
* srclang="DE-DE" />
*/
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
$this->checkTMX = 0;
Log::doLog($this->projectStructure['result']);
}
}
}
}
if (1 == $this->checkTMX) {
//this means that noone of uploaded TMX were usable for this project. Warn the user.
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
Log::doLog($this->projectStructure['result']);
return false;
}
if (!empty($this->projectStructure['result']['errors'])) {
Log::doLog("Project Creation Failed. Sent to Output all errors.");
Log::doLog($this->projectStructure['result']['errors']);
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
$rows = $this->dbHandler->fetch_array($query_visible_segments);
if (!$rows) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($uploadDir);
if (is_dir($uploadDir . '_converted')) {
Utils::deleteDir($uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
}
