/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return void
*/
function doAction()
{
//if some error occured, stop execution.
if (count(@$this->result['errors'])) {
return;
}
try {
$tmService = new TMSService();
$tmService->setTmKey($this->key);
//validate the key
try {
$keyExists = $tmService->checkCorrectKey();
} catch (Exception $e) {
/* PROVIDED KEY IS NOT VALID OR WRONG, $keyExists IS NOT SET */
Log::doLog($e->getMessage());
}
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid.", -4);
}
$tmKeyStruct = new TmKeyManagement_TmKeyStruct();
$tmKeyStruct->key = $this->key;
$tmKeyStruct->name = $this->description;
$tmKeyStruct->tm = true;
$tmKeyStruct->glos = true;
$mkDao = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$memoryKeyToUpdate = new TmKeyManagement_MemoryKeyStruct();
$memoryKeyToUpdate->uid = $this->uid;
$memoryKeyToUpdate->tm_key = $tmKeyStruct;
switch ($this->exec) {
case 'delete':
$userMemoryKeys = $mkDao->disable($memoryKeyToUpdate);
break;
case 'update':
$userMemoryKeys = $mkDao->update($memoryKeyToUpdate);
break;
case 'newKey':
$userMemoryKeys = $mkDao->create($memoryKeyToUpdate);
break;
default:
throw new Exception("Unexpected Exception", -4);
}
if (!$userMemoryKeys) {
throw new Exception("This key wasn't found in your keyring.", -3);
}
} catch (Exception $e) {
$this->result['data'] = 'KO';
$this->result['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
}
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_customer'] = 'translated_user';
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid.", -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//found TMX, enable language checking routines
$this->checkTMX = 1;
array_unshift($sortedFiles, $fileName);
} else {
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$file = new stdClass();
$file->file_path = "{$uploadDir}/{$fileName}";
$this->tmxServiceWrapper->setName($fileName);
$this->tmxServiceWrapper->setFile(array($file));
try {
$this->tmxServiceWrapper->addTmxInMyMemory();
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//in any case, skip the rest of the loop, go to the next file
continue;
}
/**
* Conversion Enforce
*
* we have to know if a file can be found in _converted directory
*
* Check Extension no more sufficient, we want check content
* if this is an idiom xlf file type, conversion are enforced
* $enforcedConversion = true; //( if conversion is enabled )
*/
$isAConvertedFile = true;
try {
$fileType = DetectProprietaryXliff::getInfo(INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'] . DIRECTORY_SEPARATOR . $fileName);
if (DetectProprietaryXliff::isXliffExtension()) {
if (INIT::$CONVERSION_ENABLED) {
//conversion enforce
if (!INIT::$FORCE_XLIFF_CONVERSION) {
//ONLY IDIOM is forced to be converted
//if file is not proprietary like idiom AND Enforce is disabled
//we take it as is
if (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
} else {
//if conversion enforce is active
//we force all xliff files but not files produced by SDL Studio because we can handle them
if ($fileType['proprietary_short_name'] == 'trados') {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
} elseif ($fileType['proprietary']) {
/**
* Application misconfiguration.
* upload should not be happened, but if we are here, raise an error.
* @see upload.class.php
* */
$this->projectStructure['result']['errors'][] = array("code" => -8, "message" => "Proprietary xlf format detected. Not able to import this XLIFF file. ({$fileName})");
setcookie("upload_session", "", time() - 10000);
return -1;
//stop execution
} elseif (!$fileType['proprietary']) {
$isAConvertedFile = false;
//ok don't convert a standard sdlxliff
}
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
$mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
$original_content = "";
/*
if it's not one of the listed formats (or it is, but you had to convert it anyway),
and conversion is enabled in first place
*/
if ($isAConvertedFile) {
//converted file is inside "_converted" directory
$fileDir = $uploadDir . '_converted';
$original_content = file_get_contents("{$uploadDir}/{$fileName}");
$sha1_original = sha1($original_content);
$original_content = gzdeflate($original_content, 5);
//file name is a xliff converted like: 'a_word_document.doc.sdlxliff'
$real_fileName = $fileName . '.sdlxliff';
} else {
//filename is already an xliff and it is in a canonical normal directory
$sha1_original = "";
$fileDir = $uploadDir;
$real_fileName = $fileName;
}
$filePathName = $fileDir . DIRECTORY_SEPARATOR . $real_fileName;
if (!file_exists($filePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = pathinfo($filePathName);
if ($info['extension'] == 'xliff' || $info['extension'] == 'sdlxliff' || $info['extension'] == 'xlf') {
$contents = file_get_contents($filePathName);
} else {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$fid = insertFile($this->projectStructure, $fileName, $mimeType, $contents, $sha1_original, $original_content);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments($contents, $fid);
unset($contents);
//free memory
//Log::doLog( $this->projectStructure['segments'] );
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File is Too large. ( {$fileName} )");
}
Log::doLog($e->getMessage());
}
//exit;
}
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
/****************/
//loop again through files to check to check for TMX loading
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$this->tmxServiceWrapper->setName($fileName);
$result = array();
//is the TM loaded?
//wait until current TMX is loaded
while (true) {
try {
$result = $this->tmxServiceWrapper->tmxUploadStatus();
if ($result['completed']) {
//"$fileName" has been loaded into MyMemory"
//exit the loop
break;
}
//"waiting for "$fileName" to be loaded into MyMemory"
sleep(3);
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
//exit project creation
return false;
}
}
//once the language is loaded, check if language is compliant (unless something useful has already been found)
if (1 == $this->checkTMX) {
//get localized target languages of TM (in case it's a multilingual TM)
$tmTargets = explode(';', $result['data']['target_lang']);
//indicates if something has been found for current memory
$found = false;
//compare localized target languages array (in case it's a multilingual project) to the TM supplied
//if nothing matches, then the TM supplied can't have matches for this project
//create an empty var and add the source language too
$project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
foreach ($project_languages as $projectTarget) {
if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
$found = true;
break;
}
}
//if this TM matches the project lagpair and something has been found
if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
//the TMX is good to go
$this->checkTMX = 0;
} elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
/*
* This means that the TMX has a srclang as specification in the header. Warn the user.
* Ex:
* <header creationtool="SDL Language Platform"
* creationtoolversion="8.0"
* datatype="rtf"
* segtype="sentence"
* adminlang="DE-DE"
* srclang="DE-DE" />
*/
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
$this->checkTMX = 0;
Log::doLog($this->projectStructure['result']);
}
}
}
}
if (1 == $this->checkTMX) {
//this means that noone of uploaded TMX were usable for this project. Warn the user.
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
Log::doLog($this->projectStructure['result']);
return false;
}
if (!empty($this->projectStructure['result']['errors'])) {
Log::doLog("Project Creation Failed. Sent to Output all errors.");
Log::doLog($this->projectStructure['result']['errors']);
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n FROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
$rows = $this->dbHandler->fetch_array($query_visible_segments);
if (!$rows) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($uploadDir);
if (is_dir($uploadDir . '_converted')) {
Utils::deleteDir($uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
/**
* TODO: remove after queue implementation
*/
// if( 0 && $project_summary[0]['project_segments'] > 50000 ) {
// $this->projectStructure[ 'status' ] = Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
//
// $msg = "
// WARNING: a project with more than 50.000 segments was created. ( " . $project_summary[0]['project_segments'] . " )\n" .
// var_export( $this->projectStructure[ 'result' ], true ) . "\n\n" .
// " " .
// var_export( $project_summary[0] , true ) . "\n";
//
// Utils::sendErrMailReport( $msg, "Alert: Project Creation Abort. - " );
//
// }
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
// var_dump($this->projectStructure);
// exit;
}
/**
* Merge the keys from CLIENT with those from DATABASE ( jobData )
*
* @param string $Json_clientKeys A json_encoded array of objects having the following structure:<br />
* <pre>
* array(
* 'key' => <private_tm_key>,
* 'name' => <tm_name>,
* 'r' => true,
* 'w' => true
* )
* </pre>
* @param string $Json_jobKeys A json_encoded array of TmKeyManagement_TmKeyStruct objects
* @param string $userRole One of the following strings: "owner", "translator", "revisor"
* @param int $uid
*
* @see TmKeyManagement_TmKeyStruct
*
* @return array TmKeyManagement_TmKeyStruct[]
*
* @throws Exception
*/
public static function mergeJsonKeys($Json_clientKeys, $Json_jobKeys, $userRole = TmKeyManagement_Filter::ROLE_TRANSLATOR, $uid = null)
{
//we put the already present job keys so they can be checked against the client keys when cycle advances
//( jobs has more elements than the client objects )
$clientDecodedJson = json_decode($Json_clientKeys, true);
Utils::raiseJsonExceptionError();
$serverDecodedJson = json_decode($Json_jobKeys, true);
Utils::raiseJsonExceptionError();
if (!array_key_exists($userRole, TmKeyManagement_Filter::$GRANTS_MAP)) {
throw new Exception("Invalid Role Type string.", 4);
}
$client_tm_keys = array_map(array('self', 'getTmKeyStructure'), $clientDecodedJson);
$client_tm_keys = array_map(array('self', 'sanitize'), $client_tm_keys);
$job_tm_keys = array_map(array('self', 'getTmKeyStructure'), $serverDecodedJson);
$server_reorder_position = array();
$reverse_lookup_client_json = array('pos' => array(), 'elements' => array(), 'unique' => array());
foreach ($client_tm_keys as $_j => $_client_tm_key) {
/**
* @var $_client_tm_key TmKeyManagement_TmKeyStruct
*/
//create a reverse lookup
$reverse_lookup_client_json['pos'][$_j] = $_client_tm_key->key;
$reverse_lookup_client_json['elements'][$_j] = $_client_tm_key;
$reverse_lookup_client_json['unique'][$_j] = $_client_tm_key->getCrypt();
if (empty($_client_tm_key->r) && empty($_client_tm_key->w)) {
throw new Exception("Please, select Lookup and/or Update to activate your TM in this project", 4);
}
if (empty($_client_tm_key->key)) {
throw new Exception("Invalid Key Provided", 5);
}
}
$uniq_num = count(array_unique($reverse_lookup_client_json['unique']));
if ($uniq_num != count($reverse_lookup_client_json['pos'])) {
throw new Exception("A key is already present in this project.", 5);
}
//update existing job keys
foreach ($job_tm_keys as $i => $_job_Key) {
/**
* @var $_job_Key TmKeyManagement_TmKeyStruct
*/
$_index_position = array_search($_job_Key->key, $reverse_lookup_client_json['pos']);
if (array_search($_job_Key->getCrypt(), $reverse_lookup_client_json['pos']) !== false) {
//DO NOTHING
//reduce the stack
$hashPosition = array_search($_job_Key->getCrypt(), $reverse_lookup_client_json['pos']);
unset($reverse_lookup_client_json['pos'][$hashPosition]);
unset($reverse_lookup_client_json['elements'][$hashPosition]);
//PASS
//take the new order
$server_reorder_position[$hashPosition] = $_job_Key;
} elseif ($_index_position !== false) {
// so, here the key exists in client
//this is an anonymous user, and a key exists in job
if ($_index_position !== false && $uid == null) {
//check anonymous user, an anonymous user can not change a not anonymous key
if ($userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) {
if ($_job_Key->uid_transl != null) {
throw new Exception("Anonymous user can not modify existent keys.", 1);
}
} elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR) {
if ($_job_Key->uid_rev != null) {
throw new Exception("Anonymous user can not modify existent keys.", 2);
}
} else {
if ($uid == null) {
throw new Exception("Anonymous user can not be OWNER", 3);
}
}
}
//override the static values
$_job_Key->tm = filter_var($reverse_lookup_client_json['elements'][$_index_position]->tm, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
$_job_Key->glos = filter_var($reverse_lookup_client_json['elements'][$_index_position]->glos, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
if ($userRole == TmKeyManagement_Filter::OWNER) {
//override grants
$_job_Key->r = filter_var($reverse_lookup_client_json['elements'][$_index_position]->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
$_job_Key->w = filter_var($reverse_lookup_client_json['elements'][$_index_position]->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
} elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR || $userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) {
//override role specific grants
$_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['r']} = filter_var($reverse_lookup_client_json['elements'][$_index_position]->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
$_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['w']} = filter_var($reverse_lookup_client_json['elements'][$_index_position]->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
}
//change name if modified
if ($_job_Key->name != $reverse_lookup_client_json['elements'][$_index_position]->name) {
$_job_Key->name = $reverse_lookup_client_json['elements'][$_index_position]->name;
}
//set as owner if it is but should be already set
// $_job_Key->owner = ( $userRole == TmKeyManagement_Filter::OWNER );
//reduce the stack
unset($reverse_lookup_client_json['pos'][$_index_position]);
unset($reverse_lookup_client_json['elements'][$_index_position]);
//take the new order
$server_reorder_position[$_index_position] = $_job_Key;
} else {
//the key must be deleted
if ($userRole == TmKeyManagement_Filter::OWNER) {
//override grants
$_job_Key->r = null;
$_job_Key->w = null;
$_job_Key->owner = false;
} elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR || $userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) {
//override role specific grants
$_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['r']} = null;
$_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['w']} = null;
}
//remove the uid property
if ($userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) {
$_job_Key->uid_transl = null;
} elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR) {
$_job_Key->uid_rev = null;
}
//if the key is no more linked to someone, don't add to the resultset, else reorder if it is not an owner key.
if ($_job_Key->owner || !is_null($_job_Key->uid_transl) || !is_null($_job_Key->uid_rev)) {
if (!$_job_Key->owner) {
//take the new order, put the deleted key at the end of the array
//a position VERY LOW ( 1 Million )
$server_reorder_position[1000000 + $i] = $_job_Key;
} else {
if ($userRole != TmKeyManagement_Filter::OWNER) {
//place on top of the owner keys, preserve the order of owner keys by adding it's normal index position
$server_reorder_position[-1000000 + $i] = $_job_Key;
} else {
// Remove the key!!!
//only the owner can remove it's keys
}
}
}
}
}
/*
* There are some new keys from client? Add them
*/
if (!empty($reverse_lookup_client_json['pos'])) {
$justCreatedKey = new TmKeyManagement_TmKeyStruct();
foreach ($reverse_lookup_client_json['elements'] as $_pos => $newClientKey) {
/**
* @var $newClientKey TmKeyManagement_TmKeyStruct
*/
//set the key value
$justCreatedKey->key = $newClientKey->key;
//override the static values
$justCreatedKey->tm = filter_var($newClientKey->tm, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
$justCreatedKey->glos = filter_var($newClientKey->glos, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
if ($userRole != TmKeyManagement_Filter::OWNER) {
$justCreatedKey->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['r']} = filter_var($newClientKey->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
$justCreatedKey->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['w']} = filter_var($newClientKey->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
} else {
//override grants
$justCreatedKey->r = filter_var($newClientKey->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
$justCreatedKey->w = filter_var($newClientKey->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
}
//set the uid property
if ($userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) {
$justCreatedKey->uid_transl = $uid;
} elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR) {
$justCreatedKey->uid_rev = $uid;
}
//choose a name instead of null
$justCreatedKey->name = $newClientKey->name;
//choose an owner instead of null
$justCreatedKey->owner = $userRole == TmKeyManagement_Filter::OWNER;
//finally append to the job keys!!
//take the new order, put the new key at the end of the array
//a position VERY LOW, but BEFORE the deleted keys, so it goes not to the end ( 100 hundred thousand )
$server_reorder_position[100000 + $_pos] = $justCreatedKey;
if ($uid != null) {
//if uid is provided, check for key and try to add to it's memory key ring
try {
/*
* Take the keys of the user
*/
$_keyDao = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => $uid, 'tm_key' => new TmKeyManagement_TmKeyStruct(array('key' => $justCreatedKey->key))));
$keyList = $_keyDao->read($dh);
if (empty($keyList)) {
// add the key to a new row struct
$dh->uid = $uid;
$dh->tm_key = $justCreatedKey;
$_keyDao->create($dh);
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
}
}
}
ksort($server_reorder_position, SORT_NUMERIC);
return array_values($server_reorder_position);
}
public function doAction()
{
//Get the guid from the guid if it exists, otherwise set the guid into the cookie
if (!isset($_COOKIE['upload_session'])) {
setcookie("upload_session", $this->guid, time() + 86400);
} else {
$this->guid = $_COOKIE['upload_session'];
}
if (isset($_COOKIE["sourceLang"]) and $_COOKIE["sourceLang"] == "_EMPTY_") {
$this->noSourceLangHistory = true;
} else {
if (!isset($_COOKIE['sourceLang'])) {
setcookie("sourceLang", "_EMPTY_", time() + 86400 * 365);
$this->noSourceLangHistory = true;
} else {
if ($_COOKIE["sourceLang"] != "_EMPTY_") {
$this->noSourceLangHistory = false;
$this->sourceLangHistory = $_COOKIE["sourceLang"];
$this->sourceLangAr = explode('||', urldecode($this->sourceLangHistory));
$tmpSourceAr = array();
$tmpSourceArAs = array();
foreach ($this->sourceLangAr as $key => $lang) {
if ($lang != '') {
$tmpSourceAr[$lang] = $this->lang_handler->getLocalizedName($lang);
$ar = array();
$ar['name'] = $this->lang_handler->getLocalizedName($lang);
$ar['code'] = $lang;
$ar['selected'] = $key == '0' ? 1 : 0;
$ar['direction'] = $this->lang_handler->isRTL(strtolower($lang)) ? 'rtl' : 'ltr';
array_push($tmpSourceArAs, $ar);
}
}
$this->sourceLangAr = $tmpSourceAr;
asort($this->sourceLangAr);
$this->array_sort_by_column($tmpSourceArAs, 'name');
$this->sourceLangArray = $tmpSourceArAs;
}
}
}
if (isset($_COOKIE["targetLang"]) and $_COOKIE["targetLang"] == "_EMPTY_") {
$this->noTargetLangHistory = true;
} else {
if (!isset($_COOKIE['targetLang'])) {
setcookie("targetLang", "_EMPTY_", time() + 86400 * 365);
$this->noTargetLangHistory = true;
} else {
if ($_COOKIE["targetLang"] != "_EMPTY_") {
$this->noTargetLangHistory = false;
$this->targetLangHistory = $_COOKIE["targetLang"];
$this->targetLangAr = explode('||', urldecode($this->targetLangHistory));
$tmpTargetAr = array();
$tmpTargetArAs = array();
foreach ($this->targetLangAr as $key => $lang) {
if ($lang != '') {
$prova = explode(',', urldecode($lang));
$cl = "";
foreach ($prova as $ll) {
$cl .= $this->lang_handler->getLocalizedName($ll) . ',';
}
$cl = substr_replace($cl, "", -1);
$tmpTargetAr[$lang] = $cl;
// $tmpTargetAr[$lang] = $this->lang_handler->getLocalizedName($lang,'en');
$ar = array();
$ar['name'] = $cl;
$ar['direction'] = $this->lang_handler->isRTL(strtolower($lang)) ? 'rtl' : 'ltr';
$ar['code'] = $lang;
$ar['selected'] = $key == '0' ? 1 : 0;
array_push($tmpTargetArAs, $ar);
}
}
$this->targetLangAr = $tmpTargetAr;
asort($this->targetLangAr);
$this->array_sort_by_column($tmpTargetArAs, 'name');
$this->targetLangArray = $tmpTargetArAs;
}
}
}
$intDir = INIT::$UPLOAD_REPOSITORY . '/' . $this->guid . '/';
if (!is_dir($intDir)) {
mkdir($intDir, 0775, true);
}
// check if user is logged and generate authURL for logging in
$this->doAuth();
$this->generateAuthURL();
list($uid, $cid) = $this->getLoginUserParams();
$engine = new EnginesModel_EngineDAO(Database::obtain());
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->type = 'MT';
if (@(bool) $_GET['amt'] == true) {
$engineQuery->uid = 'all';
} else {
$engineQuery->uid = $uid == null ? -1 : $uid;
}
$engineQuery->active = 1;
$this->mt_engines = $engine->read($engineQuery);
if ($this->isLoggedIn()) {
try {
$_keyList = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => @$_SESSION['uid']));
$keyList = $_keyList->read($dh);
foreach ($keyList as $memKey) {
//all keys are available in this condition ( we are creating a project
$this->keyList[] = $memKey->tm_key;
}
} catch (Exception $e) {
Log::doLog($e->getMessage());
}
}
}
/**
*
* Get the right su
*
* @param $match
*
* @return null|string
* @throws Exception
*/
private function __changeSuggestionSource($match)
{
$sug_source = $match['created_by'];
$key = $match['memory_key'];
//suggestion is coming from a public TM
if ($sug_source == 'Matecat') {
$description = "Public TM";
} elseif (!empty($sug_source) && stripos($sug_source, "MyMemory") === false) {
$description = $sug_source;
} elseif (preg_match("/[a-f0-9]{8,}/", $key)) {
// md5 Key
//MyMemory returns the key of the match
//Session Enabled
$this->checkLogin();
//Session Disabled
if ($this->userIsLogged) {
//check if the user can see the key.
$memoryKey = new TmKeyManagement_MemoryKeyStruct();
$memoryKey->uid = $this->uid;
$memoryKey->tm_key = new TmKeyManagement_TmKeyStruct();
$memoryKey->tm_key->key = $key;
$memoryKeyDao = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$currentUserMemoryKey = $memoryKeyDao->setCacheTTL(3600)->read($memoryKey);
if (count($currentUserMemoryKey) > 0) {
//the current user owns the key: show its description
$currentUserMemoryKey = $currentUserMemoryKey[0];
$description = $currentUserMemoryKey->tm_key->name;
}
}
}
/**
* if the description is empty, get cascading default descriptions
*/
if (empty($description)) {
$description = $this->__getDefaultDescription($key);
}
if (empty($description)) {
$description = "No description available";
//this should never be
}
return $description;
}
public function doAction()
{
$files_found = array();
$lang_handler = Langs_Languages::getInstance();
$data = getSegmentsInfo($this->jid, $this->password);
if (empty($data) or $data < 0) {
$this->job_not_found = true;
//stop execution
return;
}
//retrieve job owner. It will be useful also if the job is archived or cancelled
$this->job_owner = $data[0]['job_owner'] != "" ? $data[0]['job_owner'] : "*****@*****.**";
if ($data[0]['status'] == Constants_JobStatus::STATUS_CANCELLED) {
$this->job_cancelled = true;
//stop execution
return;
}
if ($data[0]['status'] == Constants_JobStatus::STATUS_ARCHIVED) {
$this->job_archived = true;
//stop execution
return;
}
/*
* I prefer to use a programmatic approach to the check for the archive date instead of a pure query
* because the query to check "Utils::getArchivableJobs($this->jid)" should be
* executed every time a job is loaded ( F5 or CTRL+R on browser ) and it cost some milliseconds ( ~0.1s )
* and it is little heavy for the database.
* We use the data we already have from last query and perform
* the check on the last translation only if the job is older than 30 days
*
*/
$lastUpdate = new DateTime($data[0]['last_update']);
$oneMonthAgo = new DateTime();
$oneMonthAgo->modify('-' . INIT::JOB_ARCHIVABILITY_THRESHOLD . ' days');
if ($lastUpdate < $oneMonthAgo && !$this->job_cancelled) {
$lastTranslationInJob = new Datetime(getLastTranslationDate($this->jid));
if ($lastTranslationInJob < $oneMonthAgo) {
$res = "job";
$new_status = Constants_JobStatus::STATUS_ARCHIVED;
updateJobsStatus($res, $this->jid, $new_status, null, null, $this->password);
$this->job_archived = true;
}
}
foreach ($data as $i => $job) {
$this->project_status = $job;
// get one row values for the project are the same for every row
if (empty($this->pname)) {
$this->pname = $job['pname'];
$this->downloadFileName = $job['pname'] . ".zip";
// will be overwritten below in case of one file job
}
if (empty($this->last_opened_segment)) {
$this->last_opened_segment = $job['last_opened_segment'];
}
if (empty($this->cid)) {
$this->cid = $job['cid'];
}
if (empty($this->pid)) {
$this->pid = $job['pid'];
}
if (empty($this->create_date)) {
$this->create_date = $job['create_date'];
}
if (empty($this->source_code)) {
$this->source_code = $job['source'];
}
if (empty($this->target_code)) {
$this->target_code = $job['target'];
}
if (empty($this->source)) {
$s = explode("-", $job['source']);
$source = strtoupper($s[0]);
$this->source = $source;
$this->source_rtl = $lang_handler->isRTL(strtolower($this->source)) ? ' rtl-source' : '';
}
if (empty($this->target)) {
$t = explode("-", $job['target']);
$target = strtoupper($t[0]);
$this->target = $target;
$this->target_rtl = $lang_handler->isRTL(strtolower($this->target)) ? ' rtl-target' : '';
}
//check if language belongs to supported right-to-left languages
if ($job['status'] == Constants_JobStatus::STATUS_ARCHIVED) {
$this->job_archived = true;
$this->job_owner = $data[0]['job_owner'];
}
$id_file = $job['id_file'];
if (!isset($this->data["{$id_file}"])) {
$files_found[] = $job['filename'];
}
$wStruct = new WordCount_Struct();
$wStruct->setIdJob($this->jid);
$wStruct->setJobPassword($this->password);
$wStruct->setNewWords($job['new_words']);
$wStruct->setDraftWords($job['draft_words']);
$wStruct->setTranslatedWords($job['translated_words']);
$wStruct->setApprovedWords($job['approved_words']);
$wStruct->setRejectedWords($job['rejected_words']);
unset($job['id_file']);
unset($job['source']);
unset($job['target']);
unset($job['source_code']);
unset($job['target_code']);
unset($job['mime_type']);
unset($job['filename']);
unset($job['jid']);
unset($job['pid']);
unset($job['cid']);
unset($job['tid']);
unset($job['pname']);
unset($job['create_date']);
unset($job['owner']);
unset($job['last_opened_segment']);
unset($job['new_words']);
unset($job['draft_words']);
unset($job['translated_words']);
unset($job['approved_words']);
unset($job['rejected_words']);
//For projects created with No tm analysis enabled
if ($wStruct->getTotal() == 0 && ($job['status_analysis'] == Constants_ProjectStatus::STATUS_DONE || $job['status_analysis'] == Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE)) {
$wCounter = new WordCount_Counter();
$wStruct = $wCounter->initializeJobWordCount($this->jid, $this->password);
Log::doLog("BackWard compatibility set Counter.");
}
$this->job_stats = CatUtils::getFastStatsForJob($wStruct);
}
//Needed because a just created job has last_opened segment NULL
if (empty($this->last_opened_segment)) {
$this->last_opened_segment = getFirstSegmentId($this->jid, $this->password);
}
$this->first_job_segment = $this->project_status['job_first_segment'];
$this->last_job_segment = $this->project_status['job_last_segment'];
if (count($files_found) == 1) {
$this->downloadFileName = $files_found[0];
}
/**
* get first segment of every file
*/
$fileInfo = getFirstSegmentOfFilesInJob($this->jid);
$TotalPayable = array();
foreach ($fileInfo as &$file) {
$file['file_name'] = ZipArchiveExtended::getFileName($file['file_name']);
$TotalPayable[$file['id_file']]['TOTAL_FORMATTED'] = $file['TOTAL_FORMATTED'];
}
$this->firstSegmentOfFiles = json_encode($fileInfo);
$this->fileCounter = json_encode($TotalPayable);
list($uid, $user_email) = $this->getLoginUserParams();
if (self::isRevision()) {
$this->userRole = TmKeyManagement_Filter::ROLE_REVISOR;
} elseif ($user_email == $data[0]['job_owner']) {
$this->userRole = TmKeyManagement_Filter::OWNER;
} else {
$this->userRole = TmKeyManagement_Filter::ROLE_TRANSLATOR;
}
/*
* Take the keys of the user
*/
try {
$_keyList = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => $uid));
$keyList = $_keyList->read($dh);
} catch (Exception $e) {
$keyList = array();
Log::doLog($e->getMessage());
}
$reverse_lookup_user_personal_keys = array('pos' => array(), 'elements' => array());
/**
* Set these keys as editable for the client
*
* @var $keyList TmKeyManagement_MemoryKeyStruct[]
*/
foreach ($keyList as $_j => $key) {
/**
* @var $_client_tm_key TmKeyManagement_TmKeyStruct
*/
//create a reverse lookup
$reverse_lookup_user_personal_keys['pos'][$_j] = $key->tm_key->key;
$reverse_lookup_user_personal_keys['elements'][$_j] = $key;
$this->_keyList['totals'][$_j] = new TmKeyManagement_ClientTmKeyStruct($key->tm_key);
}
/*
* Now take the JOB keys
*/
$job_keyList = json_decode($data[0]['tm_keys'], true);
$this->tid = count($job_keyList) > 0;
/**
* Start this N^2 cycle from keys of the job,
* these should be statistically lesser than the keys of the user
*
* @var $keyList array
*/
foreach ($job_keyList as $jobKey) {
$jobKey = new TmKeyManagement_ClientTmKeyStruct($jobKey);
if ($this->isLoggedIn() && count($reverse_lookup_user_personal_keys['pos'])) {
/*
* If user has some personal keys, check for the job keys if they are present, and obfuscate
* when they are not
*/
$_index_position = array_search($jobKey->key, $reverse_lookup_user_personal_keys['pos']);
if ($_index_position !== false) {
//i found a key in the job that is present in my database
//i'm owner?? and the key is an owner type key?
if (!$jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) {
$jobKey->r = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['r']};
$jobKey->w = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['w']};
$jobKey = $jobKey->hideKey($uid);
} else {
if ($jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) {
// I'm not the job owner, but i know the key because it is in my keyring
// so, i can upload and download TMX, but i don't want it to be removed from job
// in tm.html relaxed the control to "key.edit" to enable buttons
// $jobKey = $jobKey->hideKey( $uid ); // enable editing
} else {
if ($jobKey->owner && $this->userRole == TmKeyManagement_Filter::OWNER) {
//do Nothing
}
}
}
unset($this->_keyList['totals'][$_index_position]);
} else {
/*
* This is not a key of that user, set right and obfuscate
*/
$jobKey->r = true;
$jobKey->w = true;
$jobKey = $jobKey->hideKey(-1);
}
$this->_keyList['job_keys'][] = $jobKey;
} else {
/*
* This user is anonymous or it has no keys in its keyring, obfuscate all
*/
$jobKey->r = true;
$jobKey->w = true;
$this->_keyList['job_keys'][] = $jobKey->hideKey(-1);
}
}
//clean unordered keys
$this->_keyList['totals'] = array_values($this->_keyList['totals']);
/**
* Retrieve information about job errors
* ( Note: these information are fed by the revision process )
* @see setRevisionController
*/
$jobQA = new Revise_JobQA($this->jid, $this->password, $wStruct->getTotal());
$jobQA->retrieveJobErrorTotals();
$jobVote = $jobQA->evalJobVote();
$this->qa_data = json_encode($jobQA->getQaData());
$this->qa_overall = $jobVote['minText'];
$engine = new EnginesModel_EngineDAO(Database::obtain());
//this gets all engines of the user
if ($this->isLoggedIn()) {
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->type = 'MT';
$engineQuery->uid = $uid;
$engineQuery->active = 1;
$mt_engines = $engine->read($engineQuery);
} else {
$mt_engines = array();
}
// this gets MyMemory
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->type = 'TM';
$engineQuery->active = 1;
$tms_engine = $engine->setCacheTTL(3600 * 24 * 30)->read($engineQuery);
//this gets MT engine active for the job
$engineQuery = new EnginesModel_EngineStruct();
$engineQuery->id = $this->project_status['id_mt_engine'];
$engineQuery->active = 1;
$active_mt_engine = $engine->setCacheTTL(60 * 10)->read($engineQuery);
/*
* array_unique cast EnginesModel_EngineStruct to string
*
* EnginesModel_EngineStruct implements __toString method
*
*/
$this->translation_engines = array_unique(array_merge($active_mt_engine, $tms_engine, $mt_engines));
}
public function testMergeJsonKeys_validInput_clientABCGHIJKL_serverABCGHIDEF()
{
Tests_DBLoader4Test::resetDB();
//this should not to be, because a client can not send not hashed keys
//already present in job as an anonymous user
$resultMerge = TmKeyManagement_TmKeyManagement::mergeJsonKeys(self::$client_json_ABC_GHI_JKL, self::$srv_json_ABC_GHI_DEF, TmKeyManagement_Filter::ROLE_TRANSLATOR, 123);
$MemoryDao = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => 123));
$insertedKeys = $MemoryDao->read($dh);
$this->assertEquals(1, count($insertedKeys));
$this->assertEquals('0000123JKL', $insertedKeys[0]->tm_key->key);
$this->assertEquals('123', $insertedKeys[0]->uid);
/*
* we expect this result
*
* [
*
* {"tm":true,"glos":false,"owner":true,"key":"0000123ABC","name":"My ABC","r":"1","w":"1","uid_transl":123,"uid_rev":null,"r_transl":1,"w_transl":1,"r_rev":null,"w_rev":null,"source":null,"target":null},
*
* {"tm":true,"glos":false,"owner":true,"key":"0000123GHI","name":"My GHI","r":"1","w":"1","uid_transl":null,"uid_rev":null,"r_transl":null,"w_transl":null,"r_rev":null,"w_rev":null,"source":null,"target":null},
*
* {"tm":true,"glos":false,"owner":false,"key":"0000123JKL","name":"My JKL","r":null,"w":null,"uid_transl":123,"uid_rev":null,"r_transl":1,"w_transl":0,"r_rev":null,"w_rev":null,"source":null,"target":null}
*
* ]
*
*/
$this->assertTrue(is_array($resultMerge));
$this->assertEquals(3, count($resultMerge));
/**
* @var $firstKey TmKeyManagement_TmKeyStruct
*/
$firstKey = $resultMerge[0];
/**
* @var $secondKey TmKeyManagement_TmKeyStruct
*/
$secondKey = $resultMerge[1];
/**
* @var $thirdKey TmKeyManagement_TmKeyStruct
*/
$thirdKey = $resultMerge[2];
$this->assertEquals("0000123ABC", $firstKey->key);
$this->assertTrue($firstKey->r_transl);
$this->assertTrue($firstKey->w_transl);
$this->assertEquals(123, $firstKey->uid_transl);
$this->assertEquals(1, $firstKey->owner);
$this->assertEquals(1, $firstKey->r);
$this->assertEquals(1, $firstKey->w);
//This key must be untouched because client sent hashed
$this->assertEquals("0000123GHI", $secondKey->key);
$this->assertNull($secondKey->r_transl);
$this->assertNull($secondKey->w_transl);
$this->assertNull($secondKey->uid_transl);
$this->assertEquals(1, $secondKey->owner);
$this->assertEquals(1, $secondKey->r);
$this->assertEquals(1, $secondKey->w);
$this->assertEquals("0000123JKL", $thirdKey->key);
$this->assertTrue($thirdKey->r_transl);
$this->assertFalse($thirdKey->w_transl);
$this->assertEquals(123, $thirdKey->uid_transl);
$this->assertFalse($thirdKey->owner);
$this->assertNull($thirdKey->r);
$this->assertNull($thirdKey->w);
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project?
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid: " . $_tmKey['key'], -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
//check for glossary files and tmx and put them in front of the list
$infoFile = DetectProprietaryXliff::getInfo($fileName);
if (DetectProprietaryXliff::getMemoryFileType()) {
//found TMX, enable language checking routines
if (DetectProprietaryXliff::isTMXFile()) {
$this->checkTMX = 1;
}
//not used at moment but needed if we want to do a poll for status
if (DetectProprietaryXliff::isGlossaryFile()) {
$this->checkGlossary = 1;
}
//prepend in front of the list
array_unshift($sortedFiles, $fileName);
} else {
//append at the end of the list
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = $this->uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
//we are going to access the storage, get model object to manipulate it
$this->fileStorage = new FilesStorage();
$linkFiles = $this->fileStorage->getHashesFromDir($this->uploadDir);
/*
loop through all input files to
1) upload TMX and Glossaries
*/
try {
$this->_pushTMXToMyMemory();
} catch (Exception $e) {
//exit project creation
return false;
}
//TMX Management
/*
loop through all input files to
2)convert, in case, non standard XLIFF files to a format that Matecat understands
Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
*/
foreach ($this->projectStructure['array_files'] as $fileName) {
/*
Conversion Enforce
Checking Extension is no more sufficient, we want check content
$enforcedConversion = true; //( if conversion is enabled )
*/
$isAFileToConvert = $this->isConversionToEnforce($fileName);
//if it's one of the listed formats or conversion is not enabled in first place
if (!$isAFileToConvert) {
/*
filename is already an xliff and it's in upload directory
we have to make a cache package from it to avoid altering the original path
*/
//get file
$filePathName = "{$this->uploadDir}/{$fileName}";
//calculate hash + add the fileName, if i load 3 equal files with the same content
// they will be squashed to the last one
$sha1 = sha1_file($filePathName);
//make a cache package (with work/ only, emtpy orig/)
$this->fileStorage->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
//put reference to cache in upload dir to link cache to session
$this->fileStorage->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken'], $fileName);
//add newly created link to list
$linkFiles['conversionHashes']['sha'][] = $sha1 . "|" . $this->projectStructure['source_language'];
$linkFiles['conversionHashes']['fileName'][$sha1 . "|" . $this->projectStructure['source_language']][] = $fileName;
//when the same sdlxliff is uploaded more than once with different names
$linkFiles['conversionHashes']['sha'] = array_unique($linkFiles['conversionHashes']['sha']);
unset($sha1);
}
}
//now, upload dir contains only hash-links
//we start copying files to "file" dir, inserting metadata in db and extracting segments
foreach ($linkFiles['conversionHashes']['sha'] as $linkFile) {
//converted file is inside cache directory
//get hash from file name inside UUID dir
$hashFile = FilesStorage::basename_fix($linkFile);
$hashFile = explode('|', $hashFile);
//use hash and lang to fetch file from package
$cachedXliffFilePathName = $this->fileStorage->getXliffFromCache($hashFile[0], $hashFile[1]);
//get sha
$sha1_original = $hashFile[0];
//associate the hash to the right file in upload directory
//get original file name, to insert into DB and cp in storage
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
$_originalFileName = $linkFiles['conversionHashes']['fileName'][$linkFile];
unset($hashFile);
if (!file_exists($cachedXliffFilePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = FilesStorage::pathinfo_fix($cachedXliffFilePathName);
if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$yearMonthPath = date_create($this->projectStructure['create_date'])->format('Ymd');
$fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
//PLEASE NOTE, this can be an array when the same file added more
// than once and with different names
foreach ($_originalFileName as $originalFileName) {
$mimeType = FilesStorage::pathinfo_fix($originalFileName, PATHINFO_EXTENSION);
$fid = insertFile($this->projectStructure, $originalFileName, $mimeType, $fileDateSha1Path);
//move the file in the right directory from the packages to the file dir
$this->fileStorage->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid, $originalFileName);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments(file_get_contents($cachedXliffFilePathName), $fid);
}
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$originalFileName}.");
$this->fileStorage->deleteHashFromUploadDir($this->uploadDir, $linkFile);
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$originalFileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$originalFileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$originalFileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} elseif ($e->getCode() == -13) {
$this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return null;
// SEVERE EXCEPTION we can not write to disk!! Break project creation
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$originalFileName}. Please try again.");
}
Log::doLog($e->getMessage());
Log::doLog($e->getTraceAsString());
}
}
//end of conversion hash-link loop
try {
$this->_zipFileHandling($linkFiles);
} catch (Exception $e) {
//exit project creation
return false;
}
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
if (!$this->_doCheckForErrors()) {
//exit project creation
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
try {
$rows = $this->dbHandler->fetch_array($query_visible_segments);
} catch (PDOException $e) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: {$e->getMessage()} - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($this->uploadDir);
if (is_dir($this->uploadDir . '_converted')) {
Utils::deleteDir($this->uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->projectStructure['dqf_key']);
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
}
protected function _set($config)
{
$this->result['errors'] = array();
$tm_keys = $this->job_info['tm_keys'];
if (self::isRevision()) {
$this->userRole = TmKeyManagement_Filter::ROLE_REVISOR;
}
//get TM keys with read grants
$tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($tm_keys, 'w', 'glos', $this->uid, $this->userRole);
if (empty($tm_keys)) {
$APIKeySrv = new TMSService();
$newUser = (object) $APIKeySrv->createMyMemoryKey();
//throws exception
//TODO take only for hystorical reason
updateTranslatorJob($this->id_job, $newUser);
//fallback
$config['id_user'] = $newUser->id;
$new_key = TmKeyManagement_TmKeyManagement::getTmKeyStructure();
$new_key->tm = 1;
$new_key->glos = 1;
$new_key->key = $newUser->key;
$new_key->owner = $this->userMail == $this->job_info['owner'];
if (!$new_key->owner) {
$new_key->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['r']} = 1;
$new_key->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['w']} = 1;
} else {
$new_key->r = 1;
$new_key->w = 1;
}
if ($new_key->owner) {
//do nothing, this is a greedy if
} elseif ($this->userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) {
$new_key->uid_transl = $this->uid;
} elseif ($this->userRole == TmKeyManagement_Filter::ROLE_REVISOR) {
$new_key->uid_rev = $this->uid;
}
//create an empty array
$tm_keys = array();
//append new key
$tm_keys[] = $new_key;
//put the key in the job
TmKeyManagement_TmKeyManagement::setJobTmKeys($this->id_job, $this->password, $tm_keys);
//put the key in the user keiring
if ($this->userIsLogged) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newMemoryKey->tm_key = $new_key;
$newMemoryKey->uid = $this->uid;
$mkDao = new TmKeyManagement_MemoryKeyDao(Database::obtain());
$mkDao->create($newMemoryKey);
}
}
$config['segment'] = CatUtils::view2rawxliff($config['segment']);
$config['translation'] = CatUtils::view2rawxliff($config['translation']);
$config['prop'] = json_encode(CatUtils::getTMProps($this->job_info));
//prepare the error report
$set_code = array();
//set the glossary entry for each key with write grants
if (count($tm_keys)) {
/**
* @var $tm_keys TmKeyManagement_TmKeyStruct[]
*/
foreach ($tm_keys as $tm_key) {
$config['id_user'] = $tm_key->key;
$TMS_RESULT = $this->_TMS->set($config);
$set_code[] = $TMS_RESULT;
}
}
$set_successful = true;
if (array_search(false, $set_code, true)) {
//There's an error, for now skip, let's assume that are not errors
$set_successful = false;
}
if ($set_successful) {
// Often the get method after a set is not in real time, so return the same values ( FAKE )
// $TMS_GET_RESULT = $this->_TMS->get($config)->get_glossary_matches_as_array();
// $this->result['data']['matches'] = $TMS_GET_RESULT;
$this->result['data']['matches'] = array($config['segment'] => array(array('segment' => $config['segment'], 'translation' => $config['translation'], 'last_update_date' => date_create()->format('Y-m-d H:i:m'), 'last_updated_by' => "Matecat user", 'created_by' => "Matecat user", 'target_note' => $config['tnote'])));
if (isset($new_key)) {
$this->result['data']['created_tm_key'] = true;
}
} else {
$this->result['errors'][] = array("code" => -1, "message" => "We got an error, please try again.");
}
}
public function createProject()
{
// project name sanitize
$oldName = $this->projectStructure['project_name'];
$this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
if ($this->projectStructure['project_name'] == false) {
$this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
return false;
}
// create project
$this->projectStructure['ppassword'] = $this->_generatePassword();
$this->projectStructure['user_ip'] = Utils::getRealIpAddr();
$this->projectStructure['id_customer'] = 'translated_user';
$this->projectStructure['id_project'] = insertProject($this->projectStructure);
//create user (Massidda 2013-01-24)
//check if all the keys are valid MyMemory keys
if (!empty($this->projectStructure['private_tm_key'])) {
foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
$this->tmxServiceWrapper->setTmKey($_tmKey['key']);
try {
$keyExists = $this->tmxServiceWrapper->checkCorrectKey();
if (!isset($keyExists) || $keyExists === false) {
Log::doLog(__METHOD__ . " -> TM key is not valid.");
throw new Exception("TM key is not valid.", -4);
}
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//set the first key as primary
$this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
}
//check if the MyMemory keys provided by the user are already associated to him.
if ($this->projectStructure['userIsLogged']) {
$mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
$searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$searchMemoryKey->uid = $this->projectStructure['uid'];
$userMemoryKeys = $mkDao->read($searchMemoryKey);
$userTmKeys = array();
$memoryKeysToBeInserted = array();
//extract user tm keys
foreach ($userMemoryKeys as $_memoKey) {
/**
* @var $_memoKey TmKeyManagement_MemoryKeyStruct
*/
$userTmKeys[] = $_memoKey->tm_key->key;
}
foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
if (!in_array($_tmKey['key'], $userTmKeys)) {
$newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
$newTmKey = new TmKeyManagement_TmKeyStruct();
$newTmKey->key = $_tmKey['key'];
$newTmKey->tm = true;
$newTmKey->glos = true;
//TODO: take this from input
$newTmKey->name = $_tmKey['name'];
$newMemoryKey->tm_key = $newTmKey;
$newMemoryKey->uid = $this->projectStructure['uid'];
$memoryKeysToBeInserted[] = $newMemoryKey;
} else {
Log::doLog('skip insertion');
}
}
try {
$mkDao->createList($memoryKeysToBeInserted);
} catch (Exception $e) {
Log::doLog($e->getMessage());
# Here we handle the error, displaying HTML, logging, ...
$output = "<pre>\n";
$output .= $e->getMessage() . "\n\t";
$output .= "</pre>";
Utils::sendErrMailReport($output);
}
}
//the base case is when the user clicks on "generate private TM" button:
//a (user, pass, key) tuple is generated and can be inserted
//if it comes with it's own key without querying the creation API, create a (key,key,key) user
if (empty($this->projectStructure['private_tm_user'])) {
$this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
$this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
}
insertTranslator($this->projectStructure);
}
//sort files in order to process TMX first
$sortedFiles = array();
foreach ($this->projectStructure['array_files'] as $fileName) {
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//found TMX, enable language checking routines
$this->checkTMX = 1;
array_unshift($sortedFiles, $fileName);
} else {
array_push($sortedFiles, $fileName);
}
}
$this->projectStructure['array_files'] = $sortedFiles;
unset($sortedFiles);
$uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
//fetch cache links, created by converter, from upload directory
$linkFiles = scandir($uploadDir);
//remove dir hardlinks, as uninteresting, as weel as regular files; only hash-links
foreach ($linkFiles as $k => $linkFile) {
if (strpos($linkFile, '.') !== false or strpos($linkFile, '|') === false) {
unset($linkFiles[$k]);
}
}
/*
loop through all input files to
1)upload TMX
2)convert, in case, non standard XLIFF files to a format that Matecat understands
Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
*/
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
//load it into MyMemory; we'll check later on how it went
$file = new stdClass();
$file->file_path = "{$uploadDir}/{$fileName}";
$this->tmxServiceWrapper->setName($fileName);
$this->tmxServiceWrapper->setFile(array($file));
try {
$this->tmxServiceWrapper->addTmxInMyMemory();
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
return false;
}
//in any case, skip the rest of the loop, go to the next file
continue;
}
/*
Conversion Enforce
Checking Extension is no more sufficient, we want check content if this is an idiom xlf file type, conversion are enforced
$enforcedConversion = true; //( if conversion is enabled )
*/
$isAnXliffToConvert = $this->isConversionToEnforce($fileName);
//we are going to access the storage, get model object to manipulate it
$fs = new FilesStorage();
//if it's one of the listed formats or conversion is not enabled in first place
if (!$isAnXliffToConvert) {
/*
filename is already an xliff and it's in upload directory
we have to make a cache package from it to avoid altering the original path
*/
//get file
$filePathName = "{$uploadDir}/{$fileName}";
//calculate hash + add the fileName, if i load 3 equal files with the same content
// they will be squashed to the last one
$sha1 = sha1(file_get_contents($filePathName) . $filePathName);
//make a cache package (with work/ only, emtpy orig/)
$fs->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
//put reference to cache in upload dir to link cache to session
$fs->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken']);
//add newly created link to list
$linkFiles[] = $sha1 . "|" . $this->projectStructure['source_language'];
unset($sha1);
}
}
//now, upload dir contains only hash-links
//we start copying files to "file" dir, inserting metadata in db and extracting segments
foreach ($linkFiles as $linkFile) {
//converted file is inside cache directory
//get hash from file name inside UUID dir
$hashFile = basename($linkFile);
$hashFile = explode('|', $hashFile);
//use hash and lang to fetch file from package
$xliffFilePathName = $fs->getXliffFromCache($hashFile[0], $hashFile[1]);
//get sha
$sha1_original = $hashFile[0];
//get original file name
$originalFilePathName = $fs->getOriginalFromCache($hashFile[0], $hashFile[1]);
$raw_originalFilePathName = explode(DIRECTORY_SEPARATOR, $originalFilePathName);
$fileName = array_pop($raw_originalFilePathName);
unset($hashFile);
if (!file_exists($xliffFilePathName)) {
$this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
}
try {
$info = pathinfo($xliffFilePathName);
if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
throw new Exception("Failed to find Xliff - no segments found", -3);
}
$mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
$yearMonthPath = date_create()->format("Ymd");
$fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
$fid = insertFile($this->projectStructure, $fileName, $mimeType, $fileDateSha1Path);
//move the file in the right directory from the packages to the file dir
$fs->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid);
$this->projectStructure['file_id_list']->append($fid);
$this->_extractSegments(file_get_contents($xliffFilePathName), $fid);
} catch (Exception $e) {
if ($e->getCode() == -1) {
$this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
$fs->deleteHashFromUploadDir($uploadDir, $linkFile);
} elseif ($e->getCode() == -2) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
} elseif ($e->getCode() == -3) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
} elseif ($e->getCode() == -4) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
} elseif ($e->getCode() == -11) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
} elseif ($e->getCode() == -12) {
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
} elseif ($e->getCode() == -13) {
$this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
Log::doLog($e->getMessage());
return null;
// SEVERE EXCEPTION we can not write to disk!! Break project creation
} else {
//mysql insert Blob Error
$this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$fileName}. Please try again.");
}
Log::doLog($e->getMessage());
}
}
//end of hash-link loop
//check if the files language equals the source language. If not, set an error message.
if (!$this->projectStructure['skip_lang_validation']) {
$this->validateFilesLanguages();
}
/****************/
//loop again through files to check to check for TMX loading
foreach ($this->projectStructure['array_files'] as $fileName) {
//if TMX,
if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
$this->tmxServiceWrapper->setName($fileName);
$result = array();
//is the TM loaded?
//wait until current TMX is loaded
while (true) {
try {
$result = $this->tmxServiceWrapper->tmxUploadStatus();
if ($result['completed']) {
//"$fileName" has been loaded into MyMemory"
//exit the loop
break;
}
//"waiting for "$fileName" to be loaded into MyMemory"
sleep(3);
} catch (Exception $e) {
$this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
//exit project creation
return false;
}
}
//once the language is loaded, check if language is compliant (unless something useful has already been found)
if (1 == $this->checkTMX) {
//get localized target languages of TM (in case it's a multilingual TM)
$tmTargets = explode(';', $result['data']['target_lang']);
//indicates if something has been found for current memory
$found = false;
//compare localized target languages array (in case it's a multilingual project) to the TM supplied
//if nothing matches, then the TM supplied can't have matches for this project
//create an empty var and add the source language too
$project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
foreach ($project_languages as $projectTarget) {
if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
$found = true;
break;
}
}
//if this TM matches the project lagpair and something has been found
if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
//the TMX is good to go
$this->checkTMX = 0;
} elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
/*
* This means that the TMX has a srclang as specification in the header. Warn the user.
* Ex:
* <header creationtool="SDL Language Platform"
* creationtoolversion="8.0"
* datatype="rtf"
* segtype="sentence"
* adminlang="DE-DE"
* srclang="DE-DE" />
*/
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
$this->checkTMX = 0;
Log::doLog($this->projectStructure['result']);
}
}
}
}
if (1 == $this->checkTMX) {
//this means that noone of uploaded TMX were usable for this project. Warn the user.
$this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
Log::doLog($this->projectStructure['result']);
return false;
}
if (!empty($this->projectStructure['result']['errors'])) {
Log::doLog("Project Creation Failed. Sent to Output all errors.");
Log::doLog($this->projectStructure['result']['errors']);
return false;
}
//Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
//create job
if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
$owner = $_SESSION['cid'];
} else {
$_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
//default user
$owner = '';
}
$isEmptyProject = false;
//Throws exception
try {
$this->_createJobs($this->projectStructure, $owner);
//FIXME for project with pre translation this query is not enough,
//we need compare the number of segments with translations, but take an eye to the opensource
$query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
$string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
$query_visible_segments = sprintf($query_visible_segments, $string_file_list);
$rows = $this->dbHandler->fetch_array($query_visible_segments);
if (!$rows) {
Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
}
if ($rows[0]['cattool_segments'] == 0) {
Log::doLog("Segment Search: No segments in this project - \n");
$isEmptyProject = true;
}
} catch (Exception $ex) {
$this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
return false;
}
try {
Utils::deleteDir($uploadDir);
if (is_dir($uploadDir . '_converted')) {
Utils::deleteDir($uploadDir . '_converted');
}
} catch (Exception $e) {
$output = "<pre>\n";
$output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::doLog($output);
Utils::sendErrMailReport($output, $e->getMessage());
}
$this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
if ($isEmptyProject) {
$this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
}
$this->projectStructure['result']['code'] = 1;
$this->projectStructure['result']['data'] = "OK";
$this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
$this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
$this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
$this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
$this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
$this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
$this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
$this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
$this->projectStructure['result']['status'] = $this->projectStructure['status'];
$this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
$query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n ";
$query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
$project_summary = $this->dbHandler->fetch_array($query_project_summary);
$update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n ";
$update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
$this->dbHandler->query($update_project_count);
// Log::doLog( $this->projectStructure );
//create Project into DQF queue
if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
$dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
$dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
$dqfProjectStruct->project_id = $this->projectStructure['id_project'];
$dqfProjectStruct->name = $this->projectStructure['project_name'];
$dqfProjectStruct->source_language = $this->projectStructure['source_language'];
$dqfQueue = new Analysis_DqfQueueHandler();
try {
$dqfQueue->createProject($dqfProjectStruct);
//for each job, push a task into AMQ's DQF queue
foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
/**
* @var $dqfTaskStruct DQF_DqfTaskStruct
*/
$dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
$dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
$dqfTaskStruct->project_id = $this->projectStructure['id_project'];
$dqfTaskStruct->task_id = $jobID;
$dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
$dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
$dqfQueue->createTask($dqfTaskStruct);
}
} catch (Exception $exn) {
$output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
Log::doLog($output);
Utils::sendErrMailReport($output, $exn->getMessage());
}
}
}
