/** * When Called it perform the controller action to retrieve/manipulate data * * @return void */ function doAction() { //if some error occured, stop execution. if (count(@$this->result['errors'])) { return; } try { $tmService = new TMSService(); $tmService->setTmKey($this->key); //validate the key try { $keyExists = $tmService->checkCorrectKey(); } catch (Exception $e) { /* PROVIDED KEY IS NOT VALID OR WRONG, $keyExists IS NOT SET */ Log::doLog($e->getMessage()); } if (!isset($keyExists) || $keyExists === false) { Log::doLog(__METHOD__ . " -> TM key is not valid."); throw new Exception("TM key is not valid.", -4); } $tmKeyStruct = new TmKeyManagement_TmKeyStruct(); $tmKeyStruct->key = $this->key; $tmKeyStruct->name = $this->description; $tmKeyStruct->tm = true; $tmKeyStruct->glos = true; $mkDao = new TmKeyManagement_MemoryKeyDao(Database::obtain()); $memoryKeyToUpdate = new TmKeyManagement_MemoryKeyStruct(); $memoryKeyToUpdate->uid = $this->uid; $memoryKeyToUpdate->tm_key = $tmKeyStruct; switch ($this->exec) { case 'delete': $userMemoryKeys = $mkDao->disable($memoryKeyToUpdate); break; case 'update': $userMemoryKeys = $mkDao->update($memoryKeyToUpdate); break; case 'newKey': $userMemoryKeys = $mkDao->create($memoryKeyToUpdate); break; default: throw new Exception("Unexpected Exception", -4); } if (!$userMemoryKeys) { throw new Exception("This key wasn't found in your keyring.", -3); } } catch (Exception $e) { $this->result['data'] = 'KO'; $this->result['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); } }
public function createProject() { // project name sanitize $oldName = $this->projectStructure['project_name']; $this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']); if ($this->projectStructure['project_name'] == false) { $this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!"); return false; } // create project $this->projectStructure['ppassword'] = $this->_generatePassword(); $this->projectStructure['user_ip'] = Utils::getRealIpAddr(); $this->projectStructure['id_customer'] = 'translated_user'; $this->projectStructure['id_project'] = insertProject($this->projectStructure); //create user (Massidda 2013-01-24) //check if all the keys are valid MyMemory keys if (!empty($this->projectStructure['private_tm_key'])) { foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) { $this->tmxServiceWrapper->setTmKey($_tmKey['key']); try { $keyExists = $this->tmxServiceWrapper->checkCorrectKey(); if (!isset($keyExists) || $keyExists === false) { Log::doLog(__METHOD__ . " -> TM key is not valid."); throw new Exception("TM key is not valid.", -4); } } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); return false; } //set the first key as primary $this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']); } //check if the MyMemory keys provided by the user are already associated to him. if ($this->projectStructure['userIsLogged']) { $mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler); $searchMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $searchMemoryKey->uid = $this->projectStructure['uid']; $userMemoryKeys = $mkDao->read($searchMemoryKey); $userTmKeys = array(); $memoryKeysToBeInserted = array(); //extract user tm keys foreach ($userMemoryKeys as $_memoKey) { /** * @var $_memoKey TmKeyManagement_MemoryKeyStruct */ $userTmKeys[] = $_memoKey->tm_key->key; } foreach ($this->projectStructure['private_tm_key'] as $_tmKey) { if (!in_array($_tmKey['key'], $userTmKeys)) { $newMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $newTmKey = new TmKeyManagement_TmKeyStruct(); $newTmKey->key = $_tmKey['key']; $newTmKey->tm = true; $newTmKey->glos = true; //TODO: take this from input $newTmKey->name = $_tmKey['name']; $newMemoryKey->tm_key = $newTmKey; $newMemoryKey->uid = $this->projectStructure['uid']; $memoryKeysToBeInserted[] = $newMemoryKey; } else { Log::doLog('skip insertion'); } } try { $mkDao->createList($memoryKeysToBeInserted); } catch (Exception $e) { Log::doLog($e->getMessage()); # Here we handle the error, displaying HTML, logging, ... $output = "<pre>\n"; $output .= $e->getMessage() . "\n\t"; $output .= "</pre>"; Utils::sendErrMailReport($output); } } //the base case is when the user clicks on "generate private TM" button: //a (user, pass, key) tuple is generated and can be inserted //if it comes with it's own key without querying the creation API, create a (key,key,key) user if (empty($this->projectStructure['private_tm_user'])) { $this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key']; $this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key']; } insertTranslator($this->projectStructure); } //sort files in order to process TMX first $sortedFiles = array(); foreach ($this->projectStructure['array_files'] as $fileName) { if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) { //found TMX, enable language checking routines $this->checkTMX = 1; array_unshift($sortedFiles, $fileName); } else { array_push($sortedFiles, $fileName); } } $this->projectStructure['array_files'] = $sortedFiles; unset($sortedFiles); $uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken']; foreach ($this->projectStructure['array_files'] as $fileName) { //if TMX, if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) { $file = new stdClass(); $file->file_path = "{$uploadDir}/{$fileName}"; $this->tmxServiceWrapper->setName($fileName); $this->tmxServiceWrapper->setFile(array($file)); try { $this->tmxServiceWrapper->addTmxInMyMemory(); } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); return false; } //in any case, skip the rest of the loop, go to the next file continue; } /** * Conversion Enforce * * we have to know if a file can be found in _converted directory * * Check Extension no more sufficient, we want check content * if this is an idiom xlf file type, conversion are enforced * $enforcedConversion = true; //( if conversion is enabled ) */ $isAConvertedFile = true; try { $fileType = DetectProprietaryXliff::getInfo(INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'] . DIRECTORY_SEPARATOR . $fileName); if (DetectProprietaryXliff::isXliffExtension()) { if (INIT::$CONVERSION_ENABLED) { //conversion enforce if (!INIT::$FORCE_XLIFF_CONVERSION) { //ONLY IDIOM is forced to be converted //if file is not proprietary like idiom AND Enforce is disabled //we take it as is if (!$fileType['proprietary']) { $isAConvertedFile = false; //ok don't convert a standard sdlxliff } } else { //if conversion enforce is active //we force all xliff files but not files produced by SDL Studio because we can handle them if ($fileType['proprietary_short_name'] == 'trados') { $isAConvertedFile = false; //ok don't convert a standard sdlxliff } } } elseif ($fileType['proprietary']) { /** * Application misconfiguration. * upload should not be happened, but if we are here, raise an error. * @see upload.class.php * */ $this->projectStructure['result']['errors'][] = array("code" => -8, "message" => "Proprietary xlf format detected. Not able to import this XLIFF file. ({$fileName})"); setcookie("upload_session", "", time() - 10000); return -1; //stop execution } elseif (!$fileType['proprietary']) { $isAConvertedFile = false; //ok don't convert a standard sdlxliff } } } catch (Exception $e) { Log::doLog($e->getMessage()); } $mimeType = pathinfo($fileName, PATHINFO_EXTENSION); $original_content = ""; /* if it's not one of the listed formats (or it is, but you had to convert it anyway), and conversion is enabled in first place */ if ($isAConvertedFile) { //converted file is inside "_converted" directory $fileDir = $uploadDir . '_converted'; $original_content = file_get_contents("{$uploadDir}/{$fileName}"); $sha1_original = sha1($original_content); $original_content = gzdeflate($original_content, 5); //file name is a xliff converted like: 'a_word_document.doc.sdlxliff' $real_fileName = $fileName . '.sdlxliff'; } else { //filename is already an xliff and it is in a canonical normal directory $sha1_original = ""; $fileDir = $uploadDir; $real_fileName = $fileName; } $filePathName = $fileDir . DIRECTORY_SEPARATOR . $real_fileName; if (!file_exists($filePathName)) { $this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload."); } try { $info = pathinfo($filePathName); if ($info['extension'] == 'xliff' || $info['extension'] == 'sdlxliff' || $info['extension'] == 'xlf') { $contents = file_get_contents($filePathName); } else { throw new Exception("Failed to find Xliff - no segments found", -3); } $fid = insertFile($this->projectStructure, $fileName, $mimeType, $contents, $sha1_original, $original_content); $this->projectStructure['file_id_list']->append($fid); $this->_extractSegments($contents, $fid); unset($contents); //free memory //Log::doLog( $this->projectStructure['segments'] ); } catch (Exception $e) { if ($e->getCode() == -1) { $this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}."); } elseif ($e->getCode() == -2) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}"); } elseif ($e->getCode() == -3) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk"); } elseif ($e->getCode() == -4) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )"); } elseif ($e->getCode() == -11) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied"); } elseif ($e->getCode() == -12) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database"); } else { //mysql insert Blob Error $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File is Too large. ( {$fileName} )"); } Log::doLog($e->getMessage()); } //exit; } //check if the files language equals the source language. If not, set an error message. if (!$this->projectStructure['skip_lang_validation']) { $this->validateFilesLanguages(); } /****************/ //loop again through files to check to check for TMX loading foreach ($this->projectStructure['array_files'] as $fileName) { //if TMX, if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) { $this->tmxServiceWrapper->setName($fileName); $result = array(); //is the TM loaded? //wait until current TMX is loaded while (true) { try { $result = $this->tmxServiceWrapper->tmxUploadStatus(); if ($result['completed']) { //"$fileName" has been loaded into MyMemory" //exit the loop break; } //"waiting for "$fileName" to be loaded into MyMemory" sleep(3); } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString()); //exit project creation return false; } } //once the language is loaded, check if language is compliant (unless something useful has already been found) if (1 == $this->checkTMX) { //get localized target languages of TM (in case it's a multilingual TM) $tmTargets = explode(';', $result['data']['target_lang']); //indicates if something has been found for current memory $found = false; //compare localized target languages array (in case it's a multilingual project) to the TM supplied //if nothing matches, then the TM supplied can't have matches for this project //create an empty var and add the source language too $project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']); foreach ($project_languages as $projectTarget) { if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) { $found = true; break; } } //if this TM matches the project lagpair and something has been found if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) { //the TMX is good to go $this->checkTMX = 0; } elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) { /* * This means that the TMX has a srclang as specification in the header. Warn the user. * Ex: * <header creationtool="SDL Language Platform" * creationtoolversion="8.0" * datatype="rtf" * segtype="sentence" * adminlang="DE-DE" * srclang="DE-DE" /> */ $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file."); $this->checkTMX = 0; Log::doLog($this->projectStructure['result']); } } } } if (1 == $this->checkTMX) { //this means that noone of uploaded TMX were usable for this project. Warn the user. $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project."); Log::doLog($this->projectStructure['result']); return false; } if (!empty($this->projectStructure['result']['errors'])) { Log::doLog("Project Creation Failed. Sent to Output all errors."); Log::doLog($this->projectStructure['result']['errors']); return false; } //Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) ); //create job if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) { $owner = $_SESSION['cid']; } else { $_SESSION['_anonym_pid'] = $this->projectStructure['id_project']; //default user $owner = ''; } $isEmptyProject = false; //Throws exception try { $this->_createJobs($this->projectStructure, $owner); //FIXME for project with pre translation this query is not enough, //we need compare the number of segments with translations, but take an eye to the opensource $query_visible_segments = "SELECT count(*) as cattool_segments\n FROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1"; $string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy()); $query_visible_segments = sprintf($query_visible_segments, $string_file_list); $rows = $this->dbHandler->fetch_array($query_visible_segments); if (!$rows) { Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n"); throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5); } if ($rows[0]['cattool_segments'] == 0) { Log::doLog("Segment Search: No segments in this project - \n"); $isEmptyProject = true; } } catch (Exception $ex) { $this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )"); return false; } try { Utils::deleteDir($uploadDir); if (is_dir($uploadDir . '_converted')) { Utils::deleteDir($uploadDir . '_converted'); } } catch (Exception $e) { $output = "<pre>\n"; $output .= " - Exception: " . print_r($e->getMessage(), true) . "\n"; $output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n"; $output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n"; $output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n"; $output .= "\n\t"; $output .= "Aborting...\n"; $output .= "</pre>"; Log::doLog($output); Utils::sendErrMailReport($output, $e->getMessage()); } $this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE; if ($isEmptyProject) { $this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY; } $this->projectStructure['result']['code'] = 1; $this->projectStructure['result']['data'] = "OK"; $this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword']; $this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass']; $this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list']; $this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments']; $this->projectStructure['result']['id_project'] = $this->projectStructure['id_project']; $this->projectStructure['result']['project_name'] = $this->projectStructure['project_name']; $this->projectStructure['result']['source_language'] = $this->projectStructure['source_language']; $this->projectStructure['result']['target_language'] = $this->projectStructure['target_language']; $this->projectStructure['result']['status'] = $this->projectStructure['status']; $this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files']; $query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n "; $query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']); $project_summary = $this->dbHandler->fetch_array($query_project_summary); /** * TODO: remove after queue implementation */ // if( 0 && $project_summary[0]['project_segments'] > 50000 ) { // $this->projectStructure[ 'status' ] = Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE; // // $msg = " // WARNING: a project with more than 50.000 segments was created. ( " . $project_summary[0]['project_segments'] . " )\n" . // var_export( $this->projectStructure[ 'result' ], true ) . "\n\n" . // " " . // var_export( $project_summary[0] , true ) . "\n"; // // Utils::sendErrMailReport( $msg, "Alert: Project Creation Abort. - " ); // // } $update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n "; $update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']); $this->dbHandler->query($update_project_count); // Log::doLog( $this->projectStructure ); //create Project into DQF queue if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) { $dqfProjectStruct = DQF_DqfProjectStruct::getStruct(); $dqfProjectStruct->api_key = $this->projectStructure['dqf_key']; $dqfProjectStruct->project_id = $this->projectStructure['id_project']; $dqfProjectStruct->name = $this->projectStructure['project_name']; $dqfProjectStruct->source_language = $this->projectStructure['source_language']; $dqfQueue = new Analysis_DqfQueueHandler(); try { $dqfQueue->createProject($dqfProjectStruct); //for each job, push a task into AMQ's DQF queue foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) { /** * @var $dqfTaskStruct DQF_DqfTaskStruct */ $dqfTaskStruct = DQF_DqfTaskStruct::getStruct(); $dqfTaskStruct->api_key = $this->projectStructure['dqf_key']; $dqfTaskStruct->project_id = $this->projectStructure['id_project']; $dqfTaskStruct->task_id = $jobID; $dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i]; $dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name']; $dqfQueue->createTask($dqfTaskStruct); } } catch (Exception $exn) { $output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage(); Log::doLog($output); Utils::sendErrMailReport($output, $exn->getMessage()); } } // var_dump($this->projectStructure); // exit; }
/** * Merge the keys from CLIENT with those from DATABASE ( jobData ) * * @param string $Json_clientKeys A json_encoded array of objects having the following structure:<br /> * <pre> * array( * 'key' => <private_tm_key>, * 'name' => <tm_name>, * 'r' => true, * 'w' => true * ) * </pre> * @param string $Json_jobKeys A json_encoded array of TmKeyManagement_TmKeyStruct objects * @param string $userRole One of the following strings: "owner", "translator", "revisor" * @param int $uid * * @see TmKeyManagement_TmKeyStruct * * @return array TmKeyManagement_TmKeyStruct[] * * @throws Exception */ public static function mergeJsonKeys($Json_clientKeys, $Json_jobKeys, $userRole = TmKeyManagement_Filter::ROLE_TRANSLATOR, $uid = null) { //we put the already present job keys so they can be checked against the client keys when cycle advances //( jobs has more elements than the client objects ) $clientDecodedJson = json_decode($Json_clientKeys, true); Utils::raiseJsonExceptionError(); $serverDecodedJson = json_decode($Json_jobKeys, true); Utils::raiseJsonExceptionError(); if (!array_key_exists($userRole, TmKeyManagement_Filter::$GRANTS_MAP)) { throw new Exception("Invalid Role Type string.", 4); } $client_tm_keys = array_map(array('self', 'getTmKeyStructure'), $clientDecodedJson); $client_tm_keys = array_map(array('self', 'sanitize'), $client_tm_keys); $job_tm_keys = array_map(array('self', 'getTmKeyStructure'), $serverDecodedJson); $server_reorder_position = array(); $reverse_lookup_client_json = array('pos' => array(), 'elements' => array(), 'unique' => array()); foreach ($client_tm_keys as $_j => $_client_tm_key) { /** * @var $_client_tm_key TmKeyManagement_TmKeyStruct */ //create a reverse lookup $reverse_lookup_client_json['pos'][$_j] = $_client_tm_key->key; $reverse_lookup_client_json['elements'][$_j] = $_client_tm_key; $reverse_lookup_client_json['unique'][$_j] = $_client_tm_key->getCrypt(); if (empty($_client_tm_key->r) && empty($_client_tm_key->w)) { throw new Exception("Please, select Lookup and/or Update to activate your TM in this project", 4); } if (empty($_client_tm_key->key)) { throw new Exception("Invalid Key Provided", 5); } } $uniq_num = count(array_unique($reverse_lookup_client_json['unique'])); if ($uniq_num != count($reverse_lookup_client_json['pos'])) { throw new Exception("A key is already present in this project.", 5); } //update existing job keys foreach ($job_tm_keys as $i => $_job_Key) { /** * @var $_job_Key TmKeyManagement_TmKeyStruct */ $_index_position = array_search($_job_Key->key, $reverse_lookup_client_json['pos']); if (array_search($_job_Key->getCrypt(), $reverse_lookup_client_json['pos']) !== false) { //DO NOTHING //reduce the stack $hashPosition = array_search($_job_Key->getCrypt(), $reverse_lookup_client_json['pos']); unset($reverse_lookup_client_json['pos'][$hashPosition]); unset($reverse_lookup_client_json['elements'][$hashPosition]); //PASS //take the new order $server_reorder_position[$hashPosition] = $_job_Key; } elseif ($_index_position !== false) { // so, here the key exists in client //this is an anonymous user, and a key exists in job if ($_index_position !== false && $uid == null) { //check anonymous user, an anonymous user can not change a not anonymous key if ($userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) { if ($_job_Key->uid_transl != null) { throw new Exception("Anonymous user can not modify existent keys.", 1); } } elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR) { if ($_job_Key->uid_rev != null) { throw new Exception("Anonymous user can not modify existent keys.", 2); } } else { if ($uid == null) { throw new Exception("Anonymous user can not be OWNER", 3); } } } //override the static values $_job_Key->tm = filter_var($reverse_lookup_client_json['elements'][$_index_position]->tm, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); $_job_Key->glos = filter_var($reverse_lookup_client_json['elements'][$_index_position]->glos, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); if ($userRole == TmKeyManagement_Filter::OWNER) { //override grants $_job_Key->r = filter_var($reverse_lookup_client_json['elements'][$_index_position]->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); $_job_Key->w = filter_var($reverse_lookup_client_json['elements'][$_index_position]->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); } elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR || $userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) { //override role specific grants $_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['r']} = filter_var($reverse_lookup_client_json['elements'][$_index_position]->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); $_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['w']} = filter_var($reverse_lookup_client_json['elements'][$_index_position]->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); } //change name if modified if ($_job_Key->name != $reverse_lookup_client_json['elements'][$_index_position]->name) { $_job_Key->name = $reverse_lookup_client_json['elements'][$_index_position]->name; } //set as owner if it is but should be already set // $_job_Key->owner = ( $userRole == TmKeyManagement_Filter::OWNER ); //reduce the stack unset($reverse_lookup_client_json['pos'][$_index_position]); unset($reverse_lookup_client_json['elements'][$_index_position]); //take the new order $server_reorder_position[$_index_position] = $_job_Key; } else { //the key must be deleted if ($userRole == TmKeyManagement_Filter::OWNER) { //override grants $_job_Key->r = null; $_job_Key->w = null; $_job_Key->owner = false; } elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR || $userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) { //override role specific grants $_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['r']} = null; $_job_Key->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['w']} = null; } //remove the uid property if ($userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) { $_job_Key->uid_transl = null; } elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR) { $_job_Key->uid_rev = null; } //if the key is no more linked to someone, don't add to the resultset, else reorder if it is not an owner key. if ($_job_Key->owner || !is_null($_job_Key->uid_transl) || !is_null($_job_Key->uid_rev)) { if (!$_job_Key->owner) { //take the new order, put the deleted key at the end of the array //a position VERY LOW ( 1 Million ) $server_reorder_position[1000000 + $i] = $_job_Key; } else { if ($userRole != TmKeyManagement_Filter::OWNER) { //place on top of the owner keys, preserve the order of owner keys by adding it's normal index position $server_reorder_position[-1000000 + $i] = $_job_Key; } else { // Remove the key!!! //only the owner can remove it's keys } } } } } /* * There are some new keys from client? Add them */ if (!empty($reverse_lookup_client_json['pos'])) { $justCreatedKey = new TmKeyManagement_TmKeyStruct(); foreach ($reverse_lookup_client_json['elements'] as $_pos => $newClientKey) { /** * @var $newClientKey TmKeyManagement_TmKeyStruct */ //set the key value $justCreatedKey->key = $newClientKey->key; //override the static values $justCreatedKey->tm = filter_var($newClientKey->tm, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); $justCreatedKey->glos = filter_var($newClientKey->glos, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); if ($userRole != TmKeyManagement_Filter::OWNER) { $justCreatedKey->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['r']} = filter_var($newClientKey->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); $justCreatedKey->{TmKeyManagement_Filter::$GRANTS_MAP[$userRole]['w']} = filter_var($newClientKey->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); } else { //override grants $justCreatedKey->r = filter_var($newClientKey->r, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); $justCreatedKey->w = filter_var($newClientKey->w, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); } //set the uid property if ($userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) { $justCreatedKey->uid_transl = $uid; } elseif ($userRole == TmKeyManagement_Filter::ROLE_REVISOR) { $justCreatedKey->uid_rev = $uid; } //choose a name instead of null $justCreatedKey->name = $newClientKey->name; //choose an owner instead of null $justCreatedKey->owner = $userRole == TmKeyManagement_Filter::OWNER; //finally append to the job keys!! //take the new order, put the new key at the end of the array //a position VERY LOW, but BEFORE the deleted keys, so it goes not to the end ( 100 hundred thousand ) $server_reorder_position[100000 + $_pos] = $justCreatedKey; if ($uid != null) { //if uid is provided, check for key and try to add to it's memory key ring try { /* * Take the keys of the user */ $_keyDao = new TmKeyManagement_MemoryKeyDao(Database::obtain()); $dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => $uid, 'tm_key' => new TmKeyManagement_TmKeyStruct(array('key' => $justCreatedKey->key)))); $keyList = $_keyDao->read($dh); if (empty($keyList)) { // add the key to a new row struct $dh->uid = $uid; $dh->tm_key = $justCreatedKey; $_keyDao->create($dh); } } catch (Exception $e) { Log::doLog($e->getMessage()); } } } } ksort($server_reorder_position, SORT_NUMERIC); return array_values($server_reorder_position); }
public function doAction() { //Get the guid from the guid if it exists, otherwise set the guid into the cookie if (!isset($_COOKIE['upload_session'])) { setcookie("upload_session", $this->guid, time() + 86400); } else { $this->guid = $_COOKIE['upload_session']; } if (isset($_COOKIE["sourceLang"]) and $_COOKIE["sourceLang"] == "_EMPTY_") { $this->noSourceLangHistory = true; } else { if (!isset($_COOKIE['sourceLang'])) { setcookie("sourceLang", "_EMPTY_", time() + 86400 * 365); $this->noSourceLangHistory = true; } else { if ($_COOKIE["sourceLang"] != "_EMPTY_") { $this->noSourceLangHistory = false; $this->sourceLangHistory = $_COOKIE["sourceLang"]; $this->sourceLangAr = explode('||', urldecode($this->sourceLangHistory)); $tmpSourceAr = array(); $tmpSourceArAs = array(); foreach ($this->sourceLangAr as $key => $lang) { if ($lang != '') { $tmpSourceAr[$lang] = $this->lang_handler->getLocalizedName($lang); $ar = array(); $ar['name'] = $this->lang_handler->getLocalizedName($lang); $ar['code'] = $lang; $ar['selected'] = $key == '0' ? 1 : 0; $ar['direction'] = $this->lang_handler->isRTL(strtolower($lang)) ? 'rtl' : 'ltr'; array_push($tmpSourceArAs, $ar); } } $this->sourceLangAr = $tmpSourceAr; asort($this->sourceLangAr); $this->array_sort_by_column($tmpSourceArAs, 'name'); $this->sourceLangArray = $tmpSourceArAs; } } } if (isset($_COOKIE["targetLang"]) and $_COOKIE["targetLang"] == "_EMPTY_") { $this->noTargetLangHistory = true; } else { if (!isset($_COOKIE['targetLang'])) { setcookie("targetLang", "_EMPTY_", time() + 86400 * 365); $this->noTargetLangHistory = true; } else { if ($_COOKIE["targetLang"] != "_EMPTY_") { $this->noTargetLangHistory = false; $this->targetLangHistory = $_COOKIE["targetLang"]; $this->targetLangAr = explode('||', urldecode($this->targetLangHistory)); $tmpTargetAr = array(); $tmpTargetArAs = array(); foreach ($this->targetLangAr as $key => $lang) { if ($lang != '') { $prova = explode(',', urldecode($lang)); $cl = ""; foreach ($prova as $ll) { $cl .= $this->lang_handler->getLocalizedName($ll) . ','; } $cl = substr_replace($cl, "", -1); $tmpTargetAr[$lang] = $cl; // $tmpTargetAr[$lang] = $this->lang_handler->getLocalizedName($lang,'en'); $ar = array(); $ar['name'] = $cl; $ar['direction'] = $this->lang_handler->isRTL(strtolower($lang)) ? 'rtl' : 'ltr'; $ar['code'] = $lang; $ar['selected'] = $key == '0' ? 1 : 0; array_push($tmpTargetArAs, $ar); } } $this->targetLangAr = $tmpTargetAr; asort($this->targetLangAr); $this->array_sort_by_column($tmpTargetArAs, 'name'); $this->targetLangArray = $tmpTargetArAs; } } } $intDir = INIT::$UPLOAD_REPOSITORY . '/' . $this->guid . '/'; if (!is_dir($intDir)) { mkdir($intDir, 0775, true); } // check if user is logged and generate authURL for logging in $this->doAuth(); $this->generateAuthURL(); list($uid, $cid) = $this->getLoginUserParams(); $engine = new EnginesModel_EngineDAO(Database::obtain()); $engineQuery = new EnginesModel_EngineStruct(); $engineQuery->type = 'MT'; if (@(bool) $_GET['amt'] == true) { $engineQuery->uid = 'all'; } else { $engineQuery->uid = $uid == null ? -1 : $uid; } $engineQuery->active = 1; $this->mt_engines = $engine->read($engineQuery); if ($this->isLoggedIn()) { try { $_keyList = new TmKeyManagement_MemoryKeyDao(Database::obtain()); $dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => @$_SESSION['uid'])); $keyList = $_keyList->read($dh); foreach ($keyList as $memKey) { //all keys are available in this condition ( we are creating a project $this->keyList[] = $memKey->tm_key; } } catch (Exception $e) { Log::doLog($e->getMessage()); } } }
/** * * Get the right su * * @param $match * * @return null|string * @throws Exception */ private function __changeSuggestionSource($match) { $sug_source = $match['created_by']; $key = $match['memory_key']; //suggestion is coming from a public TM if ($sug_source == 'Matecat') { $description = "Public TM"; } elseif (!empty($sug_source) && stripos($sug_source, "MyMemory") === false) { $description = $sug_source; } elseif (preg_match("/[a-f0-9]{8,}/", $key)) { // md5 Key //MyMemory returns the key of the match //Session Enabled $this->checkLogin(); //Session Disabled if ($this->userIsLogged) { //check if the user can see the key. $memoryKey = new TmKeyManagement_MemoryKeyStruct(); $memoryKey->uid = $this->uid; $memoryKey->tm_key = new TmKeyManagement_TmKeyStruct(); $memoryKey->tm_key->key = $key; $memoryKeyDao = new TmKeyManagement_MemoryKeyDao(Database::obtain()); $currentUserMemoryKey = $memoryKeyDao->setCacheTTL(3600)->read($memoryKey); if (count($currentUserMemoryKey) > 0) { //the current user owns the key: show its description $currentUserMemoryKey = $currentUserMemoryKey[0]; $description = $currentUserMemoryKey->tm_key->name; } } } /** * if the description is empty, get cascading default descriptions */ if (empty($description)) { $description = $this->__getDefaultDescription($key); } if (empty($description)) { $description = "No description available"; //this should never be } return $description; }
public function doAction() { $files_found = array(); $lang_handler = Langs_Languages::getInstance(); $data = getSegmentsInfo($this->jid, $this->password); if (empty($data) or $data < 0) { $this->job_not_found = true; //stop execution return; } //retrieve job owner. It will be useful also if the job is archived or cancelled $this->job_owner = $data[0]['job_owner'] != "" ? $data[0]['job_owner'] : "*****@*****.**"; if ($data[0]['status'] == Constants_JobStatus::STATUS_CANCELLED) { $this->job_cancelled = true; //stop execution return; } if ($data[0]['status'] == Constants_JobStatus::STATUS_ARCHIVED) { $this->job_archived = true; //stop execution return; } /* * I prefer to use a programmatic approach to the check for the archive date instead of a pure query * because the query to check "Utils::getArchivableJobs($this->jid)" should be * executed every time a job is loaded ( F5 or CTRL+R on browser ) and it cost some milliseconds ( ~0.1s ) * and it is little heavy for the database. * We use the data we already have from last query and perform * the check on the last translation only if the job is older than 30 days * */ $lastUpdate = new DateTime($data[0]['last_update']); $oneMonthAgo = new DateTime(); $oneMonthAgo->modify('-' . INIT::JOB_ARCHIVABILITY_THRESHOLD . ' days'); if ($lastUpdate < $oneMonthAgo && !$this->job_cancelled) { $lastTranslationInJob = new Datetime(getLastTranslationDate($this->jid)); if ($lastTranslationInJob < $oneMonthAgo) { $res = "job"; $new_status = Constants_JobStatus::STATUS_ARCHIVED; updateJobsStatus($res, $this->jid, $new_status, null, null, $this->password); $this->job_archived = true; } } foreach ($data as $i => $job) { $this->project_status = $job; // get one row values for the project are the same for every row if (empty($this->pname)) { $this->pname = $job['pname']; $this->downloadFileName = $job['pname'] . ".zip"; // will be overwritten below in case of one file job } if (empty($this->last_opened_segment)) { $this->last_opened_segment = $job['last_opened_segment']; } if (empty($this->cid)) { $this->cid = $job['cid']; } if (empty($this->pid)) { $this->pid = $job['pid']; } if (empty($this->create_date)) { $this->create_date = $job['create_date']; } if (empty($this->source_code)) { $this->source_code = $job['source']; } if (empty($this->target_code)) { $this->target_code = $job['target']; } if (empty($this->source)) { $s = explode("-", $job['source']); $source = strtoupper($s[0]); $this->source = $source; $this->source_rtl = $lang_handler->isRTL(strtolower($this->source)) ? ' rtl-source' : ''; } if (empty($this->target)) { $t = explode("-", $job['target']); $target = strtoupper($t[0]); $this->target = $target; $this->target_rtl = $lang_handler->isRTL(strtolower($this->target)) ? ' rtl-target' : ''; } //check if language belongs to supported right-to-left languages if ($job['status'] == Constants_JobStatus::STATUS_ARCHIVED) { $this->job_archived = true; $this->job_owner = $data[0]['job_owner']; } $id_file = $job['id_file']; if (!isset($this->data["{$id_file}"])) { $files_found[] = $job['filename']; } $wStruct = new WordCount_Struct(); $wStruct->setIdJob($this->jid); $wStruct->setJobPassword($this->password); $wStruct->setNewWords($job['new_words']); $wStruct->setDraftWords($job['draft_words']); $wStruct->setTranslatedWords($job['translated_words']); $wStruct->setApprovedWords($job['approved_words']); $wStruct->setRejectedWords($job['rejected_words']); unset($job['id_file']); unset($job['source']); unset($job['target']); unset($job['source_code']); unset($job['target_code']); unset($job['mime_type']); unset($job['filename']); unset($job['jid']); unset($job['pid']); unset($job['cid']); unset($job['tid']); unset($job['pname']); unset($job['create_date']); unset($job['owner']); unset($job['last_opened_segment']); unset($job['new_words']); unset($job['draft_words']); unset($job['translated_words']); unset($job['approved_words']); unset($job['rejected_words']); //For projects created with No tm analysis enabled if ($wStruct->getTotal() == 0 && ($job['status_analysis'] == Constants_ProjectStatus::STATUS_DONE || $job['status_analysis'] == Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE)) { $wCounter = new WordCount_Counter(); $wStruct = $wCounter->initializeJobWordCount($this->jid, $this->password); Log::doLog("BackWard compatibility set Counter."); } $this->job_stats = CatUtils::getFastStatsForJob($wStruct); } //Needed because a just created job has last_opened segment NULL if (empty($this->last_opened_segment)) { $this->last_opened_segment = getFirstSegmentId($this->jid, $this->password); } $this->first_job_segment = $this->project_status['job_first_segment']; $this->last_job_segment = $this->project_status['job_last_segment']; if (count($files_found) == 1) { $this->downloadFileName = $files_found[0]; } /** * get first segment of every file */ $fileInfo = getFirstSegmentOfFilesInJob($this->jid); $TotalPayable = array(); foreach ($fileInfo as &$file) { $file['file_name'] = ZipArchiveExtended::getFileName($file['file_name']); $TotalPayable[$file['id_file']]['TOTAL_FORMATTED'] = $file['TOTAL_FORMATTED']; } $this->firstSegmentOfFiles = json_encode($fileInfo); $this->fileCounter = json_encode($TotalPayable); list($uid, $user_email) = $this->getLoginUserParams(); if (self::isRevision()) { $this->userRole = TmKeyManagement_Filter::ROLE_REVISOR; } elseif ($user_email == $data[0]['job_owner']) { $this->userRole = TmKeyManagement_Filter::OWNER; } else { $this->userRole = TmKeyManagement_Filter::ROLE_TRANSLATOR; } /* * Take the keys of the user */ try { $_keyList = new TmKeyManagement_MemoryKeyDao(Database::obtain()); $dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => $uid)); $keyList = $_keyList->read($dh); } catch (Exception $e) { $keyList = array(); Log::doLog($e->getMessage()); } $reverse_lookup_user_personal_keys = array('pos' => array(), 'elements' => array()); /** * Set these keys as editable for the client * * @var $keyList TmKeyManagement_MemoryKeyStruct[] */ foreach ($keyList as $_j => $key) { /** * @var $_client_tm_key TmKeyManagement_TmKeyStruct */ //create a reverse lookup $reverse_lookup_user_personal_keys['pos'][$_j] = $key->tm_key->key; $reverse_lookup_user_personal_keys['elements'][$_j] = $key; $this->_keyList['totals'][$_j] = new TmKeyManagement_ClientTmKeyStruct($key->tm_key); } /* * Now take the JOB keys */ $job_keyList = json_decode($data[0]['tm_keys'], true); $this->tid = count($job_keyList) > 0; /** * Start this N^2 cycle from keys of the job, * these should be statistically lesser than the keys of the user * * @var $keyList array */ foreach ($job_keyList as $jobKey) { $jobKey = new TmKeyManagement_ClientTmKeyStruct($jobKey); if ($this->isLoggedIn() && count($reverse_lookup_user_personal_keys['pos'])) { /* * If user has some personal keys, check for the job keys if they are present, and obfuscate * when they are not */ $_index_position = array_search($jobKey->key, $reverse_lookup_user_personal_keys['pos']); if ($_index_position !== false) { //i found a key in the job that is present in my database //i'm owner?? and the key is an owner type key? if (!$jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) { $jobKey->r = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['r']}; $jobKey->w = $jobKey->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['w']}; $jobKey = $jobKey->hideKey($uid); } else { if ($jobKey->owner && $this->userRole != TmKeyManagement_Filter::OWNER) { // I'm not the job owner, but i know the key because it is in my keyring // so, i can upload and download TMX, but i don't want it to be removed from job // in tm.html relaxed the control to "key.edit" to enable buttons // $jobKey = $jobKey->hideKey( $uid ); // enable editing } else { if ($jobKey->owner && $this->userRole == TmKeyManagement_Filter::OWNER) { //do Nothing } } } unset($this->_keyList['totals'][$_index_position]); } else { /* * This is not a key of that user, set right and obfuscate */ $jobKey->r = true; $jobKey->w = true; $jobKey = $jobKey->hideKey(-1); } $this->_keyList['job_keys'][] = $jobKey; } else { /* * This user is anonymous or it has no keys in its keyring, obfuscate all */ $jobKey->r = true; $jobKey->w = true; $this->_keyList['job_keys'][] = $jobKey->hideKey(-1); } } //clean unordered keys $this->_keyList['totals'] = array_values($this->_keyList['totals']); /** * Retrieve information about job errors * ( Note: these information are fed by the revision process ) * @see setRevisionController */ $jobQA = new Revise_JobQA($this->jid, $this->password, $wStruct->getTotal()); $jobQA->retrieveJobErrorTotals(); $jobVote = $jobQA->evalJobVote(); $this->qa_data = json_encode($jobQA->getQaData()); $this->qa_overall = $jobVote['minText']; $engine = new EnginesModel_EngineDAO(Database::obtain()); //this gets all engines of the user if ($this->isLoggedIn()) { $engineQuery = new EnginesModel_EngineStruct(); $engineQuery->type = 'MT'; $engineQuery->uid = $uid; $engineQuery->active = 1; $mt_engines = $engine->read($engineQuery); } else { $mt_engines = array(); } // this gets MyMemory $engineQuery = new EnginesModel_EngineStruct(); $engineQuery->type = 'TM'; $engineQuery->active = 1; $tms_engine = $engine->setCacheTTL(3600 * 24 * 30)->read($engineQuery); //this gets MT engine active for the job $engineQuery = new EnginesModel_EngineStruct(); $engineQuery->id = $this->project_status['id_mt_engine']; $engineQuery->active = 1; $active_mt_engine = $engine->setCacheTTL(60 * 10)->read($engineQuery); /* * array_unique cast EnginesModel_EngineStruct to string * * EnginesModel_EngineStruct implements __toString method * */ $this->translation_engines = array_unique(array_merge($active_mt_engine, $tms_engine, $mt_engines)); }
public function testMergeJsonKeys_validInput_clientABCGHIJKL_serverABCGHIDEF() { Tests_DBLoader4Test::resetDB(); //this should not to be, because a client can not send not hashed keys //already present in job as an anonymous user $resultMerge = TmKeyManagement_TmKeyManagement::mergeJsonKeys(self::$client_json_ABC_GHI_JKL, self::$srv_json_ABC_GHI_DEF, TmKeyManagement_Filter::ROLE_TRANSLATOR, 123); $MemoryDao = new TmKeyManagement_MemoryKeyDao(Database::obtain()); $dh = new TmKeyManagement_MemoryKeyStruct(array('uid' => 123)); $insertedKeys = $MemoryDao->read($dh); $this->assertEquals(1, count($insertedKeys)); $this->assertEquals('0000123JKL', $insertedKeys[0]->tm_key->key); $this->assertEquals('123', $insertedKeys[0]->uid); /* * we expect this result * * [ * * {"tm":true,"glos":false,"owner":true,"key":"0000123ABC","name":"My ABC","r":"1","w":"1","uid_transl":123,"uid_rev":null,"r_transl":1,"w_transl":1,"r_rev":null,"w_rev":null,"source":null,"target":null}, * * {"tm":true,"glos":false,"owner":true,"key":"0000123GHI","name":"My GHI","r":"1","w":"1","uid_transl":null,"uid_rev":null,"r_transl":null,"w_transl":null,"r_rev":null,"w_rev":null,"source":null,"target":null}, * * {"tm":true,"glos":false,"owner":false,"key":"0000123JKL","name":"My JKL","r":null,"w":null,"uid_transl":123,"uid_rev":null,"r_transl":1,"w_transl":0,"r_rev":null,"w_rev":null,"source":null,"target":null} * * ] * */ $this->assertTrue(is_array($resultMerge)); $this->assertEquals(3, count($resultMerge)); /** * @var $firstKey TmKeyManagement_TmKeyStruct */ $firstKey = $resultMerge[0]; /** * @var $secondKey TmKeyManagement_TmKeyStruct */ $secondKey = $resultMerge[1]; /** * @var $thirdKey TmKeyManagement_TmKeyStruct */ $thirdKey = $resultMerge[2]; $this->assertEquals("0000123ABC", $firstKey->key); $this->assertTrue($firstKey->r_transl); $this->assertTrue($firstKey->w_transl); $this->assertEquals(123, $firstKey->uid_transl); $this->assertEquals(1, $firstKey->owner); $this->assertEquals(1, $firstKey->r); $this->assertEquals(1, $firstKey->w); //This key must be untouched because client sent hashed $this->assertEquals("0000123GHI", $secondKey->key); $this->assertNull($secondKey->r_transl); $this->assertNull($secondKey->w_transl); $this->assertNull($secondKey->uid_transl); $this->assertEquals(1, $secondKey->owner); $this->assertEquals(1, $secondKey->r); $this->assertEquals(1, $secondKey->w); $this->assertEquals("0000123JKL", $thirdKey->key); $this->assertTrue($thirdKey->r_transl); $this->assertFalse($thirdKey->w_transl); $this->assertEquals(123, $thirdKey->uid_transl); $this->assertFalse($thirdKey->owner); $this->assertNull($thirdKey->r); $this->assertNull($thirdKey->w); }
public function createProject() { // project name sanitize $oldName = $this->projectStructure['project_name']; $this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']); if ($this->projectStructure['project_name'] == false) { $this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!"); return false; } // create project? $this->projectStructure['ppassword'] = $this->_generatePassword(); $this->projectStructure['user_ip'] = Utils::getRealIpAddr(); $this->projectStructure['id_project'] = insertProject($this->projectStructure); //create user (Massidda 2013-01-24) //check if all the keys are valid MyMemory keys if (!empty($this->projectStructure['private_tm_key'])) { foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) { $this->tmxServiceWrapper->setTmKey($_tmKey['key']); try { $keyExists = $this->tmxServiceWrapper->checkCorrectKey(); if (!isset($keyExists) || $keyExists === false) { Log::doLog(__METHOD__ . " -> TM key is not valid."); throw new Exception("TM key is not valid: " . $_tmKey['key'], -4); } } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); return false; } //set the first key as primary $this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']); } //check if the MyMemory keys provided by the user are already associated to him. if ($this->projectStructure['userIsLogged']) { $mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler); $searchMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $searchMemoryKey->uid = $this->projectStructure['uid']; $userMemoryKeys = $mkDao->read($searchMemoryKey); $userTmKeys = array(); $memoryKeysToBeInserted = array(); //extract user tm keys foreach ($userMemoryKeys as $_memoKey) { /** * @var $_memoKey TmKeyManagement_MemoryKeyStruct */ $userTmKeys[] = $_memoKey->tm_key->key; } foreach ($this->projectStructure['private_tm_key'] as $_tmKey) { if (!in_array($_tmKey['key'], $userTmKeys)) { $newMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $newTmKey = new TmKeyManagement_TmKeyStruct(); $newTmKey->key = $_tmKey['key']; $newTmKey->tm = true; $newTmKey->glos = true; //TODO: take this from input $newTmKey->name = $_tmKey['name']; $newMemoryKey->tm_key = $newTmKey; $newMemoryKey->uid = $this->projectStructure['uid']; $memoryKeysToBeInserted[] = $newMemoryKey; } else { Log::doLog('skip insertion'); } } try { $mkDao->createList($memoryKeysToBeInserted); } catch (Exception $e) { Log::doLog($e->getMessage()); # Here we handle the error, displaying HTML, logging, ... $output = "<pre>\n"; $output .= $e->getMessage() . "\n\t"; $output .= "</pre>"; Utils::sendErrMailReport($output); } } //the base case is when the user clicks on "generate private TM" button: //a (user, pass, key) tuple is generated and can be inserted //if it comes with it's own key without querying the creation API, create a (key,key,key) user if (empty($this->projectStructure['private_tm_user'])) { $this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key']; $this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key']; } insertTranslator($this->projectStructure); } //sort files in order to process TMX first $sortedFiles = array(); foreach ($this->projectStructure['array_files'] as $fileName) { //check for glossary files and tmx and put them in front of the list $infoFile = DetectProprietaryXliff::getInfo($fileName); if (DetectProprietaryXliff::getMemoryFileType()) { //found TMX, enable language checking routines if (DetectProprietaryXliff::isTMXFile()) { $this->checkTMX = 1; } //not used at moment but needed if we want to do a poll for status if (DetectProprietaryXliff::isGlossaryFile()) { $this->checkGlossary = 1; } //prepend in front of the list array_unshift($sortedFiles, $fileName); } else { //append at the end of the list array_push($sortedFiles, $fileName); } } $this->projectStructure['array_files'] = $sortedFiles; unset($sortedFiles); $uploadDir = $this->uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken']; //we are going to access the storage, get model object to manipulate it $this->fileStorage = new FilesStorage(); $linkFiles = $this->fileStorage->getHashesFromDir($this->uploadDir); /* loop through all input files to 1) upload TMX and Glossaries */ try { $this->_pushTMXToMyMemory(); } catch (Exception $e) { //exit project creation return false; } //TMX Management /* loop through all input files to 2)convert, in case, non standard XLIFF files to a format that Matecat understands Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow */ foreach ($this->projectStructure['array_files'] as $fileName) { /* Conversion Enforce Checking Extension is no more sufficient, we want check content $enforcedConversion = true; //( if conversion is enabled ) */ $isAFileToConvert = $this->isConversionToEnforce($fileName); //if it's one of the listed formats or conversion is not enabled in first place if (!$isAFileToConvert) { /* filename is already an xliff and it's in upload directory we have to make a cache package from it to avoid altering the original path */ //get file $filePathName = "{$this->uploadDir}/{$fileName}"; //calculate hash + add the fileName, if i load 3 equal files with the same content // they will be squashed to the last one $sha1 = sha1_file($filePathName); //make a cache package (with work/ only, emtpy orig/) $this->fileStorage->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName); //put reference to cache in upload dir to link cache to session $this->fileStorage->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken'], $fileName); //add newly created link to list $linkFiles['conversionHashes']['sha'][] = $sha1 . "|" . $this->projectStructure['source_language']; $linkFiles['conversionHashes']['fileName'][$sha1 . "|" . $this->projectStructure['source_language']][] = $fileName; //when the same sdlxliff is uploaded more than once with different names $linkFiles['conversionHashes']['sha'] = array_unique($linkFiles['conversionHashes']['sha']); unset($sha1); } } //now, upload dir contains only hash-links //we start copying files to "file" dir, inserting metadata in db and extracting segments foreach ($linkFiles['conversionHashes']['sha'] as $linkFile) { //converted file is inside cache directory //get hash from file name inside UUID dir $hashFile = FilesStorage::basename_fix($linkFile); $hashFile = explode('|', $hashFile); //use hash and lang to fetch file from package $cachedXliffFilePathName = $this->fileStorage->getXliffFromCache($hashFile[0], $hashFile[1]); //get sha $sha1_original = $hashFile[0]; //associate the hash to the right file in upload directory //get original file name, to insert into DB and cp in storage //PLEASE NOTE, this can be an array when the same file added more // than once and with different names $_originalFileName = $linkFiles['conversionHashes']['fileName'][$linkFile]; unset($hashFile); if (!file_exists($cachedXliffFilePathName)) { $this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload."); } try { $info = FilesStorage::pathinfo_fix($cachedXliffFilePathName); if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) { throw new Exception("Failed to find Xliff - no segments found", -3); } $yearMonthPath = date_create($this->projectStructure['create_date'])->format('Ymd'); $fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original; //PLEASE NOTE, this can be an array when the same file added more // than once and with different names foreach ($_originalFileName as $originalFileName) { $mimeType = FilesStorage::pathinfo_fix($originalFileName, PATHINFO_EXTENSION); $fid = insertFile($this->projectStructure, $originalFileName, $mimeType, $fileDateSha1Path); //move the file in the right directory from the packages to the file dir $this->fileStorage->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid, $originalFileName); $this->projectStructure['file_id_list']->append($fid); $this->_extractSegments(file_get_contents($cachedXliffFilePathName), $fid); } } catch (Exception $e) { if ($e->getCode() == -1) { $this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$originalFileName}."); $this->fileStorage->deleteHashFromUploadDir($this->uploadDir, $linkFile); } elseif ($e->getCode() == -2) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$originalFileName}"); } elseif ($e->getCode() == -3) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$originalFileName} not found. Failed to save XLIFF conversion on disk"); } elseif ($e->getCode() == -4) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$originalFileName} )"); } elseif ($e->getCode() == -11) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied"); } elseif ($e->getCode() == -12) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database"); } elseif ($e->getCode() == -13) { $this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage()); Log::doLog($e->getMessage()); return null; // SEVERE EXCEPTION we can not write to disk!! Break project creation } else { //mysql insert Blob Error $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$originalFileName}. Please try again."); } Log::doLog($e->getMessage()); Log::doLog($e->getTraceAsString()); } } //end of conversion hash-link loop try { $this->_zipFileHandling($linkFiles); } catch (Exception $e) { //exit project creation return false; } //check if the files language equals the source language. If not, set an error message. if (!$this->projectStructure['skip_lang_validation']) { $this->validateFilesLanguages(); } if (!$this->_doCheckForErrors()) { //exit project creation return false; } //Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) ); //create job if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) { $owner = $_SESSION['cid']; } else { $_SESSION['_anonym_pid'] = $this->projectStructure['id_project']; //default user $owner = ''; } $isEmptyProject = false; //Throws exception try { $this->_createJobs($this->projectStructure, $owner); //FIXME for project with pre translation this query is not enough, //we need compare the number of segments with translations, but take an eye to the opensource $query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1"; $string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy()); $query_visible_segments = sprintf($query_visible_segments, $string_file_list); try { $rows = $this->dbHandler->fetch_array($query_visible_segments); } catch (PDOException $e) { Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: {$e->getMessage()} - \n"); throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5); } if ($rows[0]['cattool_segments'] == 0) { Log::doLog("Segment Search: No segments in this project - \n"); $isEmptyProject = true; } } catch (Exception $ex) { $this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )"); return false; } try { Utils::deleteDir($this->uploadDir); if (is_dir($this->uploadDir . '_converted')) { Utils::deleteDir($this->uploadDir . '_converted'); } } catch (Exception $e) { $output = "<pre>\n"; $output .= " - Exception: " . print_r($e->getMessage(), true) . "\n"; $output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n"; $output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n"; $output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n"; $output .= "\n\t"; $output .= "Aborting...\n"; $output .= "</pre>"; Log::doLog($output); Utils::sendErrMailReport($output, $e->getMessage()); } $this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE; if ($isEmptyProject) { $this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY; } $this->projectStructure['result']['code'] = 1; $this->projectStructure['result']['data'] = "OK"; $this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword']; $this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass']; $this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list']; $this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments']; $this->projectStructure['result']['id_project'] = $this->projectStructure['id_project']; $this->projectStructure['result']['project_name'] = $this->projectStructure['project_name']; $this->projectStructure['result']['source_language'] = $this->projectStructure['source_language']; $this->projectStructure['result']['target_language'] = $this->projectStructure['target_language']; $this->projectStructure['result']['status'] = $this->projectStructure['status']; $this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files']; $query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n "; $query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']); $project_summary = $this->dbHandler->fetch_array($query_project_summary); $update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n "; $update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']); $this->dbHandler->query($update_project_count); // Log::doLog( $this->projectStructure ); //create Project into DQF queue if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) { $dqfProjectStruct = DQF_DqfProjectStruct::getStruct(); $dqfProjectStruct->api_key = $this->projectStructure['dqf_key']; $dqfProjectStruct->project_id = $this->projectStructure['id_project']; $dqfProjectStruct->name = $this->projectStructure['project_name']; $dqfProjectStruct->source_language = $this->projectStructure['source_language']; $dqfQueue = new Analysis_DqfQueueHandler(); try { $projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->projectStructure['dqf_key']); $dqfQueue->createProject($dqfProjectStruct); //for each job, push a task into AMQ's DQF queue foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) { /** * @var $dqfTaskStruct DQF_DqfTaskStruct */ $dqfTaskStruct = DQF_DqfTaskStruct::getStruct(); $dqfTaskStruct->api_key = $this->projectStructure['dqf_key']; $dqfTaskStruct->project_id = $this->projectStructure['id_project']; $dqfTaskStruct->task_id = $jobID; $dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i]; $dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name']; $dqfQueue->createTask($dqfTaskStruct); } } catch (Exception $exn) { $output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage(); Log::doLog($output); Utils::sendErrMailReport($output, $exn->getMessage()); } } }
protected function _set($config) { $this->result['errors'] = array(); $tm_keys = $this->job_info['tm_keys']; if (self::isRevision()) { $this->userRole = TmKeyManagement_Filter::ROLE_REVISOR; } //get TM keys with read grants $tm_keys = TmKeyManagement_TmKeyManagement::getJobTmKeys($tm_keys, 'w', 'glos', $this->uid, $this->userRole); if (empty($tm_keys)) { $APIKeySrv = new TMSService(); $newUser = (object) $APIKeySrv->createMyMemoryKey(); //throws exception //TODO take only for hystorical reason updateTranslatorJob($this->id_job, $newUser); //fallback $config['id_user'] = $newUser->id; $new_key = TmKeyManagement_TmKeyManagement::getTmKeyStructure(); $new_key->tm = 1; $new_key->glos = 1; $new_key->key = $newUser->key; $new_key->owner = $this->userMail == $this->job_info['owner']; if (!$new_key->owner) { $new_key->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['r']} = 1; $new_key->{TmKeyManagement_Filter::$GRANTS_MAP[$this->userRole]['w']} = 1; } else { $new_key->r = 1; $new_key->w = 1; } if ($new_key->owner) { //do nothing, this is a greedy if } elseif ($this->userRole == TmKeyManagement_Filter::ROLE_TRANSLATOR) { $new_key->uid_transl = $this->uid; } elseif ($this->userRole == TmKeyManagement_Filter::ROLE_REVISOR) { $new_key->uid_rev = $this->uid; } //create an empty array $tm_keys = array(); //append new key $tm_keys[] = $new_key; //put the key in the job TmKeyManagement_TmKeyManagement::setJobTmKeys($this->id_job, $this->password, $tm_keys); //put the key in the user keiring if ($this->userIsLogged) { $newMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $newMemoryKey->tm_key = $new_key; $newMemoryKey->uid = $this->uid; $mkDao = new TmKeyManagement_MemoryKeyDao(Database::obtain()); $mkDao->create($newMemoryKey); } } $config['segment'] = CatUtils::view2rawxliff($config['segment']); $config['translation'] = CatUtils::view2rawxliff($config['translation']); $config['prop'] = json_encode(CatUtils::getTMProps($this->job_info)); //prepare the error report $set_code = array(); //set the glossary entry for each key with write grants if (count($tm_keys)) { /** * @var $tm_keys TmKeyManagement_TmKeyStruct[] */ foreach ($tm_keys as $tm_key) { $config['id_user'] = $tm_key->key; $TMS_RESULT = $this->_TMS->set($config); $set_code[] = $TMS_RESULT; } } $set_successful = true; if (array_search(false, $set_code, true)) { //There's an error, for now skip, let's assume that are not errors $set_successful = false; } if ($set_successful) { // Often the get method after a set is not in real time, so return the same values ( FAKE ) // $TMS_GET_RESULT = $this->_TMS->get($config)->get_glossary_matches_as_array(); // $this->result['data']['matches'] = $TMS_GET_RESULT; $this->result['data']['matches'] = array($config['segment'] => array(array('segment' => $config['segment'], 'translation' => $config['translation'], 'last_update_date' => date_create()->format('Y-m-d H:i:m'), 'last_updated_by' => "Matecat user", 'created_by' => "Matecat user", 'target_note' => $config['tnote']))); if (isset($new_key)) { $this->result['data']['created_tm_key'] = true; } } else { $this->result['errors'][] = array("code" => -1, "message" => "We got an error, please try again."); } }
public function createProject() { // project name sanitize $oldName = $this->projectStructure['project_name']; $this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']); if ($this->projectStructure['project_name'] == false) { $this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!"); return false; } // create project $this->projectStructure['ppassword'] = $this->_generatePassword(); $this->projectStructure['user_ip'] = Utils::getRealIpAddr(); $this->projectStructure['id_customer'] = 'translated_user'; $this->projectStructure['id_project'] = insertProject($this->projectStructure); //create user (Massidda 2013-01-24) //check if all the keys are valid MyMemory keys if (!empty($this->projectStructure['private_tm_key'])) { foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) { $this->tmxServiceWrapper->setTmKey($_tmKey['key']); try { $keyExists = $this->tmxServiceWrapper->checkCorrectKey(); if (!isset($keyExists) || $keyExists === false) { Log::doLog(__METHOD__ . " -> TM key is not valid."); throw new Exception("TM key is not valid.", -4); } } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); return false; } //set the first key as primary $this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']); } //check if the MyMemory keys provided by the user are already associated to him. if ($this->projectStructure['userIsLogged']) { $mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler); $searchMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $searchMemoryKey->uid = $this->projectStructure['uid']; $userMemoryKeys = $mkDao->read($searchMemoryKey); $userTmKeys = array(); $memoryKeysToBeInserted = array(); //extract user tm keys foreach ($userMemoryKeys as $_memoKey) { /** * @var $_memoKey TmKeyManagement_MemoryKeyStruct */ $userTmKeys[] = $_memoKey->tm_key->key; } foreach ($this->projectStructure['private_tm_key'] as $_tmKey) { if (!in_array($_tmKey['key'], $userTmKeys)) { $newMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $newTmKey = new TmKeyManagement_TmKeyStruct(); $newTmKey->key = $_tmKey['key']; $newTmKey->tm = true; $newTmKey->glos = true; //TODO: take this from input $newTmKey->name = $_tmKey['name']; $newMemoryKey->tm_key = $newTmKey; $newMemoryKey->uid = $this->projectStructure['uid']; $memoryKeysToBeInserted[] = $newMemoryKey; } else { Log::doLog('skip insertion'); } } try { $mkDao->createList($memoryKeysToBeInserted); } catch (Exception $e) { Log::doLog($e->getMessage()); # Here we handle the error, displaying HTML, logging, ... $output = "<pre>\n"; $output .= $e->getMessage() . "\n\t"; $output .= "</pre>"; Utils::sendErrMailReport($output); } } //the base case is when the user clicks on "generate private TM" button: //a (user, pass, key) tuple is generated and can be inserted //if it comes with it's own key without querying the creation API, create a (key,key,key) user if (empty($this->projectStructure['private_tm_user'])) { $this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key']; $this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key']; } insertTranslator($this->projectStructure); } //sort files in order to process TMX first $sortedFiles = array(); foreach ($this->projectStructure['array_files'] as $fileName) { if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) { //found TMX, enable language checking routines $this->checkTMX = 1; array_unshift($sortedFiles, $fileName); } else { array_push($sortedFiles, $fileName); } } $this->projectStructure['array_files'] = $sortedFiles; unset($sortedFiles); $uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken']; //fetch cache links, created by converter, from upload directory $linkFiles = scandir($uploadDir); //remove dir hardlinks, as uninteresting, as weel as regular files; only hash-links foreach ($linkFiles as $k => $linkFile) { if (strpos($linkFile, '.') !== false or strpos($linkFile, '|') === false) { unset($linkFiles[$k]); } } /* loop through all input files to 1)upload TMX 2)convert, in case, non standard XLIFF files to a format that Matecat understands Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow */ foreach ($this->projectStructure['array_files'] as $fileName) { //if TMX, if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) { //load it into MyMemory; we'll check later on how it went $file = new stdClass(); $file->file_path = "{$uploadDir}/{$fileName}"; $this->tmxServiceWrapper->setName($fileName); $this->tmxServiceWrapper->setFile(array($file)); try { $this->tmxServiceWrapper->addTmxInMyMemory(); } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); return false; } //in any case, skip the rest of the loop, go to the next file continue; } /* Conversion Enforce Checking Extension is no more sufficient, we want check content if this is an idiom xlf file type, conversion are enforced $enforcedConversion = true; //( if conversion is enabled ) */ $isAnXliffToConvert = $this->isConversionToEnforce($fileName); //we are going to access the storage, get model object to manipulate it $fs = new FilesStorage(); //if it's one of the listed formats or conversion is not enabled in first place if (!$isAnXliffToConvert) { /* filename is already an xliff and it's in upload directory we have to make a cache package from it to avoid altering the original path */ //get file $filePathName = "{$uploadDir}/{$fileName}"; //calculate hash + add the fileName, if i load 3 equal files with the same content // they will be squashed to the last one $sha1 = sha1(file_get_contents($filePathName) . $filePathName); //make a cache package (with work/ only, emtpy orig/) $fs->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName); //put reference to cache in upload dir to link cache to session $fs->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken']); //add newly created link to list $linkFiles[] = $sha1 . "|" . $this->projectStructure['source_language']; unset($sha1); } } //now, upload dir contains only hash-links //we start copying files to "file" dir, inserting metadata in db and extracting segments foreach ($linkFiles as $linkFile) { //converted file is inside cache directory //get hash from file name inside UUID dir $hashFile = basename($linkFile); $hashFile = explode('|', $hashFile); //use hash and lang to fetch file from package $xliffFilePathName = $fs->getXliffFromCache($hashFile[0], $hashFile[1]); //get sha $sha1_original = $hashFile[0]; //get original file name $originalFilePathName = $fs->getOriginalFromCache($hashFile[0], $hashFile[1]); $raw_originalFilePathName = explode(DIRECTORY_SEPARATOR, $originalFilePathName); $fileName = array_pop($raw_originalFilePathName); unset($hashFile); if (!file_exists($xliffFilePathName)) { $this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload."); } try { $info = pathinfo($xliffFilePathName); if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) { throw new Exception("Failed to find Xliff - no segments found", -3); } $mimeType = pathinfo($fileName, PATHINFO_EXTENSION); $yearMonthPath = date_create()->format("Ymd"); $fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original; $fid = insertFile($this->projectStructure, $fileName, $mimeType, $fileDateSha1Path); //move the file in the right directory from the packages to the file dir $fs->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid); $this->projectStructure['file_id_list']->append($fid); $this->_extractSegments(file_get_contents($xliffFilePathName), $fid); } catch (Exception $e) { if ($e->getCode() == -1) { $this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}."); $fs->deleteHashFromUploadDir($uploadDir, $linkFile); } elseif ($e->getCode() == -2) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}"); } elseif ($e->getCode() == -3) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk"); } elseif ($e->getCode() == -4) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )"); } elseif ($e->getCode() == -11) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied"); } elseif ($e->getCode() == -12) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database"); } elseif ($e->getCode() == -13) { $this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage()); Log::doLog($e->getMessage()); return null; // SEVERE EXCEPTION we can not write to disk!! Break project creation } else { //mysql insert Blob Error $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$fileName}. Please try again."); } Log::doLog($e->getMessage()); } } //end of hash-link loop //check if the files language equals the source language. If not, set an error message. if (!$this->projectStructure['skip_lang_validation']) { $this->validateFilesLanguages(); } /****************/ //loop again through files to check to check for TMX loading foreach ($this->projectStructure['array_files'] as $fileName) { //if TMX, if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) { $this->tmxServiceWrapper->setName($fileName); $result = array(); //is the TM loaded? //wait until current TMX is loaded while (true) { try { $result = $this->tmxServiceWrapper->tmxUploadStatus(); if ($result['completed']) { //"$fileName" has been loaded into MyMemory" //exit the loop break; } //"waiting for "$fileName" to be loaded into MyMemory" sleep(3); } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString()); //exit project creation return false; } } //once the language is loaded, check if language is compliant (unless something useful has already been found) if (1 == $this->checkTMX) { //get localized target languages of TM (in case it's a multilingual TM) $tmTargets = explode(';', $result['data']['target_lang']); //indicates if something has been found for current memory $found = false; //compare localized target languages array (in case it's a multilingual project) to the TM supplied //if nothing matches, then the TM supplied can't have matches for this project //create an empty var and add the source language too $project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']); foreach ($project_languages as $projectTarget) { if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) { $found = true; break; } } //if this TM matches the project lagpair and something has been found if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) { //the TMX is good to go $this->checkTMX = 0; } elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) { /* * This means that the TMX has a srclang as specification in the header. Warn the user. * Ex: * <header creationtool="SDL Language Platform" * creationtoolversion="8.0" * datatype="rtf" * segtype="sentence" * adminlang="DE-DE" * srclang="DE-DE" /> */ $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file."); $this->checkTMX = 0; Log::doLog($this->projectStructure['result']); } } } } if (1 == $this->checkTMX) { //this means that noone of uploaded TMX were usable for this project. Warn the user. $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project."); Log::doLog($this->projectStructure['result']); return false; } if (!empty($this->projectStructure['result']['errors'])) { Log::doLog("Project Creation Failed. Sent to Output all errors."); Log::doLog($this->projectStructure['result']['errors']); return false; } //Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) ); //create job if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) { $owner = $_SESSION['cid']; } else { $_SESSION['_anonym_pid'] = $this->projectStructure['id_project']; //default user $owner = ''; } $isEmptyProject = false; //Throws exception try { $this->_createJobs($this->projectStructure, $owner); //FIXME for project with pre translation this query is not enough, //we need compare the number of segments with translations, but take an eye to the opensource $query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1"; $string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy()); $query_visible_segments = sprintf($query_visible_segments, $string_file_list); $rows = $this->dbHandler->fetch_array($query_visible_segments); if (!$rows) { Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n"); throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5); } if ($rows[0]['cattool_segments'] == 0) { Log::doLog("Segment Search: No segments in this project - \n"); $isEmptyProject = true; } } catch (Exception $ex) { $this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )"); return false; } try { Utils::deleteDir($uploadDir); if (is_dir($uploadDir . '_converted')) { Utils::deleteDir($uploadDir . '_converted'); } } catch (Exception $e) { $output = "<pre>\n"; $output .= " - Exception: " . print_r($e->getMessage(), true) . "\n"; $output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n"; $output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n"; $output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n"; $output .= "\n\t"; $output .= "Aborting...\n"; $output .= "</pre>"; Log::doLog($output); Utils::sendErrMailReport($output, $e->getMessage()); } $this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE; if ($isEmptyProject) { $this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY; } $this->projectStructure['result']['code'] = 1; $this->projectStructure['result']['data'] = "OK"; $this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword']; $this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass']; $this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list']; $this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments']; $this->projectStructure['result']['id_project'] = $this->projectStructure['id_project']; $this->projectStructure['result']['project_name'] = $this->projectStructure['project_name']; $this->projectStructure['result']['source_language'] = $this->projectStructure['source_language']; $this->projectStructure['result']['target_language'] = $this->projectStructure['target_language']; $this->projectStructure['result']['status'] = $this->projectStructure['status']; $this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files']; $query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n "; $query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']); $project_summary = $this->dbHandler->fetch_array($query_project_summary); $update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n "; $update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']); $this->dbHandler->query($update_project_count); // Log::doLog( $this->projectStructure ); //create Project into DQF queue if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) { $dqfProjectStruct = DQF_DqfProjectStruct::getStruct(); $dqfProjectStruct->api_key = $this->projectStructure['dqf_key']; $dqfProjectStruct->project_id = $this->projectStructure['id_project']; $dqfProjectStruct->name = $this->projectStructure['project_name']; $dqfProjectStruct->source_language = $this->projectStructure['source_language']; $dqfQueue = new Analysis_DqfQueueHandler(); try { $dqfQueue->createProject($dqfProjectStruct); //for each job, push a task into AMQ's DQF queue foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) { /** * @var $dqfTaskStruct DQF_DqfTaskStruct */ $dqfTaskStruct = DQF_DqfTaskStruct::getStruct(); $dqfTaskStruct->api_key = $this->projectStructure['dqf_key']; $dqfTaskStruct->project_id = $this->projectStructure['id_project']; $dqfTaskStruct->task_id = $jobID; $dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i]; $dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name']; $dqfQueue->createTask($dqfTaskStruct); } } catch (Exception $exn) { $output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage(); Log::doLog($output); Utils::sendErrMailReport($output, $exn->getMessage()); } } }