Beispiel #1
0
 public function createProject()
 {
     // project name sanitize
     $oldName = $this->projectStructure['project_name'];
     $this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
     if ($this->projectStructure['project_name'] == false) {
         $this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
         return false;
     }
     // create project?
     $this->projectStructure['ppassword'] = $this->_generatePassword();
     $this->projectStructure['user_ip'] = Utils::getRealIpAddr();
     $this->projectStructure['id_project'] = insertProject($this->projectStructure);
     //create user (Massidda 2013-01-24)
     //check if all the keys are valid MyMemory keys
     if (!empty($this->projectStructure['private_tm_key'])) {
         foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
             $this->tmxServiceWrapper->setTmKey($_tmKey['key']);
             try {
                 $keyExists = $this->tmxServiceWrapper->checkCorrectKey();
                 if (!isset($keyExists) || $keyExists === false) {
                     Log::doLog(__METHOD__ . " -> TM key is not valid.");
                     throw new Exception("TM key is not valid: " . $_tmKey['key'], -4);
                 }
             } catch (Exception $e) {
                 $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
                 return false;
             }
             //set the first key as primary
             $this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
         }
         //check if the MyMemory keys provided by the user are already associated to him.
         if ($this->projectStructure['userIsLogged']) {
             $mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
             $searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
             $searchMemoryKey->uid = $this->projectStructure['uid'];
             $userMemoryKeys = $mkDao->read($searchMemoryKey);
             $userTmKeys = array();
             $memoryKeysToBeInserted = array();
             //extract user tm keys
             foreach ($userMemoryKeys as $_memoKey) {
                 /**
                  * @var $_memoKey TmKeyManagement_MemoryKeyStruct
                  */
                 $userTmKeys[] = $_memoKey->tm_key->key;
             }
             foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
                 if (!in_array($_tmKey['key'], $userTmKeys)) {
                     $newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
                     $newTmKey = new TmKeyManagement_TmKeyStruct();
                     $newTmKey->key = $_tmKey['key'];
                     $newTmKey->tm = true;
                     $newTmKey->glos = true;
                     //TODO: take this from input
                     $newTmKey->name = $_tmKey['name'];
                     $newMemoryKey->tm_key = $newTmKey;
                     $newMemoryKey->uid = $this->projectStructure['uid'];
                     $memoryKeysToBeInserted[] = $newMemoryKey;
                 } else {
                     Log::doLog('skip insertion');
                 }
             }
             try {
                 $mkDao->createList($memoryKeysToBeInserted);
             } catch (Exception $e) {
                 Log::doLog($e->getMessage());
                 # Here we handle the error, displaying HTML, logging, ...
                 $output = "<pre>\n";
                 $output .= $e->getMessage() . "\n\t";
                 $output .= "</pre>";
                 Utils::sendErrMailReport($output);
             }
         }
         //the base case is when the user clicks on "generate private TM" button:
         //a (user, pass, key) tuple is generated and can be inserted
         //if it comes with it's own key without querying the creation API, create a (key,key,key) user
         if (empty($this->projectStructure['private_tm_user'])) {
             $this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
             $this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
         }
         insertTranslator($this->projectStructure);
     }
     //sort files in order to process TMX first
     $sortedFiles = array();
     foreach ($this->projectStructure['array_files'] as $fileName) {
         //check for glossary files and tmx and put them in front of the list
         $infoFile = DetectProprietaryXliff::getInfo($fileName);
         if (DetectProprietaryXliff::getMemoryFileType()) {
             //found TMX, enable language checking routines
             if (DetectProprietaryXliff::isTMXFile()) {
                 $this->checkTMX = 1;
             }
             //not used at moment but needed if we want to do a poll for status
             if (DetectProprietaryXliff::isGlossaryFile()) {
                 $this->checkGlossary = 1;
             }
             //prepend in front of the list
             array_unshift($sortedFiles, $fileName);
         } else {
             //append at the end of the list
             array_push($sortedFiles, $fileName);
         }
     }
     $this->projectStructure['array_files'] = $sortedFiles;
     unset($sortedFiles);
     $uploadDir = $this->uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
     //we are going to access the storage, get model object to manipulate it
     $this->fileStorage = new FilesStorage();
     $linkFiles = $this->fileStorage->getHashesFromDir($this->uploadDir);
     /*
         loop through all input files to
         1) upload TMX and Glossaries
     */
     try {
         $this->_pushTMXToMyMemory();
     } catch (Exception $e) {
         //exit project creation
         return false;
     }
     //TMX Management
     /*
        loop through all input files to
        2)convert, in case, non standard XLIFF files to a format that Matecat understands
     
        Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
     */
     foreach ($this->projectStructure['array_files'] as $fileName) {
         /*
           Conversion Enforce
           Checking Extension is no more sufficient, we want check content
           $enforcedConversion = true; //( if conversion is enabled )
         */
         $isAFileToConvert = $this->isConversionToEnforce($fileName);
         //if it's one of the listed formats or conversion is not enabled in first place
         if (!$isAFileToConvert) {
             /*
               filename is already an xliff and it's in upload directory
               we have to make a cache package from it to avoid altering the original path
             */
             //get file
             $filePathName = "{$this->uploadDir}/{$fileName}";
             //calculate hash + add the fileName, if i load 3 equal files with the same content
             // they will be squashed to the last one
             $sha1 = sha1_file($filePathName);
             //make a cache package (with work/ only, emtpy orig/)
             $this->fileStorage->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
             //put reference to cache in upload dir to link cache to session
             $this->fileStorage->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken'], $fileName);
             //add newly created link to list
             $linkFiles['conversionHashes']['sha'][] = $sha1 . "|" . $this->projectStructure['source_language'];
             $linkFiles['conversionHashes']['fileName'][$sha1 . "|" . $this->projectStructure['source_language']][] = $fileName;
             //when the same sdlxliff is uploaded more than once with different names
             $linkFiles['conversionHashes']['sha'] = array_unique($linkFiles['conversionHashes']['sha']);
             unset($sha1);
         }
     }
     //now, upload dir contains only hash-links
     //we start copying files to "file" dir, inserting metadata in db and extracting segments
     foreach ($linkFiles['conversionHashes']['sha'] as $linkFile) {
         //converted file is inside cache directory
         //get hash from file name inside UUID dir
         $hashFile = FilesStorage::basename_fix($linkFile);
         $hashFile = explode('|', $hashFile);
         //use hash and lang to fetch file from package
         $cachedXliffFilePathName = $this->fileStorage->getXliffFromCache($hashFile[0], $hashFile[1]);
         //get sha
         $sha1_original = $hashFile[0];
         //associate the hash to the right file in upload directory
         //get original file name, to insert into DB and cp in storage
         //PLEASE NOTE, this can be an array when the same file added more
         // than once and with different names
         $_originalFileName = $linkFiles['conversionHashes']['fileName'][$linkFile];
         unset($hashFile);
         if (!file_exists($cachedXliffFilePathName)) {
             $this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
         }
         try {
             $info = FilesStorage::pathinfo_fix($cachedXliffFilePathName);
             if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
                 throw new Exception("Failed to find Xliff - no segments found", -3);
             }
             $yearMonthPath = date_create($this->projectStructure['create_date'])->format('Ymd');
             $fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
             //PLEASE NOTE, this can be an array when the same file added more
             // than once and with different names
             foreach ($_originalFileName as $originalFileName) {
                 $mimeType = FilesStorage::pathinfo_fix($originalFileName, PATHINFO_EXTENSION);
                 $fid = insertFile($this->projectStructure, $originalFileName, $mimeType, $fileDateSha1Path);
                 //move the file in the right directory from the packages to the file dir
                 $this->fileStorage->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid, $originalFileName);
                 $this->projectStructure['file_id_list']->append($fid);
                 $this->_extractSegments(file_get_contents($cachedXliffFilePathName), $fid);
             }
         } catch (Exception $e) {
             if ($e->getCode() == -1) {
                 $this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$originalFileName}.");
                 $this->fileStorage->deleteHashFromUploadDir($this->uploadDir, $linkFile);
             } elseif ($e->getCode() == -2) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$originalFileName}");
             } elseif ($e->getCode() == -3) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$originalFileName} not found. Failed to save XLIFF conversion on disk");
             } elseif ($e->getCode() == -4) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$originalFileName} )");
             } elseif ($e->getCode() == -11) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
             } elseif ($e->getCode() == -12) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
             } elseif ($e->getCode() == -13) {
                 $this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
                 Log::doLog($e->getMessage());
                 return null;
                 // SEVERE EXCEPTION we can not write to disk!! Break project creation
             } else {
                 //mysql insert Blob Error
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$originalFileName}. Please try again.");
             }
             Log::doLog($e->getMessage());
             Log::doLog($e->getTraceAsString());
         }
     }
     //end of conversion hash-link loop
     try {
         $this->_zipFileHandling($linkFiles);
     } catch (Exception $e) {
         //exit project creation
         return false;
     }
     //check if the files language equals the source language. If not, set an error message.
     if (!$this->projectStructure['skip_lang_validation']) {
         $this->validateFilesLanguages();
     }
     if (!$this->_doCheckForErrors()) {
         //exit project creation
         return false;
     }
     //Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
     //create job
     if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
         $owner = $_SESSION['cid'];
     } else {
         $_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
         //default user
         $owner = '';
     }
     $isEmptyProject = false;
     //Throws exception
     try {
         $this->_createJobs($this->projectStructure, $owner);
         //FIXME for project with pre translation this query is not enough,
         //we need compare the number of segments with translations, but take an eye to the opensource
         $query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
         $string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
         $query_visible_segments = sprintf($query_visible_segments, $string_file_list);
         try {
             $rows = $this->dbHandler->fetch_array($query_visible_segments);
         } catch (PDOException $e) {
             Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: {$e->getMessage()} - \n");
             throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
         }
         if ($rows[0]['cattool_segments'] == 0) {
             Log::doLog("Segment Search: No segments in this project - \n");
             $isEmptyProject = true;
         }
     } catch (Exception $ex) {
         $this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
         return false;
     }
     try {
         Utils::deleteDir($this->uploadDir);
         if (is_dir($this->uploadDir . '_converted')) {
             Utils::deleteDir($this->uploadDir . '_converted');
         }
     } catch (Exception $e) {
         $output = "<pre>\n";
         $output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
         $output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
         $output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
         $output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
         $output .= "\n\t";
         $output .= "Aborting...\n";
         $output .= "</pre>";
         Log::doLog($output);
         Utils::sendErrMailReport($output, $e->getMessage());
     }
     $this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
     if ($isEmptyProject) {
         $this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
     }
     $this->projectStructure['result']['code'] = 1;
     $this->projectStructure['result']['data'] = "OK";
     $this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
     $this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
     $this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
     $this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
     $this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
     $this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
     $this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
     $this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
     $this->projectStructure['result']['status'] = $this->projectStructure['status'];
     $this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
     $query_project_summary = "\n            SELECT\n                 COUNT( s.id ) AS project_segments,\n                 SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n            FROM segments s\n            INNER JOIN files_job fj ON fj.id_file = s.id_file\n            INNER JOIN jobs j ON j.id= fj.id_job\n            LEFT JOIN segment_translations st ON s.id = st.id_segment\n            WHERE j.id_project = %u\n        ";
     $query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
     $project_summary = $this->dbHandler->fetch_array($query_project_summary);
     $update_project_count = "\n            UPDATE projects\n              SET\n                standard_analysis_wc = %.2F,\n                status_analysis = '%s'\n            WHERE id = %u\n        ";
     $update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
     $this->dbHandler->query($update_project_count);
     //        Log::doLog( $this->projectStructure );
     //create Project into DQF queue
     if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
         $dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
         $dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
         $dqfProjectStruct->project_id = $this->projectStructure['id_project'];
         $dqfProjectStruct->name = $this->projectStructure['project_name'];
         $dqfProjectStruct->source_language = $this->projectStructure['source_language'];
         $dqfQueue = new Analysis_DqfQueueHandler();
         try {
             $projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->projectStructure['dqf_key']);
             $dqfQueue->createProject($dqfProjectStruct);
             //for each job, push a task into AMQ's DQF queue
             foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
                 /**
                  * @var $dqfTaskStruct DQF_DqfTaskStruct
                  */
                 $dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
                 $dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
                 $dqfTaskStruct->project_id = $this->projectStructure['id_project'];
                 $dqfTaskStruct->task_id = $jobID;
                 $dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
                 $dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
                 $dqfQueue->createTask($dqfTaskStruct);
             }
         } catch (Exception $exn) {
             $output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
             Log::doLog($output);
             Utils::sendErrMailReport($output, $exn->getMessage());
         }
     }
 }
 public function createProject()
 {
     // project name sanitize
     $oldName = $this->projectStructure['project_name'];
     $this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
     if ($this->projectStructure['project_name'] == false) {
         $this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
         return false;
     }
     // create project
     $this->projectStructure['ppassword'] = $this->_generatePassword();
     $this->projectStructure['user_ip'] = Utils::getRealIpAddr();
     $this->projectStructure['id_customer'] = 'translated_user';
     $this->projectStructure['id_project'] = insertProject($this->projectStructure);
     //create user (Massidda 2013-01-24)
     //check if all the keys are valid MyMemory keys
     if (!empty($this->projectStructure['private_tm_key'])) {
         foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
             $this->tmxServiceWrapper->setTmKey($_tmKey['key']);
             try {
                 $keyExists = $this->tmxServiceWrapper->checkCorrectKey();
                 if (!isset($keyExists) || $keyExists === false) {
                     Log::doLog(__METHOD__ . " -> TM key is not valid.");
                     throw new Exception("TM key is not valid.", -4);
                 }
             } catch (Exception $e) {
                 $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
                 return false;
             }
             //set the first key as primary
             $this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
         }
         //check if the MyMemory keys provided by the user are already associated to him.
         if ($this->projectStructure['userIsLogged']) {
             $mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
             $searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
             $searchMemoryKey->uid = $this->projectStructure['uid'];
             $userMemoryKeys = $mkDao->read($searchMemoryKey);
             $userTmKeys = array();
             $memoryKeysToBeInserted = array();
             //extract user tm keys
             foreach ($userMemoryKeys as $_memoKey) {
                 /**
                  * @var $_memoKey TmKeyManagement_MemoryKeyStruct
                  */
                 $userTmKeys[] = $_memoKey->tm_key->key;
             }
             foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
                 if (!in_array($_tmKey['key'], $userTmKeys)) {
                     $newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
                     $newTmKey = new TmKeyManagement_TmKeyStruct();
                     $newTmKey->key = $_tmKey['key'];
                     $newTmKey->tm = true;
                     $newTmKey->glos = true;
                     //TODO: take this from input
                     $newTmKey->name = $_tmKey['name'];
                     $newMemoryKey->tm_key = $newTmKey;
                     $newMemoryKey->uid = $this->projectStructure['uid'];
                     $memoryKeysToBeInserted[] = $newMemoryKey;
                 } else {
                     Log::doLog('skip insertion');
                 }
             }
             try {
                 $mkDao->createList($memoryKeysToBeInserted);
             } catch (Exception $e) {
                 Log::doLog($e->getMessage());
                 # Here we handle the error, displaying HTML, logging, ...
                 $output = "<pre>\n";
                 $output .= $e->getMessage() . "\n\t";
                 $output .= "</pre>";
                 Utils::sendErrMailReport($output);
             }
         }
         //the base case is when the user clicks on "generate private TM" button:
         //a (user, pass, key) tuple is generated and can be inserted
         //if it comes with it's own key without querying the creation API, create a (key,key,key) user
         if (empty($this->projectStructure['private_tm_user'])) {
             $this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
             $this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
         }
         insertTranslator($this->projectStructure);
     }
     //sort files in order to process TMX first
     $sortedFiles = array();
     foreach ($this->projectStructure['array_files'] as $fileName) {
         if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
             //found TMX, enable language checking routines
             $this->checkTMX = 1;
             array_unshift($sortedFiles, $fileName);
         } else {
             array_push($sortedFiles, $fileName);
         }
     }
     $this->projectStructure['array_files'] = $sortedFiles;
     unset($sortedFiles);
     $uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
     foreach ($this->projectStructure['array_files'] as $fileName) {
         //if TMX,
         if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
             $file = new stdClass();
             $file->file_path = "{$uploadDir}/{$fileName}";
             $this->tmxServiceWrapper->setName($fileName);
             $this->tmxServiceWrapper->setFile(array($file));
             try {
                 $this->tmxServiceWrapper->addTmxInMyMemory();
             } catch (Exception $e) {
                 $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
                 return false;
             }
             //in any case, skip the rest of the loop, go to the next file
             continue;
         }
         /**
          * Conversion Enforce
          *
          * we have to know if a file can be found in _converted directory
          *
          * Check Extension no more sufficient, we want check content
          * if this is an idiom xlf file type, conversion are enforced
          * $enforcedConversion = true; //( if conversion is enabled )
          */
         $isAConvertedFile = true;
         try {
             $fileType = DetectProprietaryXliff::getInfo(INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'] . DIRECTORY_SEPARATOR . $fileName);
             if (DetectProprietaryXliff::isXliffExtension()) {
                 if (INIT::$CONVERSION_ENABLED) {
                     //conversion enforce
                     if (!INIT::$FORCE_XLIFF_CONVERSION) {
                         //ONLY IDIOM is forced to be converted
                         //if file is not proprietary like idiom AND Enforce is disabled
                         //we take it as is
                         if (!$fileType['proprietary']) {
                             $isAConvertedFile = false;
                             //ok don't convert a standard sdlxliff
                         }
                     } else {
                         //if conversion enforce is active
                         //we force all xliff files but not files produced by SDL Studio because we can handle them
                         if ($fileType['proprietary_short_name'] == 'trados') {
                             $isAConvertedFile = false;
                             //ok don't convert a standard sdlxliff
                         }
                     }
                 } elseif ($fileType['proprietary']) {
                     /**
                      * Application misconfiguration.
                      * upload should not be happened, but if we are here, raise an error.
                      * @see upload.class.php
                      * */
                     $this->projectStructure['result']['errors'][] = array("code" => -8, "message" => "Proprietary xlf format detected. Not able to import this XLIFF file. ({$fileName})");
                     setcookie("upload_session", "", time() - 10000);
                     return -1;
                     //stop execution
                 } elseif (!$fileType['proprietary']) {
                     $isAConvertedFile = false;
                     //ok don't convert a standard sdlxliff
                 }
             }
         } catch (Exception $e) {
             Log::doLog($e->getMessage());
         }
         $mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
         $original_content = "";
         /*
           if it's not one of the listed formats (or it is, but you had to convert it anyway), 
           and conversion is enabled in first place
         */
         if ($isAConvertedFile) {
             //converted file is inside "_converted" directory
             $fileDir = $uploadDir . '_converted';
             $original_content = file_get_contents("{$uploadDir}/{$fileName}");
             $sha1_original = sha1($original_content);
             $original_content = gzdeflate($original_content, 5);
             //file name is a xliff converted like: 'a_word_document.doc.sdlxliff'
             $real_fileName = $fileName . '.sdlxliff';
         } else {
             //filename is already an xliff and it is in a canonical normal directory
             $sha1_original = "";
             $fileDir = $uploadDir;
             $real_fileName = $fileName;
         }
         $filePathName = $fileDir . DIRECTORY_SEPARATOR . $real_fileName;
         if (!file_exists($filePathName)) {
             $this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
         }
         try {
             $info = pathinfo($filePathName);
             if ($info['extension'] == 'xliff' || $info['extension'] == 'sdlxliff' || $info['extension'] == 'xlf') {
                 $contents = file_get_contents($filePathName);
             } else {
                 throw new Exception("Failed to find Xliff - no segments found", -3);
             }
             $fid = insertFile($this->projectStructure, $fileName, $mimeType, $contents, $sha1_original, $original_content);
             $this->projectStructure['file_id_list']->append($fid);
             $this->_extractSegments($contents, $fid);
             unset($contents);
             //free memory
             //Log::doLog( $this->projectStructure['segments'] );
         } catch (Exception $e) {
             if ($e->getCode() == -1) {
                 $this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
             } elseif ($e->getCode() == -2) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
             } elseif ($e->getCode() == -3) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
             } elseif ($e->getCode() == -4) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
             } elseif ($e->getCode() == -11) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
             } elseif ($e->getCode() == -12) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
             } else {
                 //mysql insert Blob Error
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File is Too large. ( {$fileName} )");
             }
             Log::doLog($e->getMessage());
         }
         //exit;
     }
     //check if the files language equals the source language. If not, set an error message.
     if (!$this->projectStructure['skip_lang_validation']) {
         $this->validateFilesLanguages();
     }
     /****************/
     //loop again through files to check to check for TMX loading
     foreach ($this->projectStructure['array_files'] as $fileName) {
         //if TMX,
         if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
             $this->tmxServiceWrapper->setName($fileName);
             $result = array();
             //is the TM loaded?
             //wait until current TMX is loaded
             while (true) {
                 try {
                     $result = $this->tmxServiceWrapper->tmxUploadStatus();
                     if ($result['completed']) {
                         //"$fileName" has been loaded into MyMemory"
                         //exit the loop
                         break;
                     }
                     //"waiting for "$fileName" to be loaded into MyMemory"
                     sleep(3);
                 } catch (Exception $e) {
                     $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
                     Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
                     //exit project creation
                     return false;
                 }
             }
             //once the language is loaded, check if language is compliant (unless something useful has already been found)
             if (1 == $this->checkTMX) {
                 //get localized target languages of TM (in case it's a multilingual TM)
                 $tmTargets = explode(';', $result['data']['target_lang']);
                 //indicates if something has been found for current memory
                 $found = false;
                 //compare localized target languages array (in case it's a multilingual project) to the TM supplied
                 //if nothing matches, then the TM supplied can't have matches for this project
                 //create an empty var and add the source language too
                 $project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
                 foreach ($project_languages as $projectTarget) {
                     if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
                         $found = true;
                         break;
                     }
                 }
                 //if this TM matches the project lagpair and something has been found
                 if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
                     //the TMX is good to go
                     $this->checkTMX = 0;
                 } elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
                     /*
                      * This means that the TMX has a srclang as specification in the header. Warn the user.
                      * Ex:
                      * <header creationtool="SDL Language Platform"
                      *      creationtoolversion="8.0"
                      *      datatype="rtf"
                      *      segtype="sentence"
                      *      adminlang="DE-DE"
                      *      srclang="DE-DE" />
                      */
                     $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
                     $this->checkTMX = 0;
                     Log::doLog($this->projectStructure['result']);
                 }
             }
         }
     }
     if (1 == $this->checkTMX) {
         //this means that noone of uploaded TMX were usable for this project. Warn the user.
         $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
         Log::doLog($this->projectStructure['result']);
         return false;
     }
     if (!empty($this->projectStructure['result']['errors'])) {
         Log::doLog("Project Creation Failed. Sent to Output all errors.");
         Log::doLog($this->projectStructure['result']['errors']);
         return false;
     }
     //Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
     //create job
     if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
         $owner = $_SESSION['cid'];
     } else {
         $_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
         //default user
         $owner = '';
     }
     $isEmptyProject = false;
     //Throws exception
     try {
         $this->_createJobs($this->projectStructure, $owner);
         //FIXME for project with pre translation this query is not enough,
         //we need compare the number of segments with translations, but take an eye to the opensource
         $query_visible_segments = "SELECT count(*) as cattool_segments\n                FROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
         $string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
         $query_visible_segments = sprintf($query_visible_segments, $string_file_list);
         $rows = $this->dbHandler->fetch_array($query_visible_segments);
         if (!$rows) {
             Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
             throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
         }
         if ($rows[0]['cattool_segments'] == 0) {
             Log::doLog("Segment Search: No segments in this project - \n");
             $isEmptyProject = true;
         }
     } catch (Exception $ex) {
         $this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
         return false;
     }
     try {
         Utils::deleteDir($uploadDir);
         if (is_dir($uploadDir . '_converted')) {
             Utils::deleteDir($uploadDir . '_converted');
         }
     } catch (Exception $e) {
         $output = "<pre>\n";
         $output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
         $output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
         $output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
         $output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
         $output .= "\n\t";
         $output .= "Aborting...\n";
         $output .= "</pre>";
         Log::doLog($output);
         Utils::sendErrMailReport($output, $e->getMessage());
     }
     $this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
     if ($isEmptyProject) {
         $this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
     }
     $this->projectStructure['result']['code'] = 1;
     $this->projectStructure['result']['data'] = "OK";
     $this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
     $this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
     $this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
     $this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
     $this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
     $this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
     $this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
     $this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
     $this->projectStructure['result']['status'] = $this->projectStructure['status'];
     $this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
     $query_project_summary = "\n            SELECT\n                 COUNT( s.id ) AS project_segments,\n                 SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n            FROM segments s\n            INNER JOIN files_job fj ON fj.id_file = s.id_file\n            INNER JOIN jobs j ON j.id= fj.id_job\n            LEFT JOIN segment_translations st ON s.id = st.id_segment\n            WHERE j.id_project = %u\n        ";
     $query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
     $project_summary = $this->dbHandler->fetch_array($query_project_summary);
     /**
      * TODO: remove after queue implementation
      */
     //        if( 0 && $project_summary[0]['project_segments'] > 50000 ) {
     //            $this->projectStructure[ 'status' ] = Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
     //
     //            $msg = "
     //        WARNING: a project with more than 50.000 segments was created. ( " . $project_summary[0]['project_segments'] . " )\n" .
     //        var_export( $this->projectStructure[ 'result' ], true ) . "\n\n" .
     //                    "  " .
     //        var_export( $project_summary[0] , true ) . "\n";
     //
     //            Utils::sendErrMailReport( $msg, "Alert: Project Creation Abort. - " );
     //
     //        }
     $update_project_count = "\n            UPDATE projects\n              SET\n                standard_analysis_wc = %.2F,\n                status_analysis = '%s'\n            WHERE id = %u\n        ";
     $update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
     $this->dbHandler->query($update_project_count);
     //        Log::doLog( $this->projectStructure );
     //create Project into DQF queue
     if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
         $dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
         $dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
         $dqfProjectStruct->project_id = $this->projectStructure['id_project'];
         $dqfProjectStruct->name = $this->projectStructure['project_name'];
         $dqfProjectStruct->source_language = $this->projectStructure['source_language'];
         $dqfQueue = new Analysis_DqfQueueHandler();
         try {
             $dqfQueue->createProject($dqfProjectStruct);
             //for each job, push a task into AMQ's DQF queue
             foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
                 /**
                  * @var $dqfTaskStruct DQF_DqfTaskStruct
                  */
                 $dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
                 $dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
                 $dqfTaskStruct->project_id = $this->projectStructure['id_project'];
                 $dqfTaskStruct->task_id = $jobID;
                 $dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
                 $dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
                 $dqfQueue->createTask($dqfTaskStruct);
             }
         } catch (Exception $exn) {
             $output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
             Log::doLog($output);
             Utils::sendErrMailReport($output, $exn->getMessage());
         }
     }
     //    var_dump($this->projectStructure);
     //        exit;
 }
Beispiel #3
0
 public function createProject()
 {
     // project name sanitize
     $oldName = $this->projectStructure['project_name'];
     $this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']);
     if ($this->projectStructure['project_name'] == false) {
         $this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!");
         return false;
     }
     // create project
     $this->projectStructure['ppassword'] = $this->_generatePassword();
     $this->projectStructure['user_ip'] = Utils::getRealIpAddr();
     $this->projectStructure['id_customer'] = 'translated_user';
     $this->projectStructure['id_project'] = insertProject($this->projectStructure);
     //create user (Massidda 2013-01-24)
     //check if all the keys are valid MyMemory keys
     if (!empty($this->projectStructure['private_tm_key'])) {
         foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) {
             $this->tmxServiceWrapper->setTmKey($_tmKey['key']);
             try {
                 $keyExists = $this->tmxServiceWrapper->checkCorrectKey();
                 if (!isset($keyExists) || $keyExists === false) {
                     Log::doLog(__METHOD__ . " -> TM key is not valid.");
                     throw new Exception("TM key is not valid.", -4);
                 }
             } catch (Exception $e) {
                 $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
                 return false;
             }
             //set the first key as primary
             $this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']);
         }
         //check if the MyMemory keys provided by the user are already associated to him.
         if ($this->projectStructure['userIsLogged']) {
             $mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler);
             $searchMemoryKey = new TmKeyManagement_MemoryKeyStruct();
             $searchMemoryKey->uid = $this->projectStructure['uid'];
             $userMemoryKeys = $mkDao->read($searchMemoryKey);
             $userTmKeys = array();
             $memoryKeysToBeInserted = array();
             //extract user tm keys
             foreach ($userMemoryKeys as $_memoKey) {
                 /**
                  * @var $_memoKey TmKeyManagement_MemoryKeyStruct
                  */
                 $userTmKeys[] = $_memoKey->tm_key->key;
             }
             foreach ($this->projectStructure['private_tm_key'] as $_tmKey) {
                 if (!in_array($_tmKey['key'], $userTmKeys)) {
                     $newMemoryKey = new TmKeyManagement_MemoryKeyStruct();
                     $newTmKey = new TmKeyManagement_TmKeyStruct();
                     $newTmKey->key = $_tmKey['key'];
                     $newTmKey->tm = true;
                     $newTmKey->glos = true;
                     //TODO: take this from input
                     $newTmKey->name = $_tmKey['name'];
                     $newMemoryKey->tm_key = $newTmKey;
                     $newMemoryKey->uid = $this->projectStructure['uid'];
                     $memoryKeysToBeInserted[] = $newMemoryKey;
                 } else {
                     Log::doLog('skip insertion');
                 }
             }
             try {
                 $mkDao->createList($memoryKeysToBeInserted);
             } catch (Exception $e) {
                 Log::doLog($e->getMessage());
                 # Here we handle the error, displaying HTML, logging, ...
                 $output = "<pre>\n";
                 $output .= $e->getMessage() . "\n\t";
                 $output .= "</pre>";
                 Utils::sendErrMailReport($output);
             }
         }
         //the base case is when the user clicks on "generate private TM" button:
         //a (user, pass, key) tuple is generated and can be inserted
         //if it comes with it's own key without querying the creation API, create a (key,key,key) user
         if (empty($this->projectStructure['private_tm_user'])) {
             $this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key'];
             $this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key'];
         }
         insertTranslator($this->projectStructure);
     }
     //sort files in order to process TMX first
     $sortedFiles = array();
     foreach ($this->projectStructure['array_files'] as $fileName) {
         if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
             //found TMX, enable language checking routines
             $this->checkTMX = 1;
             array_unshift($sortedFiles, $fileName);
         } else {
             array_push($sortedFiles, $fileName);
         }
     }
     $this->projectStructure['array_files'] = $sortedFiles;
     unset($sortedFiles);
     $uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken'];
     //fetch cache links, created by converter, from upload directory
     $linkFiles = scandir($uploadDir);
     //remove dir hardlinks, as uninteresting, as weel as regular files; only hash-links
     foreach ($linkFiles as $k => $linkFile) {
         if (strpos($linkFile, '.') !== false or strpos($linkFile, '|') === false) {
             unset($linkFiles[$k]);
         }
     }
     /*
        loop through all input files to
        1)upload TMX
        2)convert, in case, non standard XLIFF files to a format that Matecat understands
     
        Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow
     */
     foreach ($this->projectStructure['array_files'] as $fileName) {
         //if TMX,
         if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
             //load it into MyMemory; we'll check later on how it went
             $file = new stdClass();
             $file->file_path = "{$uploadDir}/{$fileName}";
             $this->tmxServiceWrapper->setName($fileName);
             $this->tmxServiceWrapper->setFile(array($file));
             try {
                 $this->tmxServiceWrapper->addTmxInMyMemory();
             } catch (Exception $e) {
                 $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
                 return false;
             }
             //in any case, skip the rest of the loop, go to the next file
             continue;
         }
         /*
           Conversion Enforce
           Checking Extension is no more sufficient, we want check content if this is an idiom xlf file type, conversion are enforced
           $enforcedConversion = true; //( if conversion is enabled )
         */
         $isAnXliffToConvert = $this->isConversionToEnforce($fileName);
         //we are going to access the storage, get model object to manipulate it
         $fs = new FilesStorage();
         //if it's one of the listed formats or conversion is not enabled in first place
         if (!$isAnXliffToConvert) {
             /*
               filename is already an xliff and it's in upload directory
               we have to make a cache package from it to avoid altering the original path
             */
             //get file
             $filePathName = "{$uploadDir}/{$fileName}";
             //calculate hash + add the fileName, if i load 3 equal files with the same content
             // they will be squashed to the last one
             $sha1 = sha1(file_get_contents($filePathName) . $filePathName);
             //make a cache package (with work/ only, emtpy orig/)
             $fs->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName);
             //put reference to cache in upload dir to link cache to session
             $fs->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken']);
             //add newly created link to list
             $linkFiles[] = $sha1 . "|" . $this->projectStructure['source_language'];
             unset($sha1);
         }
     }
     //now, upload dir contains only hash-links
     //we start copying files to "file" dir, inserting metadata in db and extracting segments
     foreach ($linkFiles as $linkFile) {
         //converted file is inside cache directory
         //get hash from file name inside UUID dir
         $hashFile = basename($linkFile);
         $hashFile = explode('|', $hashFile);
         //use hash and lang to fetch file from package
         $xliffFilePathName = $fs->getXliffFromCache($hashFile[0], $hashFile[1]);
         //get sha
         $sha1_original = $hashFile[0];
         //get original file name
         $originalFilePathName = $fs->getOriginalFromCache($hashFile[0], $hashFile[1]);
         $raw_originalFilePathName = explode(DIRECTORY_SEPARATOR, $originalFilePathName);
         $fileName = array_pop($raw_originalFilePathName);
         unset($hashFile);
         if (!file_exists($xliffFilePathName)) {
             $this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload.");
         }
         try {
             $info = pathinfo($xliffFilePathName);
             if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) {
                 throw new Exception("Failed to find Xliff - no segments found", -3);
             }
             $mimeType = pathinfo($fileName, PATHINFO_EXTENSION);
             $yearMonthPath = date_create()->format("Ymd");
             $fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original;
             $fid = insertFile($this->projectStructure, $fileName, $mimeType, $fileDateSha1Path);
             //move the file in the right directory from the packages to the file dir
             $fs->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid);
             $this->projectStructure['file_id_list']->append($fid);
             $this->_extractSegments(file_get_contents($xliffFilePathName), $fid);
         } catch (Exception $e) {
             if ($e->getCode() == -1) {
                 $this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$fileName}.");
                 $fs->deleteHashFromUploadDir($uploadDir, $linkFile);
             } elseif ($e->getCode() == -2) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$fileName}");
             } elseif ($e->getCode() == -3) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$fileName} not found. Failed to save XLIFF conversion on disk");
             } elseif ($e->getCode() == -4) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$fileName} )");
             } elseif ($e->getCode() == -11) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied");
             } elseif ($e->getCode() == -12) {
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database");
             } elseif ($e->getCode() == -13) {
                 $this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage());
                 Log::doLog($e->getMessage());
                 return null;
                 // SEVERE EXCEPTION we can not write to disk!! Break project creation
             } else {
                 //mysql insert Blob Error
                 $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$fileName}. Please try again.");
             }
             Log::doLog($e->getMessage());
         }
     }
     //end of hash-link loop
     //check if the files language equals the source language. If not, set an error message.
     if (!$this->projectStructure['skip_lang_validation']) {
         $this->validateFilesLanguages();
     }
     /****************/
     //loop again through files to check to check for TMX loading
     foreach ($this->projectStructure['array_files'] as $fileName) {
         //if TMX,
         if ('tmx' == pathinfo($fileName, PATHINFO_EXTENSION)) {
             $this->tmxServiceWrapper->setName($fileName);
             $result = array();
             //is the TM loaded?
             //wait until current TMX is loaded
             while (true) {
                 try {
                     $result = $this->tmxServiceWrapper->tmxUploadStatus();
                     if ($result['completed']) {
                         //"$fileName" has been loaded into MyMemory"
                         //exit the loop
                         break;
                     }
                     //"waiting for "$fileName" to be loaded into MyMemory"
                     sleep(3);
                 } catch (Exception $e) {
                     $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage());
                     Log::doLog($e->getMessage() . "\n" . $e->getTraceAsString());
                     //exit project creation
                     return false;
                 }
             }
             //once the language is loaded, check if language is compliant (unless something useful has already been found)
             if (1 == $this->checkTMX) {
                 //get localized target languages of TM (in case it's a multilingual TM)
                 $tmTargets = explode(';', $result['data']['target_lang']);
                 //indicates if something has been found for current memory
                 $found = false;
                 //compare localized target languages array (in case it's a multilingual project) to the TM supplied
                 //if nothing matches, then the TM supplied can't have matches for this project
                 //create an empty var and add the source language too
                 $project_languages = array_merge((array) $this->projectStructure['target_language'], (array) $this->projectStructure['source_language']);
                 foreach ($project_languages as $projectTarget) {
                     if (in_array($this->langService->getLocalizedName($projectTarget), $tmTargets)) {
                         $found = true;
                         break;
                     }
                 }
                 //if this TM matches the project lagpair and something has been found
                 if ($found and $result['data']['source_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
                     //the TMX is good to go
                     $this->checkTMX = 0;
                 } elseif ($found and $result['data']['target_lang'] == $this->langService->getLocalizedName($this->projectStructure['source_language'])) {
                     /*
                      * This means that the TMX has a srclang as specification in the header. Warn the user.
                      * Ex:
                      * <header creationtool="SDL Language Platform"
                      *      creationtoolversion="8.0"
                      *      datatype="rtf"
                      *      segtype="sentence"
                      *      adminlang="DE-DE"
                      *      srclang="DE-DE" />
                      */
                     $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX you provided explicitly specifies {$result['data']['source_lang']} as source language. Check that the specified language source in the TMX file match the language source of your project or remove that specification in TMX file.");
                     $this->checkTMX = 0;
                     Log::doLog($this->projectStructure['result']);
                 }
             }
         }
     }
     if (1 == $this->checkTMX) {
         //this means that noone of uploaded TMX were usable for this project. Warn the user.
         $this->projectStructure['result']['errors'][] = array("code" => -16, "message" => "The TMX did not contain any usable segment. Check that the languages in the TMX file match the languages of your project.");
         Log::doLog($this->projectStructure['result']);
         return false;
     }
     if (!empty($this->projectStructure['result']['errors'])) {
         Log::doLog("Project Creation Failed. Sent to Output all errors.");
         Log::doLog($this->projectStructure['result']['errors']);
         return false;
     }
     //Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) );
     //create job
     if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) {
         $owner = $_SESSION['cid'];
     } else {
         $_SESSION['_anonym_pid'] = $this->projectStructure['id_project'];
         //default user
         $owner = '';
     }
     $isEmptyProject = false;
     //Throws exception
     try {
         $this->_createJobs($this->projectStructure, $owner);
         //FIXME for project with pre translation this query is not enough,
         //we need compare the number of segments with translations, but take an eye to the opensource
         $query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1";
         $string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy());
         $query_visible_segments = sprintf($query_visible_segments, $string_file_list);
         $rows = $this->dbHandler->fetch_array($query_visible_segments);
         if (!$rows) {
             Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: " . var_export($this->dbHandler->get_error(), true) . " - \n");
             throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5);
         }
         if ($rows[0]['cattool_segments'] == 0) {
             Log::doLog("Segment Search: No segments in this project - \n");
             $isEmptyProject = true;
         }
     } catch (Exception $ex) {
         $this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )");
         return false;
     }
     try {
         Utils::deleteDir($uploadDir);
         if (is_dir($uploadDir . '_converted')) {
             Utils::deleteDir($uploadDir . '_converted');
         }
     } catch (Exception $e) {
         $output = "<pre>\n";
         $output .= " - Exception: " . print_r($e->getMessage(), true) . "\n";
         $output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
         $output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
         $output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n";
         $output .= "\n\t";
         $output .= "Aborting...\n";
         $output .= "</pre>";
         Log::doLog($output);
         Utils::sendErrMailReport($output, $e->getMessage());
     }
     $this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE;
     if ($isEmptyProject) {
         $this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY;
     }
     $this->projectStructure['result']['code'] = 1;
     $this->projectStructure['result']['data'] = "OK";
     $this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword'];
     $this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass'];
     $this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list'];
     $this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments'];
     $this->projectStructure['result']['id_project'] = $this->projectStructure['id_project'];
     $this->projectStructure['result']['project_name'] = $this->projectStructure['project_name'];
     $this->projectStructure['result']['source_language'] = $this->projectStructure['source_language'];
     $this->projectStructure['result']['target_language'] = $this->projectStructure['target_language'];
     $this->projectStructure['result']['status'] = $this->projectStructure['status'];
     $this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files'];
     $query_project_summary = "\n            SELECT\n                 COUNT( s.id ) AS project_segments,\n                 SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n            FROM segments s\n            INNER JOIN files_job fj ON fj.id_file = s.id_file\n            INNER JOIN jobs j ON j.id= fj.id_job\n            LEFT JOIN segment_translations st ON s.id = st.id_segment\n            WHERE j.id_project = %u\n        ";
     $query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']);
     $project_summary = $this->dbHandler->fetch_array($query_project_summary);
     $update_project_count = "\n            UPDATE projects\n              SET\n                standard_analysis_wc = %.2F,\n                status_analysis = '%s'\n            WHERE id = %u\n        ";
     $update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']);
     $this->dbHandler->query($update_project_count);
     //        Log::doLog( $this->projectStructure );
     //create Project into DQF queue
     if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) {
         $dqfProjectStruct = DQF_DqfProjectStruct::getStruct();
         $dqfProjectStruct->api_key = $this->projectStructure['dqf_key'];
         $dqfProjectStruct->project_id = $this->projectStructure['id_project'];
         $dqfProjectStruct->name = $this->projectStructure['project_name'];
         $dqfProjectStruct->source_language = $this->projectStructure['source_language'];
         $dqfQueue = new Analysis_DqfQueueHandler();
         try {
             $dqfQueue->createProject($dqfProjectStruct);
             //for each job, push a task into AMQ's DQF queue
             foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) {
                 /**
                  * @var $dqfTaskStruct DQF_DqfTaskStruct
                  */
                 $dqfTaskStruct = DQF_DqfTaskStruct::getStruct();
                 $dqfTaskStruct->api_key = $this->projectStructure['dqf_key'];
                 $dqfTaskStruct->project_id = $this->projectStructure['id_project'];
                 $dqfTaskStruct->task_id = $jobID;
                 $dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i];
                 $dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name'];
                 $dqfQueue->createTask($dqfTaskStruct);
             }
         } catch (Exception $exn) {
             $output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage();
             Log::doLog($output);
             Utils::sendErrMailReport($output, $exn->getMessage());
         }
     }
 }