public function createProject() { // project name sanitize $oldName = $this->projectStructure['project_name']; $this->projectStructure['project_name'] = $this->_sanitizeName($this->projectStructure['project_name']); if ($this->projectStructure['project_name'] == false) { $this->projectStructure['result']['errors'][] = array("code" => -5, "message" => "Invalid Project Name " . $oldName . ": it should only contain numbers and letters!"); return false; } // create project? $this->projectStructure['ppassword'] = $this->_generatePassword(); $this->projectStructure['user_ip'] = Utils::getRealIpAddr(); $this->projectStructure['id_project'] = insertProject($this->projectStructure); //create user (Massidda 2013-01-24) //check if all the keys are valid MyMemory keys if (!empty($this->projectStructure['private_tm_key'])) { foreach ($this->projectStructure['private_tm_key'] as $i => $_tmKey) { $this->tmxServiceWrapper->setTmKey($_tmKey['key']); try { $keyExists = $this->tmxServiceWrapper->checkCorrectKey(); if (!isset($keyExists) || $keyExists === false) { Log::doLog(__METHOD__ . " -> TM key is not valid."); throw new Exception("TM key is not valid: " . $_tmKey['key'], -4); } } catch (Exception $e) { $this->projectStructure['result']['errors'][] = array("code" => $e->getCode(), "message" => $e->getMessage()); return false; } //set the first key as primary $this->tmxServiceWrapper->setTmKey($this->projectStructure['private_tm_key'][0]['key']); } //check if the MyMemory keys provided by the user are already associated to him. if ($this->projectStructure['userIsLogged']) { $mkDao = new TmKeyManagement_MemoryKeyDao($this->dbHandler); $searchMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $searchMemoryKey->uid = $this->projectStructure['uid']; $userMemoryKeys = $mkDao->read($searchMemoryKey); $userTmKeys = array(); $memoryKeysToBeInserted = array(); //extract user tm keys foreach ($userMemoryKeys as $_memoKey) { /** * @var $_memoKey TmKeyManagement_MemoryKeyStruct */ $userTmKeys[] = $_memoKey->tm_key->key; } foreach ($this->projectStructure['private_tm_key'] as $_tmKey) { if (!in_array($_tmKey['key'], $userTmKeys)) { $newMemoryKey = new TmKeyManagement_MemoryKeyStruct(); $newTmKey = new TmKeyManagement_TmKeyStruct(); $newTmKey->key = $_tmKey['key']; $newTmKey->tm = true; $newTmKey->glos = true; //TODO: take this from input $newTmKey->name = $_tmKey['name']; $newMemoryKey->tm_key = $newTmKey; $newMemoryKey->uid = $this->projectStructure['uid']; $memoryKeysToBeInserted[] = $newMemoryKey; } else { Log::doLog('skip insertion'); } } try { $mkDao->createList($memoryKeysToBeInserted); } catch (Exception $e) { Log::doLog($e->getMessage()); # Here we handle the error, displaying HTML, logging, ... $output = "<pre>\n"; $output .= $e->getMessage() . "\n\t"; $output .= "</pre>"; Utils::sendErrMailReport($output); } } //the base case is when the user clicks on "generate private TM" button: //a (user, pass, key) tuple is generated and can be inserted //if it comes with it's own key without querying the creation API, create a (key,key,key) user if (empty($this->projectStructure['private_tm_user'])) { $this->projectStructure['private_tm_user'] = $this->projectStructure['private_tm_key'][0]['key']; $this->projectStructure['private_tm_pass'] = $this->projectStructure['private_tm_key'][0]['key']; } insertTranslator($this->projectStructure); } //sort files in order to process TMX first $sortedFiles = array(); foreach ($this->projectStructure['array_files'] as $fileName) { //check for glossary files and tmx and put them in front of the list $infoFile = DetectProprietaryXliff::getInfo($fileName); if (DetectProprietaryXliff::getMemoryFileType()) { //found TMX, enable language checking routines if (DetectProprietaryXliff::isTMXFile()) { $this->checkTMX = 1; } //not used at moment but needed if we want to do a poll for status if (DetectProprietaryXliff::isGlossaryFile()) { $this->checkGlossary = 1; } //prepend in front of the list array_unshift($sortedFiles, $fileName); } else { //append at the end of the list array_push($sortedFiles, $fileName); } } $this->projectStructure['array_files'] = $sortedFiles; unset($sortedFiles); $uploadDir = $this->uploadDir = INIT::$UPLOAD_REPOSITORY . DIRECTORY_SEPARATOR . $this->projectStructure['uploadToken']; //we are going to access the storage, get model object to manipulate it $this->fileStorage = new FilesStorage(); $linkFiles = $this->fileStorage->getHashesFromDir($this->uploadDir); /* loop through all input files to 1) upload TMX and Glossaries */ try { $this->_pushTMXToMyMemory(); } catch (Exception $e) { //exit project creation return false; } //TMX Management /* loop through all input files to 2)convert, in case, non standard XLIFF files to a format that Matecat understands Note that XLIFF that don't need conversion are moved anyway as they are to cache in order not to alter the workflow */ foreach ($this->projectStructure['array_files'] as $fileName) { /* Conversion Enforce Checking Extension is no more sufficient, we want check content $enforcedConversion = true; //( if conversion is enabled ) */ $isAFileToConvert = $this->isConversionToEnforce($fileName); //if it's one of the listed formats or conversion is not enabled in first place if (!$isAFileToConvert) { /* filename is already an xliff and it's in upload directory we have to make a cache package from it to avoid altering the original path */ //get file $filePathName = "{$this->uploadDir}/{$fileName}"; //calculate hash + add the fileName, if i load 3 equal files with the same content // they will be squashed to the last one $sha1 = sha1_file($filePathName); //make a cache package (with work/ only, emtpy orig/) $this->fileStorage->makeCachePackage($sha1, $this->projectStructure['source_language'], false, $filePathName); //put reference to cache in upload dir to link cache to session $this->fileStorage->linkSessionToCache($sha1, $this->projectStructure['source_language'], $this->projectStructure['uploadToken'], $fileName); //add newly created link to list $linkFiles['conversionHashes']['sha'][] = $sha1 . "|" . $this->projectStructure['source_language']; $linkFiles['conversionHashes']['fileName'][$sha1 . "|" . $this->projectStructure['source_language']][] = $fileName; //when the same sdlxliff is uploaded more than once with different names $linkFiles['conversionHashes']['sha'] = array_unique($linkFiles['conversionHashes']['sha']); unset($sha1); } } //now, upload dir contains only hash-links //we start copying files to "file" dir, inserting metadata in db and extracting segments foreach ($linkFiles['conversionHashes']['sha'] as $linkFile) { //converted file is inside cache directory //get hash from file name inside UUID dir $hashFile = FilesStorage::basename_fix($linkFile); $hashFile = explode('|', $hashFile); //use hash and lang to fetch file from package $cachedXliffFilePathName = $this->fileStorage->getXliffFromCache($hashFile[0], $hashFile[1]); //get sha $sha1_original = $hashFile[0]; //associate the hash to the right file in upload directory //get original file name, to insert into DB and cp in storage //PLEASE NOTE, this can be an array when the same file added more // than once and with different names $_originalFileName = $linkFiles['conversionHashes']['fileName'][$linkFile]; unset($hashFile); if (!file_exists($cachedXliffFilePathName)) { $this->projectStructure['result']['errors'][] = array("code" => -6, "message" => "File not found on server after upload."); } try { $info = FilesStorage::pathinfo_fix($cachedXliffFilePathName); if (!in_array($info['extension'], array('xliff', 'sdlxliff', 'xlf'))) { throw new Exception("Failed to find Xliff - no segments found", -3); } $yearMonthPath = date_create($this->projectStructure['create_date'])->format('Ymd'); $fileDateSha1Path = $yearMonthPath . DIRECTORY_SEPARATOR . $sha1_original; //PLEASE NOTE, this can be an array when the same file added more // than once and with different names foreach ($_originalFileName as $originalFileName) { $mimeType = FilesStorage::pathinfo_fix($originalFileName, PATHINFO_EXTENSION); $fid = insertFile($this->projectStructure, $originalFileName, $mimeType, $fileDateSha1Path); //move the file in the right directory from the packages to the file dir $this->fileStorage->moveFromCacheToFileDir($fileDateSha1Path, $this->projectStructure['source_language'], $fid, $originalFileName); $this->projectStructure['file_id_list']->append($fid); $this->_extractSegments(file_get_contents($cachedXliffFilePathName), $fid); } } catch (Exception $e) { if ($e->getCode() == -1) { $this->projectStructure['result']['errors'][] = array("code" => -1, "message" => "No text to translate in the file {$originalFileName}."); $this->fileStorage->deleteHashFromUploadDir($this->uploadDir, $linkFile); } elseif ($e->getCode() == -2) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store segments in database for {$originalFileName}"); } elseif ($e->getCode() == -3) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "File {$originalFileName} not found. Failed to save XLIFF conversion on disk"); } elseif ($e->getCode() == -4) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Internal Error. Xliff Import: Error parsing. ( {$originalFileName} )"); } elseif ($e->getCode() == -11) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files on disk. Permission denied"); } elseif ($e->getCode() == -12) { $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to store reference files in database"); } elseif ($e->getCode() == -13) { $this->projectStructure['result']['errors'][] = array("code" => -13, "message" => $e->getMessage()); Log::doLog($e->getMessage()); return null; // SEVERE EXCEPTION we can not write to disk!! Break project creation } else { //mysql insert Blob Error $this->projectStructure['result']['errors'][] = array("code" => -7, "message" => "Failed to create project. Database Error on {$originalFileName}. Please try again."); } Log::doLog($e->getMessage()); Log::doLog($e->getTraceAsString()); } } //end of conversion hash-link loop try { $this->_zipFileHandling($linkFiles); } catch (Exception $e) { //exit project creation return false; } //check if the files language equals the source language. If not, set an error message. if (!$this->projectStructure['skip_lang_validation']) { $this->validateFilesLanguages(); } if (!$this->_doCheckForErrors()) { //exit project creation return false; } //Log::doLog( array_pop( array_chunk( $SegmentTranslations[$fid], 25, true ) ) ); //create job if (isset($_SESSION['cid']) and !empty($_SESSION['cid'])) { $owner = $_SESSION['cid']; } else { $_SESSION['_anonym_pid'] = $this->projectStructure['id_project']; //default user $owner = ''; } $isEmptyProject = false; //Throws exception try { $this->_createJobs($this->projectStructure, $owner); //FIXME for project with pre translation this query is not enough, //we need compare the number of segments with translations, but take an eye to the opensource $query_visible_segments = "SELECT count(*) as cattool_segments\n\t\t\t\tFROM segments WHERE id_file IN ( %s ) and show_in_cattool = 1"; $string_file_list = implode(",", $this->projectStructure['file_id_list']->getArrayCopy()); $query_visible_segments = sprintf($query_visible_segments, $string_file_list); try { $rows = $this->dbHandler->fetch_array($query_visible_segments); } catch (PDOException $e) { Log::doLog("Segment Search: Failed Retrieve min_segment/max_segment for files ( {$string_file_list} ) - DB Error: {$e->getMessage()} - \n"); throw new Exception("Segment Search: Failed Retrieve min_segment/max_segment for job", -5); } if ($rows[0]['cattool_segments'] == 0) { Log::doLog("Segment Search: No segments in this project - \n"); $isEmptyProject = true; } } catch (Exception $ex) { $this->projectStructure['result']['errors'][] = array("code" => -9, "message" => "Fail to create Job. ( {$ex->getMessage()} )"); return false; } try { Utils::deleteDir($this->uploadDir); if (is_dir($this->uploadDir . '_converted')) { Utils::deleteDir($this->uploadDir . '_converted'); } } catch (Exception $e) { $output = "<pre>\n"; $output .= " - Exception: " . print_r($e->getMessage(), true) . "\n"; $output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n"; $output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n"; $output .= " - Trace: \n" . print_r($e->getTraceAsString(), true) . "\n"; $output .= "\n\t"; $output .= "Aborting...\n"; $output .= "</pre>"; Log::doLog($output); Utils::sendErrMailReport($output, $e->getMessage()); } $this->projectStructure['status'] = INIT::$VOLUME_ANALYSIS_ENABLED ? Constants_ProjectStatus::STATUS_NEW : Constants_ProjectStatus::STATUS_NOT_TO_ANALYZE; if ($isEmptyProject) { $this->projectStructure['status'] = Constants_ProjectStatus::STATUS_EMPTY; } $this->projectStructure['result']['code'] = 1; $this->projectStructure['result']['data'] = "OK"; $this->projectStructure['result']['ppassword'] = $this->projectStructure['ppassword']; $this->projectStructure['result']['password'] = $this->projectStructure['array_jobs']['job_pass']; $this->projectStructure['result']['id_job'] = $this->projectStructure['array_jobs']['job_list']; $this->projectStructure['result']['job_segments'] = $this->projectStructure['array_jobs']['job_segments']; $this->projectStructure['result']['id_project'] = $this->projectStructure['id_project']; $this->projectStructure['result']['project_name'] = $this->projectStructure['project_name']; $this->projectStructure['result']['source_language'] = $this->projectStructure['source_language']; $this->projectStructure['result']['target_language'] = $this->projectStructure['target_language']; $this->projectStructure['result']['status'] = $this->projectStructure['status']; $this->projectStructure['result']['lang_detect'] = $this->projectStructure['lang_detect_files']; $query_project_summary = "\n SELECT\n COUNT( s.id ) AS project_segments,\n SUM( IF( IFNULL( st.eq_word_count, -1 ) = -1, s.raw_word_count, st.eq_word_count ) ) AS project_raw_wordcount\n FROM segments s\n INNER JOIN files_job fj ON fj.id_file = s.id_file\n INNER JOIN jobs j ON j.id= fj.id_job\n LEFT JOIN segment_translations st ON s.id = st.id_segment\n WHERE j.id_project = %u\n "; $query_project_summary = sprintf($query_project_summary, $this->projectStructure['id_project']); $project_summary = $this->dbHandler->fetch_array($query_project_summary); $update_project_count = "\n UPDATE projects\n SET\n standard_analysis_wc = %.2F,\n status_analysis = '%s'\n WHERE id = %u\n "; $update_project_count = sprintf($update_project_count, $project_summary[0]['project_raw_wordcount'], $this->projectStructure['status'], $this->projectStructure['id_project']); $this->dbHandler->query($update_project_count); // Log::doLog( $this->projectStructure ); //create Project into DQF queue if (INIT::$DQF_ENABLED && !empty($this->projectStructure['dqf_key'])) { $dqfProjectStruct = DQF_DqfProjectStruct::getStruct(); $dqfProjectStruct->api_key = $this->projectStructure['dqf_key']; $dqfProjectStruct->project_id = $this->projectStructure['id_project']; $dqfProjectStruct->name = $this->projectStructure['project_name']; $dqfProjectStruct->source_language = $this->projectStructure['source_language']; $dqfQueue = new Analysis_DqfQueueHandler(); try { $projectManagerInfo = $dqfQueue->checkProjectManagerKey($this->projectStructure['dqf_key']); $dqfQueue->createProject($dqfProjectStruct); //for each job, push a task into AMQ's DQF queue foreach ($this->projectStructure['array_jobs']['job_list'] as $i => $jobID) { /** * @var $dqfTaskStruct DQF_DqfTaskStruct */ $dqfTaskStruct = DQF_DqfTaskStruct::getStruct(); $dqfTaskStruct->api_key = $this->projectStructure['dqf_key']; $dqfTaskStruct->project_id = $this->projectStructure['id_project']; $dqfTaskStruct->task_id = $jobID; $dqfTaskStruct->target_language = $this->projectStructure['target_language'][$i]; $dqfTaskStruct->file_name = uniqid('', true) . $this->projectStructure['project_name']; $dqfQueue->createTask($dqfTaskStruct); } } catch (Exception $exn) { $output = __METHOD__ . " (code " . $exn->getCode() . " ) - " . $exn->getMessage(); Log::doLog($output); Utils::sendErrMailReport($output, $exn->getMessage()); } } }