public function __construct()
{
parent::__construct();
self::$last_job_file_name = dirname(__FILE__) . DIRECTORY_SEPARATOR . '.lastjobprocessed_jpeer';
self::$sleeptime = 60 * 60 * 24 * 30 * 1;
Log::$fileName = "evaluatePEE.log";
}
public function __construct() {
parent::__construct();
$filterArgs = array(
'id' => array( 'filter' => FILTER_SANITIZE_NUMBER_INT ),
'id_job' => array( 'filter' => FILTER_SANITIZE_NUMBER_INT ),
'src_content' => array( 'filter' => FILTER_UNSAFE_RAW ),
'trg_content' => array( 'filter' => FILTER_UNSAFE_RAW ),
'password' => array( 'filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH ),
'token' => array( 'filter' => FILTER_SANITIZE_STRING,
'flags' => FILTER_FLAG_STRIP_LOW ),
'logs' => array( 'filter' => FILTER_UNSAFE_RAW),
'glossaryList' => array( 'filter' => FILTER_CALLBACK, 'options' => array('self','filterString') )
);
$this->__postInput = (object)filter_input_array( INPUT_POST, $filterArgs );
if( !empty( $this->__postInput->logs ) && $this->__postInput->logs != '[]' ){
Log::$fileName = 'clientLog.log';
Log::doLog( json_decode( $this->__postInput->logs ) );
Log::$fileName = 'log.txt';
}
}
public function __construct()
{
parent::__construct();
$filterArgs = array('id_job' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'id_translator' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW), 'id_customer' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW), 'private_customer' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'private_translator' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'password' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH), 'source' => array('filter' => FILTER_UNSAFE_RAW), 'target' => array('filter' => FILTER_UNSAFE_RAW), 'source_lang' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH), 'target_lang' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH));
$this->__postInput = filter_input_array(INPUT_POST, $filterArgs);
//NOTE: This is for debug purpose only,
//NOTE: Global $_POST Overriding from CLI Test scripts
//$this->__postInput = filter_var_array( $_POST, $filterArgs );
$this->id_job = (int) $this->__postInput['id_job'];
$this->translator_username = $this->__postInput['id_translator'];
// No more used
$this->id_customer = $this->__postInput['id_customer'];
$this->private_customer = (int) $this->__postInput['private_customer'];
$this->private_translator = (int) $this->__postInput['private_translator'];
$this->password = $this->__postInput['password'];
$this->source = $this->__postInput['source'];
$this->target = $this->__postInput['target'];
$this->source_lang = $this->__postInput['source_lang'];
$this->target_lang = $this->__postInput['target_lang'];
if (empty($this->id_customer)) {
$this->id_customer = "Anonymous";
}
//XXX log contributions
Log::$fileName = "setContribution.txt";
Log::doLog($this->__postInput);
Log::$fileName = "log.txt";
}
public function __construct()
{
parent::__construct();
Log::$fileName = "languageStats.log";
self::$sleeptime = 10;
//60 * 60 * 24 * 30 * 1;
}
protected function _checkData($logName = 'log.txt')
{
//change Log file
Log::$fileName = $logName;
$this->parseIDSegment();
if (empty($this->id_segment)) {
$this->result['errors'][] = array("code" => -1, "message" => "missing id_segment");
}
// $this->result[ 'errors' ][ ] = array( "code" => -1, "message" => "prova" );
// throw new Exception( "prova", -1 );
//strtoupper transforms null to "" so check for the first element to be an empty string
if (!empty($this->split_statuses[0]) && !empty($this->split_num)) {
if (count(array_unique($this->split_statuses)) == 1) {
//IF ALL translation chunks are in the same status, we take the status for the entire segment
$this->status = $this->split_statuses[0];
} else {
$this->status = Constants_TranslationStatus::STATUS_DRAFT;
}
foreach ($this->split_statuses as $pos => $value) {
$this->_checkForStatus($value);
}
} else {
$this->_checkForStatus($this->status);
}
if (empty($this->id_job)) {
$this->result['errors'][] = array("code" => -2, "message" => "missing id_job");
} else {
// $this->result[ 'error' ][ ] = array( "code" => -1000, "message" => "test 1" );
// throw new Exception( 'prova', -1 );
//get Job Info, we need only a row of jobs ( split )
$this->jobData = $job_data = getJobData((int) $this->id_job, $this->password);
if (empty($job_data)) {
$msg = "Error : empty job data \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
}
//add check for job status archived.
if (strtolower($job_data['status']) == Constants_JobStatus::STATUS_ARCHIVED) {
$this->result['errors'][] = array("code" => -3, "message" => "job archived");
}
//check for Password correctness ( remove segment split )
$pCheck = new AjaxPasswordCheck();
if (empty($job_data) || !$pCheck->grantJobAccessByJobData($job_data, $this->password, $this->id_segment)) {
$this->result['errors'][] = array("code" => -10, "message" => "wrong password");
}
}
//ONE OR MORE ERRORS OCCURRED : EXITING
if (!empty($this->result['errors'])) {
$msg = "Error \n\n " . var_export(array_merge($this->result, $_POST), true);
throw new Exception($msg, -1);
}
if (is_null($this->translation) || $this->translation === '') {
Log::doLog("Empty Translation \n\n" . var_export($_POST, true));
// won't save empty translation but there is no need to return an errors
throw new Exception("Empty Translation \n\n" . var_export($_POST, true), 0);
}
}
/**
* Executor constructor.
*
* @param Context $_context
*/
protected function __construct(Context $_context)
{
$this->_executorPID = posix_getpid();
\Log::$fileName = $_context->loggerName;
$this->_executionContext = $_context;
try {
$this->_queueHandler = new AMQHandler();
if (!$this->_queueHandler->getRedisClient()->sadd($this->_executionContext->pid_set_name, $this->_executorPID)) {
throw new \Exception("(Executor {$this->_executorPID}) : FATAL !! cannot create my resource ID. Exiting!");
} else {
$this->_logMsg("(Executor {$this->_executorPID}) : spawned !!!");
}
$this->_queueHandler->subscribe($this->_executionContext->queue_name);
} catch (\Exception $ex) {
$msg = "****** No REDIS/AMQ instances found. Exiting. ******";
$this->_logMsg($msg);
$this->_logMsg($ex->getMessage());
die;
}
}
public function __construct()
{
parent::__construct();
$filterArgs = array('id' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'id_job' => array('filter' => FILTER_SANITIZE_NUMBER_INT), 'src_content' => array('filter' => FILTER_UNSAFE_RAW), 'trg_content' => array('filter' => FILTER_UNSAFE_RAW), 'password' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH), 'token' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW), 'logs' => array('filter' => FILTER_UNSAFE_RAW), 'glossaryList' => array('filter' => FILTER_CALLBACK, 'options' => array('self', 'filterString')), 'segment_status' => array('filter' => FILTER_SANITIZE_STRING, 'flags' => FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH));
$this->__postInput = (object) filter_input_array(INPUT_POST, $filterArgs);
/**
* Update 2015/08/11, roberto@translated.net
* getWarning needs the segment status too because of a bug:
* sometimes the client calls getWarning and sends an empty trg_content
* because the suggestion has not been loaded yet.
* This happens only if segment is in status NEW
*/
if (empty($this->__postInput->segment_status)) {
$this->__postInput->segment_status = 'draft';
}
if (!empty($this->__postInput->logs) && $this->__postInput->logs != '[]') {
Log::$fileName = 'clientLog.log';
Log::doLog(json_decode($this->__postInput->logs));
Log::$fileName = 'log.txt';
}
}
/**
* Controllers Factory
*
* Initialize the Controller Instance and route the
* API Calls to the right Controller
*
* @return mixed
*/
public static function getInstance()
{
if (isset($_REQUEST['api']) && filter_input(INPUT_GET, 'api', FILTER_VALIDATE_BOOLEAN)) {
if (!isset($_REQUEST['action']) || empty($_REQUEST['action'])) {
header("Location: " . INIT::$HTTPHOST . INIT::$BASEURL . "api/docs", true, 303);
//Redirect 303 See Other
die;
}
$_REQUEST['action'][0] = strtoupper($_REQUEST['action'][0]);
//PHP 5.2 compatibility, don't use a lambda function
$func = create_function('$c', 'return strtoupper($c[1]);');
$_REQUEST['action'] = preg_replace_callback('/_([a-z])/', $func, $_REQUEST['action']);
$_POST['action'] = $_REQUEST['action'];
//set the log to the API Log
Log::$fileName = 'API.log';
}
//Default : catController
$action = isset($_POST['action']) ? $_POST['action'] : (isset($_GET['action']) ? $_GET['action'] : 'cat');
$className = $action . "Controller";
//Put here all actions we want to be performed by ALL controllers
require_once INIT::$MODEL_ROOT . '/queries.php';
return new $className();
}
public static function fatalErrorHandler()
{
$errorType = array(E_CORE_ERROR => 'E_CORE_ERROR', E_COMPILE_ERROR => 'E_COMPILE_ERROR', E_ERROR => 'E_ERROR', E_USER_ERROR => 'E_USER_ERROR', E_RECOVERABLE_ERROR => 'E_RECOVERABLE_ERROR');
# Getting last error
$error = error_get_last();
# Checking if last error is a fatal error
switch ($error['type']) {
case E_CORE_ERROR:
case E_COMPILE_ERROR:
case E_ERROR:
case E_USER_ERROR:
case E_RECOVERABLE_ERROR:
ini_set('display_errors', 'Off');
if (!ob_get_level()) {
ob_start();
} else {
ob_end_clean();
ob_start();
}
debug_print_backtrace();
$output = ob_get_contents();
ob_end_clean();
# Here we handle the error, displaying HTML, logging, ...
$output .= "<pre>\n";
$output .= "[ {$errorType[$error['type']]} ]\n\t";
$output .= "{$error['message']}\n\t";
$output .= "Not Recoverable Error on line {$error['line']} in file " . $error['file'];
$output .= " - PHP " . PHP_VERSION . " (" . PHP_OS . ")\n";
$output .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$output .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$output .= "\n\t";
$output .= "Aborting...\n";
$output .= "</pre>";
Log::$fileName = 'fatal_errors.txt';
Log::doLog($output);
Utils::sendErrMailReport($output);
header("HTTP/1.1 200 OK");
if (isset($_SERVER['HTTP_X_REQUESTED_WITH']) && strtolower($_SERVER['HTTP_X_REQUESTED_WITH']) == 'xmlhttprequest' || $_SERVER['REQUEST_METHOD'] == 'POST') {
//json_rersponse
if (INIT::$EXCEPTION_DEBUG) {
echo json_encode(array("errors" => array(array("code" => -1000, "message" => $output)), "data" => array()));
} else {
echo json_encode(array("errors" => array(array("code" => -1000, "message" => "Oops we got an Error. Contact <a href='mailto:support@matecat.com'>support@matecat.com</a>")), "data" => array()));
}
} elseif (INIT::$EXCEPTION_DEBUG) {
echo $output;
}
break;
}
}
/**
* 设置日志文件名
*/
public static function setFileName($fileName = 'log.txt')
{
self::$fileName = $fileName;
}
public function doAction()
{
//get job language and data
//Fixed Bug: need a specific job, because we need The target Language
//Removed from within the foreach cycle, the job is always the same....
$jobData = $this->jobInfo = getJobData($this->id_job, $this->password);
$pCheck = new AjaxPasswordCheck();
//check for Password correctness
if (empty($jobData) || !$pCheck->grantJobAccessByJobData($jobData, $this->password)) {
$msg = "Error : wrong password provided for download \n\n " . var_export($_POST, true) . "\n";
Log::doLog($msg);
Utils::sendErrMailReport($msg);
return null;
}
//get storage object
$fs = new FilesStorage();
$files_job = $fs->getFilesForJob($this->id_job, $this->id_file);
$nonew = 0;
$output_content = array();
/*
the procedure:
1)original xliff file is read directly from disk; a file handler is obtained
2)the file is read chunk by chunk by a stream parser: for each trans-unit that is encountered, target is replaced (or added) with the corresponding translation obtained from the DB
3)the parsed portion of xliff in the buffer is flushed on temporary file
4)the temporary file is sent to the converter and an original file is obtained
5)the temporary file is deleted
*/
// This array will contain all the files of $files_job split by
// converter version.
$files_job_by_converter_version = array();
// Detect the converter's version to use for each file, then store
// file info accordingly.
foreach ($files_job as $file) {
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
$files_job_by_converter_version[$fileType['converter_version']][] = $file;
}
// Process files according to the converters' versions, one version
// at a time
foreach ($files_job_by_converter_version as $converter_version => $files_job) {
//file array is chuncked. Each chunk will be used for a parallel conversion request.
$files_job = array_chunk($files_job, self::FILES_CHUNK_SIZE);
foreach ($files_job as $chunk) {
$converter = new FileFormatConverter($converter_version);
$files_to_be_converted = array();
foreach ($chunk as $file) {
$mime_type = $file['mime_type'];
$fileID = $file['id_file'];
$current_filename = $file['filename'];
//get path for the output file converted to know it's right extension
$_fileName = explode(DIRECTORY_SEPARATOR, $file['xliffFilePath']);
$outputPath = INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/' . $fileID . '/' . uniqid('', true) . "_.out." . array_pop($_fileName);
//make dir if doesn't exist
if (!file_exists(dirname($outputPath))) {
Log::doLog('Create Directory ' . escapeshellarg(dirname($outputPath)) . '');
mkdir(dirname($outputPath), 0775, true);
}
$data = getSegmentsDownload($this->id_job, $this->password, $fileID, $nonew);
//prepare regexp for nest step
$regexpEntity = '/&#x(0[0-8BCEF]|1[0-9A-F]|7F);/u';
$regexpAscii = '/([\\x{00}-\\x{1F}\\x{7F}]{1})/u';
foreach ($data as $i => $k) {
//create a secondary indexing mechanism on segments' array; this will be useful
//prepend a string so non-trans unit id ( ex: numerical ) are not overwritten
$data['matecat|' . $k['internal_id']][] = $i;
//FIXME: temporary patch
$data[$i]['translation'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['translation']);
$data[$i]['segment'] = str_replace('<x id="nbsp"/>', ' ', $data[$i]['segment']);
//remove binary chars in some xliff files
$sanitized_src = preg_replace($regexpAscii, '', $data[$i]['segment']);
$sanitized_trg = preg_replace($regexpAscii, '', $data[$i]['translation']);
//clean invalid xml entities ( charactes with ascii < 32 and different from 0A, 0D and 09
$sanitized_src = preg_replace($regexpEntity, '', $sanitized_src);
$sanitized_trg = preg_replace($regexpEntity, '', $sanitized_trg);
if ($sanitized_src != null) {
$data[$i]['segment'] = $sanitized_src;
}
if ($sanitized_trg != null) {
$data[$i]['translation'] = $sanitized_trg;
}
}
//instatiate parser
$xsp = new SdlXliffSAXTranslationReplacer($file['xliffFilePath'], $data, Langs_Languages::getInstance()->getLangRegionCode($jobData['target']), $outputPath);
if ($this->download_type == 'omegat') {
$xsp->setSourceInTarget(true);
}
//run parsing
Log::doLog("work on " . $fileID . " " . $current_filename);
$xsp->replaceTranslation();
//free memory
unset($xsp);
unset($data);
$output_content[$fileID]['document_content'] = file_get_contents($outputPath);
$output_content[$fileID]['output_filename'] = $current_filename;
$fileType = DetectProprietaryXliff::getInfo($file['xliffFilePath']);
if ($this->forceXliff) {
//clean the output filename by removing
// the unique hash identifier 55e5739b467109.05614837_.out.Test_English.doc.sdlxliff
$output_content[$fileID]['output_filename'] = preg_replace('#[0-9a-f]+\\.[0-9_]+\\.out\\.#i', '', FilesStorage::basename_fix($outputPath));
if ($fileType['proprietary_short_name'] === 'matecat_converter') {
// Set the XLIFF extension to .xlf
// Internally, MateCat continues using .sdlxliff as default
// extension for the XLIFF behind the projects.
// Changing this behavior requires a huge refactoring that
// it's scheduled for future versions.
// We quickly fixed the behaviour from the user standpoint
// using the following line of code, that changes the XLIFF's
// extension just a moment before it is downloaded by the user.
$output_content[$fileID]['output_filename'] = preg_replace("|\\.sdlxliff\$|i", ".xlf", $output_content[$fileID]['output_filename']);
}
}
/**
* Conversion Enforce
*/
$convertBackToOriginal = true;
//if it is a not converted file ( sdlxliff ) we have originalFile equals to xliffFile (it has just been copied)
$file['original_file'] = file_get_contents($file['originalFilePath']);
// When the 'proprietary' flag is set to false, the xliff
// is not passed to any converter, because is handled
// directly inside MateCAT.
$xliffWasNotConverted = $fileType['proprietary'] === false;
if (!INIT::$CONVERSION_ENABLED || ($file['originalFilePath'] == $file['xliffFilePath'] and $xliffWasNotConverted) or $this->forceXliff) {
$convertBackToOriginal = false;
Log::doLog("SDLXLIFF: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
} else {
//TODO: dos2unix ??? why??
//force unix type files
Log::doLog("NO SDLXLIFF, Conversion enforced: {$file['filename']} --- " . var_export($convertBackToOriginal, true));
}
if ($convertBackToOriginal) {
$output_content[$fileID]['out_xliff_name'] = $outputPath;
$output_content[$fileID]['source'] = $jobData['source'];
$output_content[$fileID]['target'] = $jobData['target'];
$files_to_be_converted[$fileID] = $output_content[$fileID];
} elseif ($this->forceXliff) {
$this->cleanFilePath($output_content[$fileID]['document_content']);
}
}
$convertResult = $converter->multiConvertToOriginal($files_to_be_converted, $chosen_machine = false);
foreach (array_keys($files_to_be_converted) as $fileID) {
$output_content[$fileID]['document_content'] = $this->ifGlobalSightXliffRemoveTargetMarks($convertResult[$fileID]['document_content'], $files_to_be_converted[$fileID]['output_filename']);
//in case of .strings, they are required to be in UTF-16
//get extension to perform file detection
$extension = FilesStorage::pathinfo_fix($output_content[$fileID]['output_filename'], PATHINFO_EXTENSION);
if (strtoupper($extension) == 'STRINGS') {
//use this function to convert stuff
$encodingConvertedFile = CatUtils::convertEncoding('UTF-16', $output_content[$fileID]['document_content']);
//strip previously added BOM
$encodingConvertedFile[1] = $converter->stripBOM($encodingConvertedFile[1], 16);
//store new content
$output_content[$fileID]['document_content'] = $encodingConvertedFile[1];
//trash temporary data
unset($encodingConvertedFile);
}
}
unset($convertResult);
}
}
foreach ($output_content as $idFile => $fileInformations) {
$zipPathInfo = ZipArchiveExtended::zipPathInfo($output_content[$idFile]['output_filename']);
if (is_array($zipPathInfo)) {
$output_content[$idFile]['zipfilename'] = $zipPathInfo['zipfilename'];
$output_content[$idFile]['zipinternalPath'] = $zipPathInfo['dirname'];
$output_content[$idFile]['output_filename'] = $zipPathInfo['basename'];
}
}
//set the file Name
$pathinfo = FilesStorage::pathinfo_fix($this->fname);
$this->_filename = $pathinfo['filename'] . "_" . $jobData['target'] . "." . $pathinfo['extension'];
//qui prodest to check download type?
if ($this->download_type == 'omegat') {
$this->_filename .= ".zip";
$tmsService = new TMSService();
$tmsService->setOutputType('tm');
/**
* @var $tmFile SplTempFileObject
*/
$tmFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tmsService->setOutputType('mt');
/**
* @var $mtFile SplTempFileObject
*/
$mtFile = $tmsService->exportJobAsTMX($this->id_job, $this->password, $jobData['source'], $jobData['target']);
$tm_id = uniqid('tm');
$mt_id = uniqid('mt');
$output_content[$tm_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_TM . tmx");
foreach ($tmFile as $lineNumber => $content) {
$output_content[$tm_id]['document_content'] .= $content;
}
$output_content[$mt_id] = array('document_content' => '', 'output_filename' => $pathinfo['filename'] . "_" . $jobData['target'] . "_MT . tmx");
foreach ($mtFile as $lineNumber => $content) {
$output_content[$mt_id]['document_content'] .= $content;
}
$this->createOmegaTZip($output_content, $jobData['source'], $jobData['target']);
//add zip archive content here;
} else {
try {
$output_content = $this->getOutputContentsWithZipFiles($output_content);
if (count($output_content) > 1) {
//cast $output_content elements to ZipContentObject
foreach ($output_content as $key => $__output_content_elem) {
$output_content[$key] = new ZipContentObject($__output_content_elem);
}
if ($pathinfo['extension'] != 'zip') {
if ($this->forceXliff) {
$this->_filename = $this->id_job . ".zip";
} else {
$this->_filename = $pathinfo['basename'] . ".zip";
}
}
$this->content = self::composeZip($output_content);
//add zip archive content here;
} else {
//always an array with 1 element, pop it, Ex: array( array() )
$output_content = array_pop($output_content);
$this->setContent($output_content);
}
} catch (Exception $e) {
$msg = "\n\n Error retrieving file content, Conversion failed??? \n\n Error: {$e->getMessage()} \n\n" . var_export($e->getTraceAsString(), true);
$msg .= "\n\n Request: " . var_export($_REQUEST, true);
Log::$fileName = 'fatal_errors.txt';
Log::doLog($msg);
Utils::sendErrMailReport($msg);
$this->unlockToken(array("code" => -110, "message" => "Download failed. Please contact " . INIT::$SUPPORT_MAIL));
throw $e;
// avoid sent Headers and empty file content with finalize method
}
}
try {
Utils::deleteDir(INIT::$TMP_DOWNLOAD . '/' . $this->id_job . '/');
} catch (Exception $e) {
Log::doLog('Failed to delete dir:' . $e->getMessage());
}
}
ini_set("memory_limit", "2048M");
set_time_limit(0);
include_once 'main.php';
/* Write my pid to a file */
$my_pid = getmypid();
try {
$queueHandler = new Analysis_QueueHandler();
$queueHandler->getRedisClient()->rpush(Constants_AnalysisRedisKeys::FAST_PID_LIST, $my_pid);
} catch (Exception $ex) {
$msg = "****** No REDIS/AMQ instances found. Exiting. ******";
_TimeStampMsg($msg, true);
_TimeStampMsg($ex->getMessage(), true);
die;
}
Log::$fileName = "fastAnalysis.log";
$RUNNING = true;
function sigSwitch($signo)
{
global $RUNNING;
switch ($signo) {
case SIGTERM:
case SIGINT:
$RUNNING = false;
break;
case SIGHUP:
$RUNNING = false;
cleanShutDown();
break;
default:
$msg = str_pad(" CHILD " . getmypid() . " Received Signal {$signo} ", 50, "-", STR_PAD_BOTH);
<?php
set_time_limit(0);
include "main.php";
$UNIQUID = uniqid('', true);
$my_pid = getmypid();
$parent_pid = posix_getppid();
$RUNNING = true;
Log::$fileName = "tm_analysis.log";
try {
$amqHandlerSubscriber = new Analysis_QueueHandler();
$amqHandlerSubscriber->subscribe();
$amqHandlerSubscriber->getRedisClient()->rpush(Constants_AnalysisRedisKeys::FAST_PID_LIST, $my_pid);
} catch (Exception $ex) {
$msg = "****** No REDIS/AMQ instances found. Exiting. ******";
_TimeStampMsg($msg);
_TimeStampMsg($ex->getMessage());
die;
}
// PROCESS CONTROL FUNCTIONS
function sigSwitch($signo)
{
global $RUNNING;
switch ($signo) {
case SIGTERM:
case SIGINT:
case SIGHUP:
$RUNNING = false;
break;
default:
break;
/**
* When Called it perform the controller action to retrieve/manipulate data
*
* @return mixed
*/
function doAction()
{
try {
$this->streamFilePointer = $this->tmxHandler->downloadTMX();
} catch (Exception $e) {
$r = "<pre>";
$r .= print_r("User Email: " . $this->userMail, true);
$r .= print_r("User ID: " . $this->uid, true);
$r .= print_r($e->getMessage(), true);
$r .= print_r($e->getTraceAsString(), true);
$r .= "\n\n";
$r .= " - REQUEST URI: " . print_r(@$_SERVER['REQUEST_URI'], true) . "\n";
$r .= " - REQUEST Message: " . print_r($_REQUEST, true) . "\n";
$r .= "\n\n\n";
$r .= "</pre>";
Log::$fileName = 'php_errors.txt';
Log::doLog($r);
Utils::sendErrMailReport($r, "Download TMX Error: " . $e->getMessage());
$this->unlockToken();
echo $e->getMessage();
exit;
}
}
function doReplaceAll(array $queryParams)
{
$db = Database::obtain();
$trg = $queryParams['trg'];
$replacement = $queryParams['replacement'];
$where_status = "";
if ($queryParams['status'] != 'all' && $queryParams['status'] != 'new') {
$status = $queryParams['status'];
//no escape: hardcoded
$where_status = " AND st.status = '{$status}'";
}
if ($queryParams['matchCase']) {
$SQL_CASE = "BINARY ";
$modifier = 'u';
} else {
$SQL_CASE = "";
$modifier = 'iu';
}
if ($queryParams['exactMatch']) {
$Space_Left = "[[:space:]]{0,}";
$Space_Right = "[[:space:]]";
$replacement = $replacement . " ";
//add spaces to replace " a " with "b "
} else {
$Space_Left = $Space_Right = "";
// we also want to replace all occurrences in a string: replace "mod" with "dog" in "mod modifier" -> "dog dogifier"
}
// this doesn't works because of REPLACE IS ALWAYS CASE SENSITIVE, moreover, we can't perform UNDO
// $sql = "UPDATE segment_translations, jobs
// SET translation = REPLACE( translation, '{$LIKE}{$trg}{$LIKE}', '{$replacement}' )
// WHERE id_job = jobs.id
// AND id_job = {$queryParams['job']}
// AND jobs.password = '******'password']}'
// AND id_segment BETWEEN jobs.job_first_segment AND jobs.job_last_segment
// AND segment_translations.status != 'NEW'
// AND locked != 1
// $where_status
// ";
/**
* Escape Meta-characters to use in regular expression
*
*/
$_regexpEscapedTrg = preg_replace('#([\\[\\]\\(\\)\\*\\.\\?\\^\\$\\{\\}\\+\\-\\|\\\\])#', '\\\\$1', $trg);
/**
* Escape for database
*/
$regexpEscapedTrg = $db->escape($_regexpEscapedTrg);
// Log::doLog( $regexpTrg );
$sql = "SELECT id_segment, id_job, translation\n FROM segment_translations st\n JOIN jobs ON st.id_job = id AND password = '******'password']}' AND id = {$queryParams['job']}\n WHERE id_job = {$queryParams['job']}\n AND id_segment BETWEEN jobs.job_first_segment AND jobs.job_last_segment\n AND st.status != 'NEW'\n AND locked != 1\n AND translation REGEXP {$SQL_CASE}'{$Space_Left}{$regexpEscapedTrg}{$Space_Right}'\n {$where_status}\n ";
//use this for UNDO
$resultSet = $db->fetch_array($sql);
// Log::doLog( $sql );
// Log::doLog( $resultSet );
// Log::doLog( "Replace ALL Total ResultSet " . count($resultSet) );
$sqlBatch = array();
foreach ($resultSet as $key => $tRow) {
// Log::doLog( "#({$Space_Left}){$_regexpEscapedTrg}{$Space_Right}#$modifier" );
// Log::doLog( '$1'.$replacement );
//we get the spaces before needed string and re-apply before substitution because we can't know if there are
//and how much they are
$trMod = preg_replace("#({$Space_Left}){$_regexpEscapedTrg}{$Space_Right}#{$modifier}", '$1' . $replacement, $tRow['translation']);
/**
* Escape for database
*/
$trMod = $db->escape($trMod);
$sqlBatch[] = "({$tRow['id_segment']},{$tRow['id_job']},'{$trMod}')";
}
//MySQL default max_allowed_packet is 16MB, this system surely need more
//but we can assume that max translation length is more or less 2.5KB
// so, for 100 translations of that size we can have 250KB + 20% char strings for query and id.
// 300KB is a very low number compared to 16MB
$sqlBatchChunk = array_chunk($sqlBatch, 100);
foreach ($sqlBatchChunk as $k => $batch) {
//WE USE INSERT STATEMENT for it's convenience ( update multiple fields in multiple rows in batch )
//we try to insert these rows in a table wherein the primary key ( unique by definition )
//is a coupled key ( id_segment, id_job ), but these values are already present ( duplicates )
//so make an "ON DUPLICATE KEY UPDATE"
$sqlInsert = "INSERT INTO segment_translations ( id_segment, id_job, translation )\n\t\t\tVALUES %s\n\t\t\tON DUPLICATE KEY UPDATE translation = VALUES( translation )";
$sqlInsert = sprintf($sqlInsert, implode(",", $batch));
// Log::doLog( $sqlInsert );
$db->query($sqlInsert);
if (!$db->affected_rows) {
$msg = "\n\n Error ReplaceAll \n\n Integrity failure: \n\n\n\t\t\t\t- job id : " . $queryParams['job'] . "\n\t\t\t\t- original data and failed query stored in log ReplaceAll_Failures.log\n\n\n\t\t\t\t";
Log::$fileName = 'ReplaceAll_Failures.log';
Log::doLog($sql);
Log::doLog($resultSet);
Log::doLog($sqlInsert);
Log::doLog($msg);
Utils::sendErrMailReport($msg);
throw new Exception('Update translations failure.');
//bye bye translations....
}
//we must divide by 2 because Insert count as 1 but fails and duplicate key update count as 2
// Log::doLog( "Replace ALL Batch " . ($k +1) . " - Affected Rows " . ( $db->affected_rows / 2 ) );
}
// Log::doLog( "Replace ALL Done." );
}
public function __construct(AMQHandler $queueHandler)
{
\Log::$fileName = 'tm_analysis.log';
$this->_queueHandler = $queueHandler;
}
<?php
ini_set("memory_limit", "2048M");
set_time_limit(0);
include_once 'main.php';
Log::$fileName = "archive_jobs.log";
Log::doLog("[ARCHIVEJOBS] started");
Log::doLog("[ARCHIVEJOBS] inactivity days threshold: " . INIT::JOB_ARCHIVABILITY_THRESHOLD);
$lastArchivedIdFileName = ".last_archived_id";
if (!is_file($lastArchivedIdFileName)) {
touch($lastArchivedIdFileName);
}
//select last max before INIT::JOB_ARCHIVABILITY_THRESHOLD days ago
//with status not archived so we skip them in the next cycle
$first_id = file_get_contents($lastArchivedIdFileName);
$first_id = $first_id * 1;
$last_id = getMaxJobUntilDaysAgo(INIT::JOB_ARCHIVABILITY_THRESHOLD);
Log::doLog("[ARCHIVEJOBS] last job id is: " . $last_id);
//number of rows to be selected for each query,
//after some tests,decide to set the upper bound to 100
$row_interval = 50;
for ($i = $first_id; $i <= $last_id; $i += $row_interval) {
echo "[ARCHIVEJOBS] searching jobs between {$i} and " . ($i + $row_interval) . "\n";
Log::doLog("[ARCHIVEJOBS] searching jobs between {$i} and " . ($i + $row_interval));
//check for jobs with no new segment translations and take them
$jobs = getArchivableJobs(range($i, $i + $row_interval));
if ($jobs < 0) {
Log::doLog("[ARCHIVEJOBS] skipping batch..");
continue;
}
$jobsToBeArchived = count($jobs);
