/**
* Fetch data for the project
*
*/
protected function _fetchProjectData()
{
$this->_resultSet = getProjectStatsVolumeAnalysis($this->id_project);
try {
$amqHandler = new Analysis_QueueHandler();
$segmentsBeforeMine = $amqHandler->getActualForQID($this->id_project);
} catch (Exception $e) {
$segmentsBeforeMine = null;
}
$this->_others_in_queue = $segmentsBeforeMine >= 0 ? $segmentsBeforeMine : 0;
$this->total_segments = count($this->_resultSet);
//get status of project
$this->status_project = $this->_project_data[0]['status_analysis'];
}
/**
* @param null $queueName
*
* @return bool
*/
public function subscribe($queueName = null)
{
if (empty($queueName)) {
$queueName = INIT::$DQF_PROJECTS_TASKS_QUEUE_NAME;
}
return parent::subscribe($queueName);
}
function insertFastAnalysis($pid, &$fastReport, $equivalentWordMapping, $perform_Tms_Analysis = true)
{
$db = Database::obtain();
$data = array();
$amqHandler = new Analysis_QueueHandler();
$total_eq_wc = 0;
$total_standard_wc = 0;
$data['id_segment'] = null;
$data['id_job'] = null;
$data['segment_hash'] = null;
$data['match_type'] = null;
$data['eq_word_count'] = null;
$data['standard_word_count'] = null;
$segment_translations = "INSERT INTO `segment_translations` ( " . implode(", ", array_keys($data)) . " ) VALUES ";
$st_values = array();
foreach ($fastReport as $k => $v) {
$jid_fid = explode("-", $k);
$id_segment = $jid_fid[0];
$list_id_jobs_password = $jid_fid[1];
if (array_key_exists($v['match_type'], $equivalentWordMapping)) {
$eq_word = $v['wc'] * $equivalentWordMapping[$v['match_type']] / 100;
if ($v['match_type'] == "INTERNAL") {
}
} else {
$eq_word = $v['wc'];
}
$total_eq_wc += $eq_word;
$standard_words = $eq_word;
if ($v['match_type'] == "INTERNAL" or $v['match_type'] == "MT") {
$standard_words = $v['wc'] * $equivalentWordMapping["NO_MATCH"] / 100;
}
$total_standard_wc += $standard_words;
unset($fastReport[$k]['wc']);
$list_id_jobs_password = explode(',', $list_id_jobs_password);
foreach ($list_id_jobs_password as $id_job) {
list($id_job, $job_pass) = explode(":", $id_job);
$data['id_job'] = (int) $id_job;
$data['id_segment'] = (int) $fastReport[$k]['id_segment'];
$data['segment_hash'] = $db->escape($v['segment_hash']);
$data['match_type'] = $db->escape($v['match_type']);
if (!empty($v['segment_hash']) && empty($data['segment_hash'])) {
$data['segment_hash'] = $v['segment_hash'];
$msg = "mysql_real_escape_string failed!!! String was empty. Replaced with original {$v['segment_hash']}";
_TimeStampMsg($msg);
Utils::sendErrMailReport("<strong>{$msg}</strong>", "Fast Analysis mysql_real_escape_string failed.");
}
$data['eq_word_count'] = (double) $eq_word;
$data['standard_word_count'] = (double) $standard_words;
$st_values[] = " ( '" . implode("', '", array_values($data)) . "' )";
if ($data['eq_word_count'] > 0 && $perform_Tms_Analysis) {
/**
*
* IMPORTANT
* id_job will be taken from languages ( 80415:fr-FR,80416:it-IT )
*/
$fastReport[$k]['pid'] = (int) $pid;
$fastReport[$k]['date_insert'] = date_create()->format('Y-m-d H:i:s');
$fastReport[$k]['eq_word_count'] = (double) $eq_word;
$fastReport[$k]['standard_word_count'] = (double) $standard_words;
} else {
// Log::doLog( 'Skipped Fast Segment: ' . var_export( $fastReport[ $k ], true ) );
unset($fastReport[$k]);
}
}
}
unset($data);
$chunks_st = array_chunk($st_values, 200);
_TimeStampMsg('Insert Segment Translations: ' . count($st_values));
_TimeStampMsg('Queries: ' . count($chunks_st));
//USE the MySQL InnoDB isolation Level to protect from thread high concurrency access
$db->query('SET autocommit=0');
$db->query('START TRANSACTION');
foreach ($chunks_st as $k => $chunk) {
$query_st = $segment_translations . implode(", ", $chunk) . " ON DUPLICATE KEY UPDATE\n match_type = VALUES( match_type ),\n eq_word_count = VALUES( eq_word_count ),\n standard_word_count = VALUES( standard_word_count )\n ";
$db->query($query_st);
_TimeStampMsg("Executed " . ($k + 1));
$err = $db->get_error();
if ($err['error_code'] != 0) {
_TimeStampMsg($err);
return $err['error_code'] * -1;
}
}
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($st_values);
unset($chunks_st);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
/*
* IF NO TM ANALYSIS, upload the jobs global word count
*/
if (!$perform_Tms_Analysis) {
$_details = getProjectSegmentsTranslationSummary($pid);
_TimeStampMsg("--- trying to initialize job total word count.");
$project_details = array_pop($_details);
//remove rollup
foreach ($_details as $job_info) {
$counter = new WordCount_Counter();
$counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
}
}
/* IF NO TM ANALYSIS, upload the jobs global word count */
//_TimeStampMsg( "Done." );
$data2 = array('fast_analysis_wc' => $total_eq_wc);
$where = " id = {$pid}";
$db->update('projects', $data2, $where);
$err = $db->get_error();
$errno = $err['error_code'];
if ($errno != 0) {
$db->query('ROLLBACK');
$db->query('SET autocommit=1');
_TimeStampMsg($err);
return $errno * -1;
}
$db->query('COMMIT');
$db->query('SET autocommit=1');
if (count($fastReport)) {
// $chunks_st_queue = array_chunk( $fastReport, 10 );
_TimeStampMsg('Insert Segment Translations Queue: ' . count($fastReport));
_TimeStampMsg('Queries: ' . count($fastReport));
try {
$amqHandler->setTotal(array('qid' => $pid, 'queueName' => INIT::$QUEUE_NAME));
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set Total values failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
$time_start = microtime(true);
foreach ($fastReport as $k => $queue_element) {
try {
$languages_job = explode(",", $queue_element['target']);
//now target holds more than one language ex: ( 80415:fr-FR,80416:it-IT )
//in memory replacement avoid duplication of the segment list
//send in queue every element * number of languages
foreach ($languages_job as $_language) {
list($id_job, $language) = explode(":", $_language);
$queue_element['target'] = $language;
$queue_element['id_job'] = $id_job;
$jsonObj = json_encode($queue_element);
Utils::raiseJsonExceptionError();
$amqHandler->send(INIT::$QUEUE_NAME, $jsonObj, array('persistent' => $amqHandler->persistent));
_TimeStampMsg("AMQ Set Executed " . ($k + 1));
}
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set queue failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
}
_TimeStampMsg('Done in ' . (microtime(true) - $time_start) . " seconds.");
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($fastReport);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
}
$amqHandler->disconnect();
return $db->affected_rows;
}
$config = $tms->getConfigStruct();
$config = array_merge($config, $_config);
$tms_match = $tms->get($config);
//MyMemory can return null if an error occurs (e.g http response code is 404, 410, 500, 503, etc.. )
if ($tms_match !== null) {
$tms_match = $tms_match->get_matches_as_array();
} else {
_TimeStampMsg("--- (child {$my_pid}) : error from mymemory : set error and continue");
// ERROR FROM MYMEMORY
setSegmentTranslationError($sid, $jid);
// devo settarli come done e lasciare il vecchio livello di match
$amqHandlerSubscriber->tryToCloseProject($pid, $my_pid);
$amqHandlerSubscriber->ack($msg);
//set/increment the reQueue number
$objQueue['reQueueNum'] = @++$objQueue['reQueueNum'];
$amqHandlerPublisher = new Analysis_QueueHandler();
$amqHandlerPublisher->reQueue($objQueue);
$amqHandlerPublisher->disconnect();
continue;
}
}
/**
* Call External MT engine if it is a custom one ( mt not requested from MyMemory )
*/
if ($id_mt_engine > 1) {
$mt = Engine::getInstance($id_mt_engine);
$config = $mt->getConfigStruct();
$config = array_merge($config, $_config);
$mt_result = $mt->get($config);
if (isset($mt_result['error']['code'])) {
$mt_result = false;
<?php
/**
* Created by PhpStorm.
* @author domenico domenico@translated.net / ostico@gmail.com
* Date: 24/06/15
* Time: 14.26
*
*/
$root = realpath(dirname(__FILE__) . '/../../../../../');
require_once "{$root}/inc/Bootstrap.php";
Bootstrap::start();
//---------------------------
define('DEFAULT_NUM_WORKERS', include INIT::$UTILS_ROOT . '/Analysis/DefaultNumTMWorkers.php');
define('NUM_WORKERS', INIT::$UTILS_ROOT . "/Analysis/.num_processes");
$queueHandler = new Analysis_QueueHandler();
$consumers = $queueHandler->getConsumerCount();
$desiredCount = DEFAULT_NUM_WORKERS;
if (file_exists(NUM_WORKERS)) {
$desiredCount = intval(file_get_contents(NUM_WORKERS));
}
if ($desiredCount > $consumers) {
exit(127);
} elseif ($consumers > $desiredCount) {
exit(-127);
} elseif (empty($desiredCount)) {
exit(3);
}
exit(0);
<?php
set_time_limit(0);
require "main.php";
define('ANALYSIS_ROOT', INIT::$UTILS_ROOT . "/Analysis/.num_processes");
$my_pid = getmypid();
try {
$queueHandler = new Analysis_QueueHandler();
$queueHandler->getRedisClient()->set(Constants_AnalysisRedisKeys::VOLUME_ANALYSIS_PID, getmypid());
} catch (Exception $ex) {
$msg = "****** No REDIS/AMQ instances found. Exiting. ******";
_TimeStampMsg($msg, true);
_TimeStampMsg($ex->getMessage(), true);
die;
}
Log::$fileName = "tm_analysis.log";
$RUNNING = true;
// PROCESS CONTROL FUNCTIONS
function cleanShutDown()
{
global $queueHandler, $db;
//SHUTDOWN
deletePid();
$queueHandler->getRedisClient()->del(Constants_AnalysisRedisKeys::VOLUME_ANALYSIS_PID);
$queueHandler->getRedisClient()->disconnect();
$db->close();
$msg = str_pad(" TM ANALYSIS " . getmypid() . " HALTED ", 50, "-", STR_PAD_BOTH);
_TimeStampMsg($msg, true);
}
function sigSwitch($signo)
{