function insertFastAnalysis($pid, &$fastReport, $equivalentWordMapping, $perform_Tms_Analysis = true)
{
$db = Database::obtain();
$data = array();
$amqHandler = new Analysis_QueueHandler();
$total_eq_wc = 0;
$total_standard_wc = 0;
$data['id_segment'] = null;
$data['id_job'] = null;
$data['segment_hash'] = null;
$data['match_type'] = null;
$data['eq_word_count'] = null;
$data['standard_word_count'] = null;
$segment_translations = "INSERT INTO `segment_translations` ( " . implode(", ", array_keys($data)) . " ) VALUES ";
$st_values = array();
foreach ($fastReport as $k => $v) {
$jid_fid = explode("-", $k);
$id_segment = $jid_fid[0];
$list_id_jobs_password = $jid_fid[1];
if (array_key_exists($v['match_type'], $equivalentWordMapping)) {
$eq_word = $v['wc'] * $equivalentWordMapping[$v['match_type']] / 100;
if ($v['match_type'] == "INTERNAL") {
}
} else {
$eq_word = $v['wc'];
}
$total_eq_wc += $eq_word;
$standard_words = $eq_word;
if ($v['match_type'] == "INTERNAL" or $v['match_type'] == "MT") {
$standard_words = $v['wc'] * $equivalentWordMapping["NO_MATCH"] / 100;
}
$total_standard_wc += $standard_words;
unset($fastReport[$k]['wc']);
$list_id_jobs_password = explode(',', $list_id_jobs_password);
foreach ($list_id_jobs_password as $id_job) {
list($id_job, $job_pass) = explode(":", $id_job);
$data['id_job'] = (int) $id_job;
$data['id_segment'] = (int) $fastReport[$k]['id_segment'];
$data['segment_hash'] = $db->escape($v['segment_hash']);
$data['match_type'] = $db->escape($v['match_type']);
if (!empty($v['segment_hash']) && empty($data['segment_hash'])) {
$data['segment_hash'] = $v['segment_hash'];
$msg = "mysql_real_escape_string failed!!! String was empty. Replaced with original {$v['segment_hash']}";
_TimeStampMsg($msg);
Utils::sendErrMailReport("<strong>{$msg}</strong>", "Fast Analysis mysql_real_escape_string failed.");
}
$data['eq_word_count'] = (double) $eq_word;
$data['standard_word_count'] = (double) $standard_words;
$st_values[] = " ( '" . implode("', '", array_values($data)) . "' )";
if ($data['eq_word_count'] > 0 && $perform_Tms_Analysis) {
/**
*
* IMPORTANT
* id_job will be taken from languages ( 80415:fr-FR,80416:it-IT )
*/
$fastReport[$k]['pid'] = (int) $pid;
$fastReport[$k]['date_insert'] = date_create()->format('Y-m-d H:i:s');
$fastReport[$k]['eq_word_count'] = (double) $eq_word;
$fastReport[$k]['standard_word_count'] = (double) $standard_words;
} else {
// Log::doLog( 'Skipped Fast Segment: ' . var_export( $fastReport[ $k ], true ) );
unset($fastReport[$k]);
}
}
}
unset($data);
$chunks_st = array_chunk($st_values, 200);
_TimeStampMsg('Insert Segment Translations: ' . count($st_values));
_TimeStampMsg('Queries: ' . count($chunks_st));
//USE the MySQL InnoDB isolation Level to protect from thread high concurrency access
$db->query('SET autocommit=0');
$db->query('START TRANSACTION');
foreach ($chunks_st as $k => $chunk) {
$query_st = $segment_translations . implode(", ", $chunk) . " ON DUPLICATE KEY UPDATE\n match_type = VALUES( match_type ),\n eq_word_count = VALUES( eq_word_count ),\n standard_word_count = VALUES( standard_word_count )\n ";
$db->query($query_st);
_TimeStampMsg("Executed " . ($k + 1));
$err = $db->get_error();
if ($err['error_code'] != 0) {
_TimeStampMsg($err);
return $err['error_code'] * -1;
}
}
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($st_values);
unset($chunks_st);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
/*
* IF NO TM ANALYSIS, upload the jobs global word count
*/
if (!$perform_Tms_Analysis) {
$_details = getProjectSegmentsTranslationSummary($pid);
_TimeStampMsg("--- trying to initialize job total word count.");
$project_details = array_pop($_details);
//remove rollup
foreach ($_details as $job_info) {
$counter = new WordCount_Counter();
$counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
}
}
/* IF NO TM ANALYSIS, upload the jobs global word count */
//_TimeStampMsg( "Done." );
$data2 = array('fast_analysis_wc' => $total_eq_wc);
$where = " id = {$pid}";
$db->update('projects', $data2, $where);
$err = $db->get_error();
$errno = $err['error_code'];
if ($errno != 0) {
$db->query('ROLLBACK');
$db->query('SET autocommit=1');
_TimeStampMsg($err);
return $errno * -1;
}
$db->query('COMMIT');
$db->query('SET autocommit=1');
if (count($fastReport)) {
// $chunks_st_queue = array_chunk( $fastReport, 10 );
_TimeStampMsg('Insert Segment Translations Queue: ' . count($fastReport));
_TimeStampMsg('Queries: ' . count($fastReport));
try {
$amqHandler->setTotal(array('qid' => $pid, 'queueName' => INIT::$QUEUE_NAME));
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set Total values failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
$time_start = microtime(true);
foreach ($fastReport as $k => $queue_element) {
try {
$languages_job = explode(",", $queue_element['target']);
//now target holds more than one language ex: ( 80415:fr-FR,80416:it-IT )
//in memory replacement avoid duplication of the segment list
//send in queue every element * number of languages
foreach ($languages_job as $_language) {
list($id_job, $language) = explode(":", $_language);
$queue_element['target'] = $language;
$queue_element['id_job'] = $id_job;
$jsonObj = json_encode($queue_element);
Utils::raiseJsonExceptionError();
$amqHandler->send(INIT::$QUEUE_NAME, $jsonObj, array('persistent' => $amqHandler->persistent));
_TimeStampMsg("AMQ Set Executed " . ($k + 1));
}
} catch (Exception $e) {
Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set queue failed.");
_TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
throw $e;
}
}
_TimeStampMsg('Done in ' . (microtime(true) - $time_start) . " seconds.");
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
unset($fastReport);
_TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
}
$amqHandler->disconnect();
return $db->affected_rows;
}
<?php
$root = realpath(dirname(__FILE__) . '/../../../');
include_once "{$root}/inc/config.inc.php";
INIT::obtain();
require_once INIT::$MODEL_ROOT . '/queries.php';
$db = Database::obtain(INIT::$DB_SERVER, INIT::$DB_USER, INIT::$DB_PASS, INIT::$DB_DATABASE);
$db->debug = false;
$db->connect();
declare (ticks=1);
if (!function_exists('pcntl_signal')) {
$msg = "****** PCNTL EXTENSION NOT LOADED. KILLING THIS PROCESS COULD CAUSE UNPREDICTABLE ERRORS ******";
_TimeStampMsg($msg);
} else {
pcntl_signal(SIGTERM, 'sigSwitch');
pcntl_signal(SIGINT, 'sigSwitch');
pcntl_signal(SIGHUP, 'sigSwitch');
$msg = str_pad(" " . getmypid() . " Signal Handler Installed ", 50, "-", STR_PAD_BOTH);
_TimeStampMsg($msg);
}
function _TimeStampMsg($msg, $log = true)
{
if ($log) {
Log::doLog($msg);
}
echo "[" . date(DATE_RFC822) . "] " . $msg . "\n";
}
function updateTMValues($tm_data)
{
if (!empty($tm_data['suggestion_source'])) {
if (strpos($tm_data['suggestion_source'], "MT") === false) {
$tm_data['suggestion_source'] = 'TM';
} else {
$tm_data['suggestion_source'] = 'MT';
}
}
//controllare il valore di suggestion_match
if ($tm_data['suggestion_match'] == "100%" && $tm_data['pretranslate_100']) {
$tm_data['status'] = Constants_TranslationStatus::STATUS_TRANSLATED;
}
//there is not a database filed named pretranslate_100 in segment_translations, this is only a flag
unset($tm_data['pretranslate_100']);
$updateRes = setSuggestionUpdate($tm_data);
if ($updateRes < 0) {
$result['errors'][] = array("code" => -5, "message" => "error occurred during the storing (UPDATE) of the suggestions for the segment {$tm_data['id_segment']}");
_TimeStampMsg($result);
} elseif ($updateRes == 0) {
//There was not a fast Analysis??? Impossible.
_TimeStampMsg("No row found: " . $tm_data['id_segment'] . "-" . $tm_data['id_job']);
} else {
_TimeStampMsg("Row found: " . $tm_data['id_segment'] . "-" . $tm_data['id_job'] . " - UPDATED.");
}
}
function setNumProcesses()
{
// legge quanti processi lanciare
$num_processes = null;
if (file_exists(ANALYSIS_ROOT)) {
$num_processes = intval(file_get_contents(ANALYSIS_ROOT));
}
if (!is_int($num_processes)) {
_TimeStampMsg("WARNING : num processes from file is not numeric. Back to default value NUM_PROCESSES = 1", false);
$num_processes = 1;
}
return $num_processes;
}
