Exemplo n.º 1
0
 /**
  * Fetch data for the project
  *
  */
 protected function _fetchProjectData()
 {
     $this->_resultSet = getProjectStatsVolumeAnalysis($this->id_project);
     try {
         $amqHandler = new Analysis_QueueHandler();
         $segmentsBeforeMine = $amqHandler->getActualForQID($this->id_project);
     } catch (Exception $e) {
         $segmentsBeforeMine = null;
     }
     $this->_others_in_queue = $segmentsBeforeMine >= 0 ? $segmentsBeforeMine : 0;
     $this->total_segments = count($this->_resultSet);
     //get status of project
     $this->status_project = $this->_project_data[0]['status_analysis'];
 }
Exemplo n.º 2
0
 /**
  * @param null $queueName
  *
  * @return bool
  */
 public function subscribe($queueName = null)
 {
     if (empty($queueName)) {
         $queueName = INIT::$DQF_PROJECTS_TASKS_QUEUE_NAME;
     }
     return parent::subscribe($queueName);
 }
Exemplo n.º 3
0
function insertFastAnalysis($pid, &$fastReport, $equivalentWordMapping, $perform_Tms_Analysis = true)
{
    $db = Database::obtain();
    $data = array();
    $amqHandler = new Analysis_QueueHandler();
    $total_eq_wc = 0;
    $total_standard_wc = 0;
    $data['id_segment'] = null;
    $data['id_job'] = null;
    $data['segment_hash'] = null;
    $data['match_type'] = null;
    $data['eq_word_count'] = null;
    $data['standard_word_count'] = null;
    $segment_translations = "INSERT INTO `segment_translations` ( " . implode(", ", array_keys($data)) . " ) VALUES ";
    $st_values = array();
    foreach ($fastReport as $k => $v) {
        $jid_fid = explode("-", $k);
        $id_segment = $jid_fid[0];
        $list_id_jobs_password = $jid_fid[1];
        if (array_key_exists($v['match_type'], $equivalentWordMapping)) {
            $eq_word = $v['wc'] * $equivalentWordMapping[$v['match_type']] / 100;
            if ($v['match_type'] == "INTERNAL") {
            }
        } else {
            $eq_word = $v['wc'];
        }
        $total_eq_wc += $eq_word;
        $standard_words = $eq_word;
        if ($v['match_type'] == "INTERNAL" or $v['match_type'] == "MT") {
            $standard_words = $v['wc'] * $equivalentWordMapping["NO_MATCH"] / 100;
        }
        $total_standard_wc += $standard_words;
        unset($fastReport[$k]['wc']);
        $list_id_jobs_password = explode(',', $list_id_jobs_password);
        foreach ($list_id_jobs_password as $id_job) {
            list($id_job, $job_pass) = explode(":", $id_job);
            $data['id_job'] = (int) $id_job;
            $data['id_segment'] = (int) $fastReport[$k]['id_segment'];
            $data['segment_hash'] = $db->escape($v['segment_hash']);
            $data['match_type'] = $db->escape($v['match_type']);
            if (!empty($v['segment_hash']) && empty($data['segment_hash'])) {
                $data['segment_hash'] = $v['segment_hash'];
                $msg = "mysql_real_escape_string failed!!! String was empty. Replaced with original {$v['segment_hash']}";
                _TimeStampMsg($msg);
                Utils::sendErrMailReport("<strong>{$msg}</strong>", "Fast Analysis mysql_real_escape_string failed.");
            }
            $data['eq_word_count'] = (double) $eq_word;
            $data['standard_word_count'] = (double) $standard_words;
            $st_values[] = " ( '" . implode("', '", array_values($data)) . "' )";
            if ($data['eq_word_count'] > 0 && $perform_Tms_Analysis) {
                /**
                 *
                 * IMPORTANT
                 * id_job will be taken from languages ( 80415:fr-FR,80416:it-IT )
                 */
                $fastReport[$k]['pid'] = (int) $pid;
                $fastReport[$k]['date_insert'] = date_create()->format('Y-m-d H:i:s');
                $fastReport[$k]['eq_word_count'] = (double) $eq_word;
                $fastReport[$k]['standard_word_count'] = (double) $standard_words;
            } else {
                //                Log::doLog( 'Skipped Fast Segment: ' . var_export( $fastReport[ $k ], true ) );
                unset($fastReport[$k]);
            }
        }
    }
    unset($data);
    $chunks_st = array_chunk($st_values, 200);
    _TimeStampMsg('Insert Segment Translations: ' . count($st_values));
    _TimeStampMsg('Queries: ' . count($chunks_st));
    //USE the MySQL InnoDB isolation Level to protect from thread high concurrency access
    $db->query('SET autocommit=0');
    $db->query('START TRANSACTION');
    foreach ($chunks_st as $k => $chunk) {
        $query_st = $segment_translations . implode(", ", $chunk) . " ON DUPLICATE KEY UPDATE\n            match_type = VALUES( match_type ),\n                       eq_word_count = VALUES( eq_word_count ),\n                       standard_word_count = VALUES( standard_word_count )\n                           ";
        $db->query($query_st);
        _TimeStampMsg("Executed " . ($k + 1));
        $err = $db->get_error();
        if ($err['error_code'] != 0) {
            _TimeStampMsg($err);
            return $err['error_code'] * -1;
        }
    }
    _TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
    unset($st_values);
    unset($chunks_st);
    _TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
    /*
     * IF NO TM ANALYSIS, upload the jobs global word count
     */
    if (!$perform_Tms_Analysis) {
        $_details = getProjectSegmentsTranslationSummary($pid);
        _TimeStampMsg("--- trying to initialize job total word count.");
        $project_details = array_pop($_details);
        //remove rollup
        foreach ($_details as $job_info) {
            $counter = new WordCount_Counter();
            $counter->initializeJobWordCount($job_info['id_job'], $job_info['password']);
        }
    }
    /* IF NO TM ANALYSIS, upload the jobs global word count */
    //_TimeStampMsg( "Done." );
    $data2 = array('fast_analysis_wc' => $total_eq_wc);
    $where = " id = {$pid}";
    $db->update('projects', $data2, $where);
    $err = $db->get_error();
    $errno = $err['error_code'];
    if ($errno != 0) {
        $db->query('ROLLBACK');
        $db->query('SET autocommit=1');
        _TimeStampMsg($err);
        return $errno * -1;
    }
    $db->query('COMMIT');
    $db->query('SET autocommit=1');
    if (count($fastReport)) {
        //        $chunks_st_queue = array_chunk( $fastReport, 10 );
        _TimeStampMsg('Insert Segment Translations Queue: ' . count($fastReport));
        _TimeStampMsg('Queries: ' . count($fastReport));
        try {
            $amqHandler->setTotal(array('qid' => $pid, 'queueName' => INIT::$QUEUE_NAME));
        } catch (Exception $e) {
            Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set Total values failed.");
            _TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
            throw $e;
        }
        $time_start = microtime(true);
        foreach ($fastReport as $k => $queue_element) {
            try {
                $languages_job = explode(",", $queue_element['target']);
                //now target holds more than one language ex: ( 80415:fr-FR,80416:it-IT )
                //in memory replacement avoid duplication of the segment list
                //send in queue every element * number of languages
                foreach ($languages_job as $_language) {
                    list($id_job, $language) = explode(":", $_language);
                    $queue_element['target'] = $language;
                    $queue_element['id_job'] = $id_job;
                    $jsonObj = json_encode($queue_element);
                    Utils::raiseJsonExceptionError();
                    $amqHandler->send(INIT::$QUEUE_NAME, $jsonObj, array('persistent' => $amqHandler->persistent));
                    _TimeStampMsg("AMQ Set Executed " . ($k + 1));
                }
            } catch (Exception $e) {
                Utils::sendErrMailReport($e->getMessage() . "" . $e->getTraceAsString(), "Fast Analysis set queue failed.");
                _TimeStampMsg($e->getMessage() . "" . $e->getTraceAsString());
                throw $e;
            }
        }
        _TimeStampMsg('Done in ' . (microtime(true) - $time_start) . " seconds.");
        _TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
        unset($fastReport);
        _TimeStampMsg("Memory: " . memory_get_usage(true) / (1024 * 1024) . "MB");
    }
    $amqHandler->disconnect();
    return $db->affected_rows;
}
Exemplo n.º 4
0
     $config = $tms->getConfigStruct();
     $config = array_merge($config, $_config);
     $tms_match = $tms->get($config);
     //MyMemory can return null if an error occurs (e.g http response code is 404, 410, 500, 503, etc.. )
     if ($tms_match !== null) {
         $tms_match = $tms_match->get_matches_as_array();
     } else {
         _TimeStampMsg("--- (child {$my_pid}) : error from mymemory : set error and continue");
         // ERROR FROM MYMEMORY
         setSegmentTranslationError($sid, $jid);
         // devo settarli come done e lasciare il vecchio livello di match
         $amqHandlerSubscriber->tryToCloseProject($pid, $my_pid);
         $amqHandlerSubscriber->ack($msg);
         //set/increment the reQueue number
         $objQueue['reQueueNum'] = @++$objQueue['reQueueNum'];
         $amqHandlerPublisher = new Analysis_QueueHandler();
         $amqHandlerPublisher->reQueue($objQueue);
         $amqHandlerPublisher->disconnect();
         continue;
     }
 }
 /**
  * Call External MT engine if it is a custom one ( mt not requested from MyMemory )
  */
 if ($id_mt_engine > 1) {
     $mt = Engine::getInstance($id_mt_engine);
     $config = $mt->getConfigStruct();
     $config = array_merge($config, $_config);
     $mt_result = $mt->get($config);
     if (isset($mt_result['error']['code'])) {
         $mt_result = false;
Exemplo n.º 5
0
<?php

/**
 * Created by PhpStorm.
 * @author domenico domenico@translated.net / ostico@gmail.com
 * Date: 24/06/15
 * Time: 14.26
 * 
 */
$root = realpath(dirname(__FILE__) . '/../../../../../');
require_once "{$root}/inc/Bootstrap.php";
Bootstrap::start();
//---------------------------
define('DEFAULT_NUM_WORKERS', include INIT::$UTILS_ROOT . '/Analysis/DefaultNumTMWorkers.php');
define('NUM_WORKERS', INIT::$UTILS_ROOT . "/Analysis/.num_processes");
$queueHandler = new Analysis_QueueHandler();
$consumers = $queueHandler->getConsumerCount();
$desiredCount = DEFAULT_NUM_WORKERS;
if (file_exists(NUM_WORKERS)) {
    $desiredCount = intval(file_get_contents(NUM_WORKERS));
}
if ($desiredCount > $consumers) {
    exit(127);
} elseif ($consumers > $desiredCount) {
    exit(-127);
} elseif (empty($desiredCount)) {
    exit(3);
}
exit(0);
Exemplo n.º 6
0
<?php

set_time_limit(0);
require "main.php";
define('ANALYSIS_ROOT', INIT::$UTILS_ROOT . "/Analysis/.num_processes");
$my_pid = getmypid();
try {
    $queueHandler = new Analysis_QueueHandler();
    $queueHandler->getRedisClient()->set(Constants_AnalysisRedisKeys::VOLUME_ANALYSIS_PID, getmypid());
} catch (Exception $ex) {
    $msg = "****** No REDIS/AMQ instances found. Exiting. ******";
    _TimeStampMsg($msg, true);
    _TimeStampMsg($ex->getMessage(), true);
    die;
}
Log::$fileName = "tm_analysis.log";
$RUNNING = true;
// PROCESS CONTROL FUNCTIONS
function cleanShutDown()
{
    global $queueHandler, $db;
    //SHUTDOWN
    deletePid();
    $queueHandler->getRedisClient()->del(Constants_AnalysisRedisKeys::VOLUME_ANALYSIS_PID);
    $queueHandler->getRedisClient()->disconnect();
    $db->close();
    $msg = str_pad(" TM ANALYSIS " . getmypid() . " HALTED ", 50, "-", STR_PAD_BOTH);
    _TimeStampMsg($msg, true);
}
function sigSwitch($signo)
{