public function testMultipleCurlCreate()
{
$options = array(CURLOPT_HEADER => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_HEADER => 0, CURLOPT_USERAGENT => INIT::MATECAT_USER_AGENT . INIT::$BUILD_NUMBER, CURLOPT_CONNECTTIMEOUT => 2);
$mh = new MultiCurlHandler();
$hashes = array();
$tokenHash1 = $mh->createResource('http://www.google.com/', $options);
$this->assertNotEmpty($tokenHash1);
$hashes[] = $tokenHash1;
$tokenHash2 = $mh->createResource('http://www.yahoo.com/', $options);
$this->assertNotEmpty($tokenHash2);
$hashes[] = $tokenHash2;
$tokenHash3 = $mh->createResource('http://www.bing.com/', $options);
$this->assertNotEmpty($tokenHash3);
$hashes[] = $tokenHash3;
$tokenHash4 = $mh->createResource('http://www.translated.net/', $options);
$this->assertNotEmpty($tokenHash4);
$hashes[] = $tokenHash4;
$mh->multiExec();
$multiContent = $mh->getAllContents();
$this->assertNotEmpty($multiContent);
$singleContent = $mh->getSingleContent($tokenHash2);
$this->assertNotEmpty($singleContent);
foreach ($multiContent as $hash => $result) {
$this->assertEquals($mh->getSingleContent($hash), $result);
}
}
/**
* @param $PM_KEY
*
* @return mixed
* @throws Exception
*/
public function checkProjectManagerKey($PM_KEY)
{
$options = array(CURLOPT_RETURNTRANSFER => true, CURLOPT_USERAGENT => INIT::MATECAT_USER_AGENT . INIT::$BUILD_NUMBER, CURLOPT_CONNECTTIMEOUT => 5, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, CURLOPT_HTTPHEADER => array("DQF_PMANAGER_KEY: {$PM_KEY}"));
$cHandler = new MultiCurlHandler();
$hash = $cHandler->createResource("https://dqf.taus.net/api/v1/projectmanager/", $options);
$cHandler->setRequestHeader($hash)->multiExec();
$result = json_decode($cHandler->getSingleContent($hash));
if (isset($result->code) && $result->code != 200) {
throw new Exception($result->message);
}
return $result;
}
/**
* Detect language for an array of file's segments.
*
* @param $segs_array Array An array whose keys are file IDs and values are array of segments.
*
* @return mixed
*/
public function detectLanguage($segs_array, $lang_detect_files)
{
//In this array we will put a significative string for each job.
$segmentsToBeDetected = array();
/**
* @var $segs_array ArrayObject
*/
$arrayIterator = $segs_array->getIterator();
$counter = 0;
//iterate through files and extract a significative
//string long at least 150 characters for language detection
while ($arrayIterator->valid()) {
$currFileName = key($lang_detect_files);
if ($lang_detect_files[$currFileName] == "skip") {
//this will force google to answer with "und" language code
$segmentsToBeDetected[] = "q[{$counter}]=1";
next($lang_detect_files);
$arrayIterator->next();
$counter++;
continue;
}
$currFileId = $arrayIterator->key();
$currFile = $arrayIterator->current();
/**
* @var $currFileIterator ArrayIterator
*/
$segmentArray = $currFile->getIterator()->current();
//take first 50 segments
$segmentArray = array_slice($segmentArray, 0, 50);
foreach ($segmentArray as $i => $singleSegment) {
$singleSegment = explode(",", $singleSegment);
$singleSegment = array_slice($singleSegment, 3, 1);
//remove tags, duplicated spaces and all not Unicode Letter
$singleSegment[0] = preg_replace(array("#<[^<>]*>#", "# {2,}#", '#\\PL+#u'), array("", " ", " "), $singleSegment[0]);
//remove not useful spaces
$singleSegment[0] = preg_replace("# {2,}#", " ", $singleSegment[0]);
$segmentArray[$i] = $singleSegment[0];
}
if (!function_exists('sortByStrLenAsc')) {
function sortByStrLenAsc($a, $b)
{
return strlen($a) >= strlen($b);
}
}
usort($segmentArray, array('sortByStrLenAsc'));
$textToBeDetected = "";
/**
* take first 150 characters starting from the longest segment in the slice
*/
for ($i = count($segmentArray) - 1; $i >= 0; $i--) {
$textToBeDetected .= " " . trim($segmentArray[$i], "'");
if (mb_strlen($textToBeDetected) > 150) {
break;
}
}
$segmentsToBeDetected[] = "q[{$counter}]=" . urlencode($textToBeDetected);
next($lang_detect_files);
$arrayIterator->next();
$counter++;
}
$curl_parameters = implode("&", $segmentsToBeDetected) . "&of=json";
log::dolog("DETECT LANG :", $segmentsToBeDetected);
$options = array(CURLOPT_HEADER => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_HEADER => 0, CURLOPT_USERAGENT => INIT::MATECAT_USER_AGENT . INIT::$BUILD_NUMBER, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_POST => true, CURLOPT_POSTFIELDS => $curl_parameters, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2);
$url = strtolower($this->base_url . "/" . $this->detect_language_url);
$mh = new MultiCurlHandler();
$tokenHash = $mh->createResource($url, $options);
Log::dolog("DETECT LANG TOKENHASH: {$tokenHash}");
$mh->multiExec();
$res = $mh->getAllContents();
Log::dolog("DETECT LANG RES:", $res);
return json_decode($res[$tokenHash], true);
}
/**
* Perform a quote on the remote Provider server
*
* @see OutsourceTo_AbstractProvider::performQuote
*
* @param array|null $volAnalysis
*/
public function performQuote($volAnalysis = null)
{
/**
* cache this job info for 20 minutes ( session duration )
*/
$cache_cart = Shop_Cart::getInstance('outsource_to_external_cache');
if ($volAnalysis == null) {
//call matecat API for Project status and information
$project_url_api = INIT::$HTTPHOST . INIT::$BASEURL . "api/status?id_project=" . $this->pid . "&project_pass="******"Project Not Found in Cache. Call API url for STATUS: " . $project_url_api);
$options = array(CURLOPT_HEADER => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_HEADER => 0, CURLOPT_USERAGENT => INIT::MATECAT_USER_AGENT . INIT::$BUILD_NUMBER, CURLOPT_CONNECTTIMEOUT => 5, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2);
//prepare handlers for curl to quote service
$mh = new MultiCurlHandler();
$resourceHash = $mh->createResource($project_url_api, $options);
$mh->multiExec();
if ($mh->hasError($resourceHash)) {
Log::doLog($mh->getError($resourceHash));
}
$raw_volAnalysis = $mh->getSingleContent($resourceHash);
$mh->multiCurlCloseAll();
//retrieve the project subject: pick the project's first job and get the subject
$jobData = getJobData($this->jobList[0]['jid'], $this->jobList[0]['jpassword']);
$subject = $jobData['subject'];
$itemCart = new Shop_ItemHTSQuoteJob();
$itemCart['id'] = $project_url_api;
$itemCart['show_info'] = $raw_volAnalysis;
$itemCart['subject'] = $subject;
$cache_cart->addItem($itemCart);
} else {
$tmp_project_cache = $cache_cart->getItem($project_url_api);
$raw_volAnalysis = $tmp_project_cache['show_info'];
$subject = $tmp_project_cache['subject'];
}
// Log::doLog( $raw_volAnalysis );
$volAnalysis = json_decode($raw_volAnalysis, true);
}
// Log::doLog( $volAnalysis );
$_jobLangs = array();
$options = array(CURLOPT_HEADER => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_HEADER => 0, CURLOPT_USERAGENT => INIT::MATECAT_USER_AGENT . INIT::$BUILD_NUMBER, CURLOPT_CONNECTTIMEOUT => 10, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2);
//prepare handlers for curl to quote service
$mh = new MultiCurlHandler();
foreach ($this->jobList as $job) {
//trim decimals to int
$job_payableWords = (int) $volAnalysis['data']['jobs'][$job['jid']]['totals'][$job['jpassword']]['TOTAL_PAYABLE'][0];
/*
* //languages are in the form:
*
* "langpairs":{
* "5888-e94bd2f79afd":"en-GB|fr-FR",
* "5889-c853a841dafd":"en-GB|de-DE",
* "5890-e852ca45c66e":"en-GB|it-IT",
* "5891-b43f2f067319":"en-GB|es-ES"
* },
*
*/
$langPairs = $volAnalysis['jobs']['langpairs'][$job['jid'] . "-" . $job['jpassword']];
$_langPairs_array = explode("|", $langPairs);
$source = $_langPairs_array[0];
$target = $_langPairs_array[1];
//save langpairs of the jobs
$_jobLangs[$job['jid'] . "-" . $job['jpassword']]['source'] = $source;
$_jobLangs[$job['jid'] . "-" . $job['jpassword']]['target'] = $target;
$url = "https://www.translated.net/hts/?f=quote&cid=htsdemo&p=htsdemo5&s={$source}&t={$target}&pn=MATECAT_{$job['jid']}-{$job['jpassword']}&w={$job_payableWords}&df=matecat&matecat_pid=" . $this->pid . "&matecat_ppass="******"&matecat_pname=" . $volAnalysis['data']['summary']['NAME'] . "&subject=" . $subject;
if (!$cache_cart->itemExists($job['jid'] . "-" . $job['jpassword'])) {
Log::doLog("Not Found in Cache. Call url for Quote: " . $url);
$tokenHash = $mh->createResource($url, $options, $job['jid'] . "-" . $job['jpassword']);
} else {
$cartElem = $cache_cart->getItem($job['jid'] . "-" . $job['jpassword']);
$cartElem["currency"] = $this->currency;
$cartElem["timezone"] = $this->timezone;
$cache_cart->delItem($job['jid'] . "-" . $job['jpassword']);
$cache_cart->addItem($cartElem);
}
}
$mh->multiExec();
$res = $mh->getAllContents();
$failures = array();
//fetch contents and store in cache if there are
foreach ($res as $jpid => $quote) {
if ($mh->hasError($jpid)) {
Log::doLog($mh->getError($jpid));
}
/*
* Quotes are plain text line feed separated fields in the form:
* 1
* OK
* 2014-04-16T09:30:00Z
* 488
* 46.36
* 11140320
* 1
*/
Log::doLog($quote);
$result_quote = explode("\n", $quote);
$itemCart = new Shop_ItemHTSQuoteJob();
$itemCart['id'] = $jpid;
$itemCart['project_name'] = $volAnalysis['data']['summary']['NAME'];
$itemCart['name'] = "MATECAT_{$jpid}";
$itemCart['delivery_date'] = $result_quote[2];
$itemCart['words'] = $result_quote[3];
$itemCart['price'] = $result_quote[4] ? $result_quote[4] : 0;
$itemCart['currency'] = $this->currency;
$itemCart['timezone'] = $this->timezone;
$itemCart['quote_pid'] = $result_quote[5];
$itemCart['source'] = $_jobLangs[$jpid]['source'];
//get the right language
$itemCart['target'] = $_jobLangs[$jpid]['target'];
//get the right language
$itemCart['show_info'] = $result_quote[6];
$itemCart['subject'] = $subject;
$cache_cart->addItem($itemCart);
Log::doLog($itemCart);
//Oops we got an error
if ($itemCart['price'] == 0 && empty($itemCart['words'])) {
$failures[$jpid] = $jpid;
}
}
$shopping_cart = Shop_Cart::getInstance('outsource_to_external');
//now get the right contents
foreach ($this->jobList as $job) {
$shopping_cart->delItem($job['jid'] . "-" . $job['jpassword']);
$shopping_cart->addItem($cache_cart->getItem($job['jid'] . "-" . $job['jpassword']));
$this->_quote_result = array($shopping_cart->getItem($job['jid'] . "-" . $job['jpassword']));
}
//check for failures.. destroy the cache
if (!empty($failures)) {
foreach ($failures as $jpid) {
$cache_cart->delItem($jpid);
}
}
}
protected function _call($url, array $curl_options = array())
{
$mh = new MultiCurlHandler();
$uniq_uid = uniqid('', true);
/*
* Append array elements from the second array
* to the first array while not overwriting the elements from
* the first array and not re-indexing
*
* Use the + array union operator
*/
$resourceHash = $mh->createResource($url, $this->curl_additional_params + $curl_options, $uniq_uid);
$mh->multiExec();
if ($mh->hasError($resourceHash)) {
$curl_error = $mh->getError($resourceHash);
Log::doLog('Curl Error: (http status ' . $curl_error['http_code'] . ') ' . $curl_error['errno'] . " - " . $curl_error['error'] . " " . var_export(parse_url($url), true));
$rawValue = array('error' => array('code' => -$curl_error['errno'], 'message' => " {$curl_error['error']}. Server Not Available (http status " . $curl_error['http_code'] . ")"));
//return negative number
} else {
$rawValue = $mh->getSingleContent($resourceHash);
}
$mh->multiCurlCloseAll();
if ($this->doLog) {
Log::doLog($uniq_uid . " ... Received... " . var_export($rawValue, true));
}
return $rawValue;
}
/**
* Get the number of consumers for this queue
*
* @param null $queueName
*
* @return mixed
* @throws Exception
*/
public function getConsumerCount($queueName = null)
{
if (!empty($queueName)) {
$queue = $queueName;
} elseif (!empty($this->queueName)) {
$queue = $this->queueName;
} else {
$queue = INIT::$QUEUE_NAME;
}
$queue_inteface_url = INIT::$QUEUE_JMX_ADDRESS . "/api/jolokia/read/org.apache.activemq:type=Broker,brokerName=localhost,destinationType=Queue,destinationName={$queue}/ConsumerCount";
$mHandler = new MultiCurlHandler();
$options = array(CURLOPT_HEADER => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_USERAGENT => INIT::MATECAT_USER_AGENT . INIT::$BUILD_NUMBER, CURLOPT_CONNECTTIMEOUT => 5, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, CURLOPT_HTTPHEADER => array('Authorization: Basic ' . base64_encode("admin:admin")));
$resource = $mHandler->createResource($queue_inteface_url, $options);
$mHandler->multiExec();
$result = $mHandler->getSingleContent($resource);
$mHandler->multiCurlCloseAll();
$result = json_decode($result, true);
Utils::raiseJsonExceptionError();
return $result['value'];
}
/**
* Retrieve a quote for all the jobs the user ask for, and which have not been outsourced yet
* At the end of this function, there will be in cache a quote for each one of these jobs
*
* The whole flow is composed by 2 phases:
* 1- during the first one all jobs are iterated, and for each one of them:
* if there already is something in cache telling that it has been outsourced or we have a quote, then skip it.
* Otherwise, add it to the list of the jobs we have to ask the vendor for a quote
*
* 2- In the second phase, call the vendor with the above jobs and cache al the replies in session
* @see GUIDE->"PROCEDURE"->POINT 3 for details
*
* @param string $subject
* @param array $volAnalysis
*/
private function __processNormalJobs($subject, $volAnalysis)
{
$mh = new MultiCurlHandler();
/**
************************** FIRST PART: CHECK CACHE ***************************
*/
foreach ($this->jobList as $job) {
// has the job already been outsourced? If so, it has been fully prepared in
// OutsourceTo_Translated::__processOutsourcedJobs" function. Just skip it
// NOTE: this "if" is necessary in order to not process again a job already outsourced.
// A possible alternative is to unset from the $this->jobList array all the jobs detected as
// outsourced during OutsourceTo_Translated::__processOutsourcedJobs function
if (Shop_Cart::getInstance('outsource_to_external_cache')->itemExists($job['jid'] . "-" . $job['jpassword'] . "-outsourced")) {
continue;
}
// in case we have a quote in cache, we are done with this job anyway
if (Shop_Cart::getInstance('outsource_to_external_cache')->itemExists($job['jid'] . "-" . $job['jpassword'] . "-" . $this->fixedDelivery)) {
// update the job localization info (currency and timezone), according to user preferences
$this->__updateCartElements($job['jid'] . "-" . $job['jpassword'] . "-" . $this->fixedDelivery, $this->currency, $this->timezone, $this->typeOfService);
continue;
}
// nothing in cache, we have to directly ask to the vendor: compose URL and create a proper curl resource
// get source and target languages from volume analysis
// NOTE: languages are in the form:
// "langpairs":{
// "5888-e94bd2f79afd":"en-GB|fr-FR",
// "5889-c853a841dafd":"en-GB|de-DE",
// "5890-e852ca45c66e":"en-GB|it-IT",
// "5891-b43f2f067319":"en-GB|es-ES"
// },
list($source, $target) = explode("|", $volAnalysis['jobs']['langpairs'][$job['jid'] . "-" . $job['jpassword']]);
// get words from volume analysis, and round decimals to int
// NOTE: the vendor returns an error in case words = 0 -> make sure $words is at least 1
$words = max((int) $volAnalysis['data']['jobs'][$job['jid']]['totals'][$job['jpassword']]['TOTAL_PAYABLE'][0], 1);
// get delivery date chosen by the user (if any), otherwise set it to 0 to tell the vendor no date has been specified
// NOTE: UI returns a timestamp in millis. Despite we use the one in millis for the caching ID
// (See: GUIDE->"NORMAL QUOTES vs OUTSOURCED QUOTES"), we here need to convert it in seconds
// and provide a MySQL -like date format. E.g. "1989-10-15 18:24:00"
$fixedDeliveryDateForQuote = $this->fixedDelivery > 0 ? date("Y-m-d H:i:s", $this->fixedDelivery / 1000) : "0";
$url = "https://www.translated.net/hts/matecat-endpoint.php?f=quote&cid=htsdemo&p=htsdemo5&s={$source}&t={$target}" . "&pn=MATECAT_{$job['jid']}-{$job['jpassword']}&w={$words}&df=matecat&matecat_pid=" . $this->pid . "&matecat_ppass="******"&matecat_pname=" . $volAnalysis['data']['summary']['NAME'] . "&subject={$subject}&jt=R&fd=" . urlencode($fixedDeliveryDateForQuote) . "&of=json";
Log::doLog("Not Found in Cache. Call url for Quote: " . $url);
$mh->createResource($url, $this->_curlOptions, $job['jid'] . "-" . $job['jpassword'] . "-" . $this->fixedDelivery);
}
/**
************************** SECOND PART: CALL VENDOR *****************************
*/
// execute call and retrieve the result
$mh->multiExec();
$res = $mh->getAllContents();
// for each reply
foreach ($res as $jpid => $quote) {
// if some error occurred, log it and skip this job
if ($mh->hasError($jpid)) {
Log::doLog($mh->getError($jpid));
continue;
}
Log::doLog($quote);
// parse the result and check if the vendor returned some error. In case, skip the quote
$result_quote = json_decode($quote, TRUE);
if ($result_quote['code'] != 1) {
Log::doLog("HTS returned an error. Skip quote");
continue;
}
// quote received correctly. Create a proper Shop_ItemHTSQuoteJob to hold it, and add it to the Shop_Cart
$itemCart = $this->__prepareQuotedJobCart($jpid, $volAnalysis, $subject, $result_quote);
// NOTE: In this case we only have to delete the single quote, and replace it with the new one,
// but all the other quotes the user might have asked must remain in cache,
// so do not pass the "true" parameter, as done before in "OutsourceTo_Translated::__processOutsourcedJobs".
// See GUIDE->"NORMAL QUOTES vs OUTSOURCED QUOTES" for details
$this->__addCartElement($itemCart);
Log::doLog($itemCart);
}
}
public function multiConvertToOriginal($xliffVector_array, $chosen_by_user_machine = false)
{
$multiCurlObj = new MultiCurlHandler();
$conversionObjects = array();
$temporary_files = array();
//iterate files.
//For each file prepare a curl resource
foreach ($xliffVector_array as $id_file => $xliffVector) {
$xliffContent = $xliffVector['documentContent'];
//assign converter
if (!$chosen_by_user_machine) {
$this->ip = $this->pickRandConverter();
$storage = $this->getValidStorage();
//add replace/regexp pattern because we have more than 1 replacement
//http://stackoverflow.com/questions/2222643/php-preg-replace
$xliffContent = self::replacedAddress($storage, $xliffContent);
} else {
$this->ip = $chosen_by_user_machine;
}
$url = "{$this->ip}:{$this->port}/{$this->fromXliffFunction}";
$uid_ext = $this->extractUidandExt($xliffContent);
$data['uid'] = $uid_ext[0];
//get random name for temporary location
$tmp_name = tempnam("/tmp", "MAT_BW");
$temporary_files[] = $tmp_name;
//write encoded file to temporary location
$fileSize = file_put_contents($tmp_name, $xliffContent);
$data['xliffContent'] = "@{$tmp_name}";
$xliffName = $xliffVector['out_xliff_name'];
//prepare conversion object
$this->conversionObject->ip_machine = $this->ip;
$this->conversionObject->ip_client = Utils::getRealIpAddr();
$this->conversionObject->path_name = $xliffVector['out_xliff_name'];
$this->conversionObject->file_name = pathinfo($xliffName, PATHINFO_BASENAME);
$this->conversionObject->direction = 'bw';
$this->conversionObject->src_lang = $this->lang_handler->getLangRegionCode($xliffVector['source']);
$this->conversionObject->trg_lang = $this->lang_handler->getLangRegionCode($xliffVector['target']);
$this->conversionObject->file_size = $fileSize;
$conversionObjects[$id_file] = clone $this->conversionObject;
$options = array(CURLOPT_URL => $url, CURLOPT_HEADER => 0, CURLOPT_USERAGENT => INIT::MATECAT_USER_AGENT . INIT::$BUILD_NUMBER, CURLOPT_RETURNTRANSFER => true, CURLOPT_POST => true, CURLOPT_POSTFIELDS => $data, CURLOPT_HTTPHEADER => $this->opt['httpheader']);
$multiCurlObj->createResource($url, $options, $id_file);
}
//Perform curl
Log::doLog("multicurl_start");
$multiCurlObj->multiExec();
Log::doLog("multicurl_end");
$multiInfo = $multiCurlObj->getAllInfo();
$multiResponses = $multiCurlObj->getAllContents();
//decode response and return the result
foreach ($multiResponses as $hash => $json) {
$multiResponses[$hash] = json_decode($json, true);
$conversionObjects[$hash]->conversion_time = $multiInfo[$hash]['curlinfo_total_time'];
$multiResponses[$hash] = $this->__parseOutput($multiResponses[$hash], $conversionObjects[$hash]);
}
//remove temporary files
foreach ($temporary_files as $temp_name) {
unlink($tmp_name);
}
return $multiResponses;
}
