protected function extractMessages($resultset, $offset, $limit)
{
$messages = $documents = $ret = array();
$language = $this->params['language'];
foreach ($resultset->getResults() as $document) {
$data = $document->getData();
if (!$this->server->isLocalSuggestion($data)) {
continue;
}
$title = Title::newFromText($data['localid']);
if (!$title) {
continue;
}
$handle = new MessageHandle($title);
if (!$handle->isValid()) {
continue;
}
$key = $title->getNamespace() . ':' . $title->getDBKey();
$messages[$key] = $data['content'];
}
$definitions = new MessageDefinitions($messages);
$collection = MessageCollection::newFromDefinitions($definitions, $language);
$filter = $this->params['filter'];
if ($filter === 'untranslated') {
$collection->filter('hastranslation', true);
} elseif (in_array($filter, $this->getAvailableFilters())) {
$collection->filter($filter, false);
}
$total = count($collection);
$offset = $collection->slice($offset, $limit);
$left = count($collection);
$offsets = array('start' => $offset[2], 'left' => $left, 'total' => $total);
if ($filter === 'translated' || $filter === 'fuzzy') {
$collection->loadTranslations();
}
foreach ($collection->keys() as $mkey => $title) {
$documents[$mkey]['content'] = $messages[$mkey];
if ($filter === 'translated' || $filter === 'fuzzy') {
$documents[$mkey]['content'] = $collection[$mkey]->translation();
}
$handle = new MessageHandle($title);
$documents[$mkey]['localid'] = $handle->getTitleForBase()->getPrefixedText();
$documents[$mkey]['language'] = $language;
$ret[] = $documents[$mkey];
}
return array($ret, $offsets);
}
public function __construct($config)
{
parent::__construct($config);
if (isset($config['config'])) {
$this->client = new Solarium_Client($config['config']);
} else {
$this->client = new Solarium_Client();
}
}
public function testConstruct()
{
$server = TTMServer::primary();
$this->assertEquals('FakeTTMServer', get_class($server), 'Fake server given when default server is disabled');
global $wgTranslateTranslationServices;
$wgTranslateTranslationServices['TTMServer'] = array('database' => false, 'cutoff' => 0.75, 'type' => 'ttmserver', 'public' => false);
$server = TTMServer::primary();
$this->assertEquals('DatabaseTTMServer', get_class($server), 'Real server given when default server is enabled');
unset($wgTranslateTranslationServices['TTMServer']);
}
/**
* Fetch all the translations and update them.
*/
function run()
{
$handle = new MessageHandle($this->title);
$translations = ApiQueryMessageTranslations::getTranslations($handle);
foreach ($translations as $page => $data) {
$tTitle = Title::makeTitle($this->title->getNamespace(), $page);
$tHandle = new MessageHandle($tTitle);
TTMServer::onChange($tHandle, $data[0], $tHandle->isFuzzy());
}
return true;
}
protected function getAvailableTranslationServices()
{
global $wgTranslateTranslationServices;
$good = array();
foreach ($wgTranslateTranslationServices as $id => $config) {
if (TTMServer::factory($config) instanceof SearchableTTMServer) {
$good[] = $id;
}
}
return $good;
}
public function execute()
{
$params = $this->extractRequestParams();
$server = TTMServer::primary();
$suggestions = $server->query($params['sourcelanguage'], $params['targetlanguage'], $params['text']);
$result = $this->getResult();
foreach ($suggestions as $sug) {
$result->setContent($sug, $sug['target']);
unset($sug['target']);
$result->addValue($this->getModuleName(), null, $sug);
}
$result->setIndexedTagName_internal($this->getModuleName(), 'suggestion');
}
public function getData()
{
$suggestions = array();
$text = $this->getDefinition();
$from = $this->group->getSourceLanguage();
$to = $this->handle->getCode();
// "Local" queries using a client can't be run in parallel with web services
global $wgTranslateTranslationServices;
foreach ($wgTranslateTranslationServices as $name => $config) {
$server = TTMServer::factory($config);
try {
if ($server instanceof ReadableTTMServer) {
// Except if they are public, we can call back via API
if (isset($config['public']) && $config['public'] === true) {
continue;
}
$query = $server->query($from, $to, $text);
} else {
continue;
}
} catch (Exception $e) {
// Not ideal to catch all exceptions
continue;
}
foreach ($query as $item) {
$item['service'] = $name;
$item['source_language'] = $from;
$item['local'] = $server->isLocalSuggestion($item);
$item['uri'] = $server->expandLocation($item);
$suggestions[] = $item;
}
}
// Results from web services
foreach ($this->getQueryData() as $queryData) {
$sugs = $this->formatSuggestions($queryData);
$suggestions = array_merge($suggestions, $sugs);
}
$suggestions = TTMServer::sortSuggestions($suggestions);
// Must be here to not mess up the sorting function
$suggestions['**'] = 'suggestion';
return $suggestions;
}
/**
* Returns suggestions from a translation memory.
* @param $serviceName
* @param $config
* @return string Html snippet which contains the suggestions.
*/
protected function getTTMServerBox($serviceName, $config)
{
$this->mustHaveDefinition();
$this->mustBeTranslation();
$source = $this->group->getSourceLanguage();
$code = $this->handle->getCode();
$definition = $this->getDefinition();
$suggestions = TTMServer::primary()->query($source, $code, $definition);
if (count($suggestions) === 0) {
return null;
}
foreach ($suggestions as $s) {
$accuracy = wfMsgHtml('translate-edit-tmmatch', sprintf('%.2f', $s['quality'] * 100));
$legend = array($accuracy => array());
$sourceTitle = Title::newFromText($s['context']);
$handle = new MessageHandle($sourceTitle);
$targetTitle = Title::makeTitle($sourceTitle->getNamespace(), $handle->getKey() . "/{$code}");
if ($targetTitle) {
$legend[$accuracy][] = self::ajaxEditLink($targetTitle, '•');
}
// Show the source text in a tooltip
$text = $this->suggestionField($s['target']);
$params = array('class' => 'mw-sp-translate-edit-tmsug', 'title' => $s['source']);
// Group identical suggestions together
if (isset($sugFields[$s['target']])) {
$sugFields[$s['target']][2] = array_merge_recursive($sugFields[$s['target']][2], $legend);
} else {
$sugFields[$s['target']] = array($text, $params, $legend);
}
}
foreach ($sugFields as $field) {
list($text, $params, $label) = $field;
$legend = array();
foreach ($label as $acc => $links) {
$legend[] = $acc . ' ' . implode(" ", $links);
}
$legend = implode(' | ', $legend);
$boxes[] = Html::rawElement('div', $params, self::legend($legend) . $text . self::clear()) . "\n";
}
// Limit to three best
$boxes = array_slice($boxes, 0, 3);
$result = implode("\n", $boxes);
return $result;
}
protected function processQueryResults($res, $text, $targetLanguage)
{
$timeLimit = microtime(true) + 5;
$lenA = mb_strlen($text);
$results = array();
foreach ($res as $row) {
if (microtime(true) > $timeLimit) {
// Having no suggestions is better than preventing translation
// altogether by timing out the request :(
break;
}
$a = $text;
$b = $row->tms_text;
$lenB = mb_strlen($b);
$len = min($lenA, $lenB);
if ($len > 600) {
// two strings of length 1500 ~ 10s
// two strings of length 2250 ~ 30s
$dist = $len;
} else {
$dist = self::levenshtein($a, $b, $lenA, $lenB);
}
$quality = 1 - $dist * 0.9 / $len;
if ($quality >= $this->config['cutoff']) {
$results[] = array('source' => $row->tms_text, 'target' => $row->tmt_text, 'context' => $row->tms_context, 'location' => $row->tms_context . '/' . $targetLanguage, 'quality' => $quality, 'wiki' => isset($row->tms_wiki) ? $row->tms_wiki : wfWikiId());
}
}
$results = TTMServer::sortSuggestions($results);
return $results;
}
/**
* Runs message checks, adds tp:transver tags and updates statistics.
* Hook: PageContentSaveComplete
*/
public static function onSave(WikiPage $wikiPage, $user, $content, $summary, $minor, $_, $_, $flags, $revision)
{
if ($content instanceof TextContent) {
$text = $content->getNativeData();
} else {
// Screw it, not interested
return true;
}
$title = $wikiPage->getTitle();
$handle = new MessageHandle($title);
if (!$handle->isValid()) {
return true;
}
// Update it.
if ($revision === null) {
$rev = $wikiPage->getTitle()->getLatestRevId();
} else {
$rev = $revision->getID();
}
$fuzzy = self::checkNeedsFuzzy($handle, $text);
$fuzzyOp = self::updateFuzzyTag($title, $rev, $fuzzy);
// Skip the hook if no change in status or content
if ($revision || $fuzzyOp) {
Hooks::run('TranslateEventTranslationEdit', array($handle));
}
if ($fuzzy === false) {
Hooks::run('Translate:newTranslation', array($handle, $rev, $text, $user));
}
TTMServer::onChange($handle, $text, $fuzzy);
return true;
}
public function testSearchableTTMServer()
{
global $wgTranslateTestTTMServer;
if (!$wgTranslateTestTTMServer) {
$this->markTestSkipped('No test TTMServer available');
}
$server = TTMServer::factory($wgTranslateTestTTMServer);
$solarium = $server->getSolarium();
// Empty it
$update = $solarium->createUpdate();
$update->addDeleteQuery('*:*');
$update->addCommit();
$solarium->update($update);
// Check that it is empty indeed
$select = $solarium->createSelect();
$select->setRows(0);
$select->setQuery('*:*');
$result = $solarium->select($select);
$this->assertEquals(0, $result->getNumFound());
// Add one definition
$title = Title::newFromText('MediaWiki:one/en');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('1', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setRows(1);
$select->setQuery('*:*');
$result = $solarium->select($select);
$this->assertEquals(1, $result->getNumFound());
$doc = null;
foreach ($result as $doc) {
// @todo FIXME Empty foreach statement.
}
$this->assertEquals(wfWikiId(), $doc->wiki);
$this->assertEquals('en', $doc->language);
$this->assertEquals('1', $doc->content);
$this->assertEquals(array('ttmserver-test'), $doc->group);
// Add one translation
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('yksi', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setRows(1);
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(1, $result->getNumFound());
$doc = null;
foreach ($result as $doc) {
// @todo FIXME Empty foreach statement.
}
$this->assertEquals('yksi', $doc->content);
$this->assertEquals(array('ttmserver-test'), $doc->group);
// Update definition
$title = Title::newFromText('MediaWiki:one/en');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('1-updated', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:en');
$result = $solarium->select($select);
$this->assertEquals(2, $result->getNumFound(), 'Old and new definition exists');
// Translation is fuzzied
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('!!FUZZY!!yksi', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(0, $result->getNumFound());
// Translation is udpated
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('yksi-päiv', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(1, $result->getNumFound());
$doc = null;
foreach ($result as $doc) {
// @todo FIXME Empty foreach statement.
}
$this->assertEquals('yksi-päiv', $doc->content);
// And now the messages should be orphaned
global $wgHooks;
$wgHooks['TranslatePostInitGroups'] = array();
MessageGroups::singleton()->recache();
MessageIndex::singleton()->rebuild();
self::runJobs();
$select = $solarium->createSelect();
$select->setQuery('*:*');
$result = $solarium->select($select);
$this->assertEquals(2, $result->getNumFound(), 'One definition and one translation exists');
foreach ($result as $doc) {
$this->assertEquals(null, $doc->group, 'Messages are orphaned');
}
// And message deletion
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$page->doDeleteArticle(__METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(0, $result->getNumFound());
}
/**
* Shovels the new translation into TTMServer.
* Hook: Translate:newTranslation
*/
public static function updateTM(MessageHandle $handle, $revision, $text, User $user)
{
TTMServer::primary()->update($handle, $text);
return true;
}
protected function exportGroup(MessageGroup $group, $multi = false)
{
// Make sure all existing connections are dead,
// we can't use them in forked children.
LBFactory::destroyInstance();
$server = TTMServer::primary();
$id = $group->getId();
$sourceLanguage = $group->getSourceLanguage();
if ($multi) {
$stats = MessageGroupStats::forGroup($id);
$this->statusLine("Loaded stats for {$id}\n");
} else {
$this->statusLine("Loading stats... ", 4);
$stats = MessageGroupStats::forGroup($id);
$this->output("done!", 4);
$this->statusLine("Inserting sources: ", 5);
}
$collection = $group->initCollection($sourceLanguage);
$collection->filter('ignored');
$collection->filter('optional');
$collection->initMessages();
$sids = array();
$counter = 0;
foreach ($collection->keys() as $mkey => $title) {
$def = $collection[$mkey]->definition();
$sids[$mkey] = $server->insertSource($title, $sourceLanguage, $def);
if (++$counter % $this->mBatchSize === 0 && !$multi) {
wfWaitForSlaves(10);
$this->output('.', 5);
}
}
$total = count($sids);
if ($multi) {
$this->statusLine("Inserted {$total} source entries for {$id}\n");
} else {
$this->output("{$total} entries", 5);
$this->statusLine("Inserting translations...", 6);
}
$dbw = $server->getDB(DB_MASTER);
foreach ($stats as $targetLanguage => $numbers) {
if ($targetLanguage === $sourceLanguage) {
continue;
}
if ($numbers[MessageGroupStats::TRANSLATED] === 0) {
continue;
}
if (!$multi) {
$this->output(sprintf("%19s ", $targetLanguage), $targetLanguage);
}
$collection->resetForNewLanguage($targetLanguage);
$collection->filter('ignored');
$collection->filter('optional');
$collection->filter('translated', false);
$collection->loadTranslations();
$inserts = array();
foreach ($collection->keys() as $mkey => $title) {
$inserts[] = array('tmt_sid' => $sids[$mkey], 'tmt_lang' => $targetLanguage, 'tmt_text' => $collection[$mkey]->translation());
}
do {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$dbw->insert('translate_tmt', $batch, __METHOD__);
if (!$multi) {
$this->output('.', $targetLanguage);
}
wfWaitForSlaves(10);
} while (count($inserts));
}
if ($multi) {
$this->statusLine("Inserted translations for {$id}\n");
}
}
public static function onChange(MessageHandle $handle, $text, $fuzzy)
{
if ($fuzzy) {
$text = null;
}
TTMServer::primary()->update($handle, $text);
}
public function execute($par)
{
global $wgLanguageCode;
$this->setHeaders();
$this->checkPermissions();
$server = TTMServer::primary();
if (!$server instanceof SearchableTTMServer) {
throw new ErrorPageError('tux-sst-nosolr-title', 'tux-sst-nosolr-body');
}
$out = $this->getOutput();
$out->addModuleStyles('jquery.uls.grid');
$out->addModuleStyles('ext.translate.special.searchtranslations.styles');
$out->addModuleStyles('ext.translate.special.translate.styles');
$out->addModules('ext.translate.special.searchtranslations');
$out->addModules('ext.translate.special.searchtranslations.operatorsuggest');
TranslateUtils::addSpecialHelpLink($out, 'Help:Extension:Translate#searching');
$this->opts = $opts = new FormOptions();
$opts->add('query', '');
$opts->add('sourcelanguage', $wgLanguageCode);
$opts->add('language', '');
$opts->add('group', '');
$opts->add('grouppath', '');
$opts->add('filter', '');
$opts->add('match', '');
$opts->add('case', '');
$opts->add('limit', $this->limit);
$opts->add('offset', 0);
$opts->fetchValuesFromRequest($this->getRequest());
$queryString = $opts->getValue('query');
if ($queryString === '') {
$this->showEmptySearch();
return;
}
$params = $opts->getAllValues();
$filter = $opts->getValue('filter');
try {
$translationSearch = new CrossLanguageTranslationSearchQuery($params, $server);
if (in_array($filter, $translationSearch->getAvailableFilters())) {
if ($opts->getValue('language') === '') {
$params['language'] = $this->getLanguage()->getCode();
$opts->setValue('language', $params['language']);
}
$documents = $translationSearch->getDocuments();
$total = $translationSearch->getTotalHits();
$resultset = $translationSearch->getResultSet();
} else {
$resultset = $server->search($queryString, $params, $this->hl);
$documents = $server->getDocuments($resultset);
$total = $server->getTotalHits($resultset);
}
} catch (TTMServerException $e) {
error_log('Translation search server unavailable:' . $e->getMessage());
throw new ErrorPageError('tux-sst-solr-offline-title', 'tux-sst-solr-offline-body');
}
// Part 1: facets
$facets = $server->getFacets($resultset);
$facetHtml = '';
if (count($facets['language']) > 0) {
if ($filter !== '') {
$facets['language'] = array_merge($facets['language'], array($opts->getValue('language') => $total));
}
$facetHtml = Html::element('div', array('class' => 'row facet languages', 'data-facets' => FormatJson::encode($this->getLanguages($facets['language'])), 'data-language' => $opts->getValue('language')), $this->msg('tux-sst-facet-language'));
}
if (count($facets['group']) > 0) {
$facetHtml .= Html::element('div', array('class' => 'row facet groups', 'data-facets' => FormatJson::encode($this->getGroups($facets['group'])), 'data-group' => $opts->getValue('group')), $this->msg('tux-sst-facet-group'));
}
// Part 2: results
$resultsHtml = '';
$title = Title::newFromText($queryString);
if ($title && !in_array($filter, $translationSearch->getAvailableFilters())) {
$handle = new MessageHandle($title);
$code = $handle->getCode();
$language = $opts->getValue('language');
if ($handle->isValid() && $code !== '' && $code !== $language) {
$groupId = $handle->getGroup()->getId();
$helpers = new TranslationHelpers($title, $groupId);
$document['wiki'] = wfWikiId();
$document['localid'] = $handle->getTitleForBase()->getPrefixedText();
$document['content'] = $helpers->getTranslation();
$document['language'] = $handle->getCode();
array_unshift($documents, $document);
$total++;
}
}
foreach ($documents as $document) {
$text = $document['content'];
$text = TranslateUtils::convertWhiteSpaceToHTML($text);
list($pre, $post) = $this->hl;
$text = str_replace($pre, '<strong class="tux-search-highlight">', $text);
$text = str_replace($post, '</strong>', $text);
$title = Title::newFromText($document['localid'] . '/' . $document['language']);
if (!$title) {
// Should not ever happen but who knows...
continue;
}
$resultAttribs = array('class' => 'row tux-message', 'data-title' => $title->getPrefixedText(), 'data-language' => $document['language']);
$handle = new MessageHandle($title);
if ($handle->isValid()) {
$groupId = $handle->getGroup()->getId();
$helpers = new TranslationHelpers($title, $groupId);
$resultAttribs['data-definition'] = $helpers->getDefinition();
$resultAttribs['data-translation'] = $helpers->getTranslation();
$resultAttribs['data-group'] = $groupId;
$uri = $title->getLocalUrl(array('action' => 'edit'));
$link = Html::element('a', array('href' => $uri), $this->msg('tux-sst-edit')->text());
} else {
$url = wfParseUrl($document['uri']);
$domain = $url['host'];
$link = Html::element('a', array('href' => $document['uri']), $this->msg('tux-sst-view-foreign', $domain)->text());
}
$access = Html::rawElement('div', array('class' => 'row tux-edit tux-message-item'), $link);
$titleText = $title->getPrefixedText();
$titleAttribs = array('class' => 'row tux-title', 'dir' => 'ltr');
$textAttribs = array('class' => 'row tux-text', 'lang' => wfBCP47($document['language']), 'dir' => Language::factory($document['language'])->getDir());
$resultsHtml = $resultsHtml . Html::openElement('div', $resultAttribs) . Html::rawElement('div', $textAttribs, $text) . Html::element('div', $titleAttribs, $titleText) . $access . Html::closeElement('div');
}
$resultsHtml .= Html::rawElement('hr', array('class' => 'tux-pagination-line'));
$prev = $next = '';
$offset = $this->opts->getValue('offset');
$params = $this->opts->getChangedValues();
if ($total - $offset > $this->limit) {
$newParams = array('offset' => $offset + $this->limit) + $params;
$attribs = array('class' => 'mw-ui-button pager-next', 'href' => $this->getPageTitle()->getLocalUrl($newParams));
$next = Html::element('a', $attribs, $this->msg('tux-sst-next')->text());
}
if ($offset) {
$newParams = array('offset' => max(0, $offset - $this->limit)) + $params;
$attribs = array('class' => 'mw-ui-button pager-prev', 'href' => $this->getPageTitle()->getLocalUrl($newParams));
$prev = Html::element('a', $attribs, $this->msg('tux-sst-prev')->text());
}
$resultsHtml .= Html::rawElement('div', array('class' => 'tux-pagination-links'), "{$prev} {$next}");
$search = $this->getSearchInput($queryString);
$count = $this->msg('tux-sst-count')->numParams($total);
$this->showSearch($search, $count, $facetHtml, $resultsHtml, $total);
}
protected function exportGroup(MessageGroup $group, $config)
{
$server = TTMServer::factory($config);
$server->setLogger($this);
$id = $group->getId();
$sourceLanguage = $group->getSourceLanguage();
$stats = MessageGroupStats::forGroup($id);
$collection = $group->initCollection($sourceLanguage);
$collection->filter('ignored');
$collection->initMessages();
$server->beginBatch();
$inserts = array();
foreach ($collection->keys() as $mkey => $title) {
$handle = new MessageHandle($title);
$inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->definition());
}
while ($inserts !== array()) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertDefinitions($batch);
}
$inserts = array();
foreach ($stats as $targetLanguage => $numbers) {
if ($targetLanguage === $sourceLanguage) {
continue;
}
if ($numbers[MessageGroupStats::TRANSLATED] === 0) {
continue;
}
$collection->resetForNewLanguage($targetLanguage);
$collection->filter('ignored');
$collection->filter('translated', false);
$collection->loadTranslations();
foreach ($collection->keys() as $mkey => $title) {
$handle = new MessageHandle($title);
$inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->translation());
}
while (count($inserts) >= $this->mBatchSize) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertTranslations($batch);
}
}
while ($inserts !== array()) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertTranslations($batch);
}
$server->endBatch();
}