/**
* Itterate through all groups, loading current data from the existing
* extension and opening message files for message output.
* - If the group does not define a special page alias file or magic
* words file, or that file does not exist, it is ignored silently.
* - If the file does contain a data array (e.g. $aliases) then the
* program exits.
*/
protected function openHandles() {
$this->output( "Opening file handles and loading current data...\n" );
$groups = MessageGroups::singleton()->getGroups();
$filename = null;
foreach ( $groups as $group ) {
if ( !$group instanceof ExtensionMessageGroup ) {
continue;
}
switch ( $this->type ) {
case 'special':
$filename = $group->getAliasFile();
break;
case 'magic':
$filename = $group->getMagicFile();
break;
}
if ( $filename === null ) {
continue;
}
global $wgTranslateExtensionDirectory;
$inFile = "$wgTranslateExtensionDirectory/$filename";
if ( !file_exists( $inFile ) ) {
continue;
}
include( $inFile );
switch( $this->type ) {
case 'special':
if ( isset( $aliases ) ) {
$this->messagesOld[$group->getId()] = $aliases;
unset( $aliases );
} elseif ( isset( $specialPageAliases ) ) {
$this->messagesOld[$group->getId()] = $specialPageAliases;
unset( $specialPageAliases );
} else {
die( "File '$inFile' does not contain an aliases array.\n" );
}
break;
case 'magic':
if ( !isset( $magicWords ) ) {
die( "File '$inFile' does not contain a magic words array.\n" );
}
$this->messagesOld[$group->getId()] = $magicWords;
unset( $magicWords );
break;
}
$outFile = $this->target . '/' . $filename;
wfMkdirParents( dirname( $outFile ), null, __METHOD__ );
$this->handles[$group->getId()] = fopen( $outFile, 'w' );
fwrite( $this->handles[$group->getId()], $this->readHeader( $inFile ) );
$this->output( "\t{$group->getId()}\n" );
}
}
public function rebuild($scratch = false)
{
$groups = MessageGroups::singleton()->getGroups();
$new = $old = array();
if (!$scratch) {
// To avoid inifinite recursion
$old = $this->retrieve();
}
$postponed = array();
STDOUT("Working with ", 'main');
foreach ($groups as $g) {
if (!$g->exists()) {
continue;
}
# Skip meta thingies
if ($g->isMeta()) {
$postponed[] = $g;
continue;
}
$this->checkAndAdd($new, $g);
}
foreach ($postponed as $g) {
$this->checkAndAdd($new, $g, true);
}
$this->store($new);
$this->clearMessageGroupStats($old, $new);
return $new;
}
private function displayNavigation()
{
global $wgOut;
$groupSelector = new XmlSelect('group', 'group-select');
// pull groups
$groups = MessageGroups::singleton()->getGroups();
foreach ($groups as $group) {
if (!$group->isMeta()) {
continue;
}
$groupSelector->addOption($group->getLabel(), $group->getId());
}
$fields = array();
$fields['transstats-choose-group'] = $groupSelector->getHTML();
$fields['transstats-group-mode-all'] = Xml::radio('mode', 0, empty($this->mMode));
$fields['transstats-group-mode-supress0'] = Xml::radio('mode', 1, $this->mMode == 1);
$fields['transstats-group-mode-supress100'] = Xml::radio('mode', 2, $this->mMode == 2);
$fields['transstats-group-mode-only100'] = Xml::radio('mode', 3, $this->mMode == 3);
$fields['transstats-group-langlist'] = Xml::input('langlist', false, $this->mLanglistPlain);
$out = Xml::openElement('form');
$out .= Xml::buildForm($fields);
$out .= Html::hidden('title', 'Special:' . $this->getName());
// FIXME: this is silly...
$out .= Xml::submitButton(wfMsg('transstats-submit'));
$out .= Xml::closeElement('form');
$wgOut->addHTML($out);
}
/**
* Returns translated percentage for message group in given
* languages
*
* @param $group \string Unique key identifying the group
* @param $languages \array List of language codes
* @param $threshold \int Minimum required percentage translated to
* return. Other given language codes will not be returned.
* @param $simple \bool Return only codes or code/pecentage pairs
*
* @return \array Array of key value pairs code (string)/percentage
* (float) or array of codes, depending on $simple
*/
public static function getPercentageTranslated( $group, $languages, $threshold = false, $simple = false ) {
$stats = array();
$g = MessageGroups::singleton()->getGroup( $group );
$collection = $g->initCollection( 'en' );
foreach ( $languages as $code ) {
$collection->resetForNewLanguage( $code );
// Initialise messages
$collection->filter( 'ignored' );
$collection->filter( 'optional' );
// Store the count of real messages for later calculation.
$total = count( $collection );
$collection->filter( 'translated', false );
$translated = count( $collection );
$translatedPercentage = ( $translated * 100 ) / $total;
if ( $translatedPercentage >= $threshold ) {
if ( $simple ) {
$stats[] = $code;
} else {
$stats[$code] = $translatedPercentage;
}
}
}
return $stats;
}
public function execute()
{
$server = TTMServer::primary();
if ($server instanceof FakeTTMServer) {
$this->error("Translation memory is not configured properly", 1);
}
$dbw = $server->getDB(DB_MASTER);
$this->statusLine('Deleting sources.. ', 1);
$dbw->delete('translate_tms', '*', __METHOD__);
$this->output('translations.. ', 1);
$dbw->delete('translate_tmt', '*', __METHOD__);
$this->output('fulltext.. ', 1);
$dbw->delete('translate_tmf', '*', __METHOD__);
$table = $dbw->tableName('translate_tmf');
$dbw->query("DROP INDEX tmf_text ON {$table}");
$this->output('done!', 1);
$this->statusLine('Loading groups... ', 2);
$groups = MessageGroups::singleton()->getGroups();
$this->output('done!', 2);
$threads = $this->getOption('threads', 1);
$pids = array();
foreach ($groups as $id => $group) {
if ($group->isMeta()) {
continue;
}
// Fork to avoid unbounded memory usage growth
$pid = pcntl_fork();
if ($pid === 0) {
// Child, reseed because there is no bug in PHP:
// http://bugs.php.net/bug.php?id=42465
mt_srand(getmypid());
$this->exportGroup($group, $threads > 1);
exit;
} elseif ($pid === -1) {
// Fork failed do it serialized
$this->exportGroup($group);
} else {
$this->statusLine("Forked thread {$pid} to handle {$id}\n");
$pids[$pid] = true;
// If we hit the thread limit, wait for any child to finish.
if (count($pids) >= $threads) {
$status = 0;
$pid = pcntl_wait($status);
unset($pids[$pid]);
}
}
}
// Return control after all threads have finished.
foreach (array_keys($pids) as $pid) {
$status = 0;
pcntl_waitpid($pid, $status);
}
$this->statusLine('Adding fulltext index...', 9);
$table = $dbw->tableName('translate_tmf');
$dbw->query("CREATE FULLTEXT INDEX tmf_text ON {$table} (tmf_text)");
$this->output(' done!', 9);
}
public function testHaveSingleSourceLanguage()
{
$this->setMwGlobals(array('wgTranslateGroupFiles' => array(__DIR__ . '/data/MixedSourceLanguageGroups.yaml')));
MessageGroups::singleton()->recache();
$enGroup1 = MessageGroups::getGroup('EnglishGroup1');
$enGroup2 = MessageGroups::getGroup('EnglishGroup2');
$teGroup1 = MessageGroups::getGroup('TeluguGroup1');
$this->assertEquals('en', MessageGroups::haveSingleSourceLanguage(array($enGroup1, $enGroup2)));
$this->assertEquals('', MessageGroups::haveSingleSourceLanguage(array($enGroup1, $enGroup2, $teGroup1)));
}
protected function setUp()
{
parent::setUp();
global $wgHooks;
$this->setMwGlobals(array('wgHooks' => $wgHooks, 'wgTranslateTranslationServices' => array()));
$wgHooks['TranslatePostInitGroups'] = array(array($this, 'getTestGroups'));
$mg = MessageGroups::singleton();
$mg->setCache(wfGetCache('hash'));
$mg->recache();
}
public function setUp()
{
parent::setUp();
global $wgHooks;
$this->setMwGlobals(array('wgHooks' => $wgHooks, 'wgTranslateTranslationServices' => array(), 'wgTranslateDelayedMessageIndexRebuild' => false));
$wgHooks['TranslatePostInitGroups'] = array();
$mg = MessageGroups::singleton();
$mg->setCache(wfGetCache('hash'));
$mg->recache();
MessageIndex::setInstance(new HashMessageIndex());
MessageIndex::singleton()->rebuild();
}
protected function setUp()
{
parent::setUp();
global $wgHooks;
$this->setMwGlobals(array('wgHooks' => $wgHooks, 'wgTranslateTranslationServices' => array(), 'wgTranslateCacheDirectory' => $this->getNewTempDirectory()));
$wgHooks['TranslatePostInitGroups'] = array();
$mg = MessageGroups::singleton();
$mg->setCache(wfGetCache('hash'));
$mg->recache();
MessageIndex::setInstance(new HashMessageIndex());
MessageIndex::singleton()->rebuild();
}
protected function setUp()
{
parent::setUp();
global $wgHooks;
$this->setMwGlobals(array('wgHooks' => $wgHooks, 'wgGroupPermissions' => array(), 'wgTranslateMessageNamespaces' => array(NS_MEDIAWIKI)));
$wgHooks['TranslatePostInitGroups'] = array(array($this, 'getTestGroups'));
$mg = MessageGroups::singleton();
$mg->setCache(wfGetCache('hash'));
$mg->recache();
MessageIndex::setInstance(new HashMessageIndex());
MessageIndex::singleton()->rebuild();
}
/**
* Iterate through all groups, loading current data from the existing
* extension and opening message files for message output.
* - If the group does not define a special page alias file or magic
* words file, or that file does not exist, it is ignored silently.
* - If the file does contain a data array (e.g. $aliases) then the
* program exits.
*/
protected function openHandles()
{
$this->output("Opening file handles and loading current data...\n");
$groups = MessageGroups::singleton()->getGroups();
foreach ($groups as $group) {
if (!$group instanceof MediaWikiExtensionMessageGroup) {
continue;
}
$conf = $group->getConfiguration();
$inFile = $outFile = null;
if ($this->type === 'special' && isset($conf['FILES']['aliasFile'])) {
$inFile = $conf['FILES']['aliasFileSource'];
$outFile = $conf['FILES']['aliasFile'];
}
if ($this->type === 'magic' && isset($conf['FILES']['magicFile'])) {
$inFile = $conf['FILES']['magicFileSource'];
$outFile = $conf['FILES']['magicFile'];
}
if ($inFile === null) {
continue;
}
$inFile = $group->replaceVariables($inFile, 'en');
$outFile = $this->target . '/' . $outFile;
include $inFile;
switch ($this->type) {
case 'special':
if (isset($aliases)) {
$this->messagesOld[$group->getId()] = $aliases;
unset($aliases);
} elseif (isset($specialPageAliases)) {
$this->messagesOld[$group->getId()] = $specialPageAliases;
unset($specialPageAliases);
} else {
$this->error("File '{$inFile}' does not contain an aliases array.");
continue;
}
break;
case 'magic':
if (!isset($magicWords)) {
$this->error("File '{$inFile}' does not contain a magic words array.");
continue;
}
$this->messagesOld[$group->getId()] = $magicWords;
unset($magicWords);
break;
}
wfMkdirParents(dirname($outFile), null, __METHOD__);
$this->handles[$group->getId()] = fopen($outFile, 'w');
fwrite($this->handles[$group->getId()], $this->readHeader($inFile));
$this->output("\t{$group->getId()}\n");
}
}
public static function populateStats()
{
// remove all records
$dbw = wfGetDB(DB_MASTER);
$dbw->delete('groupstats', '*');
$groups = MessageGroups::singleton()->getGroups();
// iterate over all groups
foreach ($groups as $g) {
/* @var $g MessageGroup */
echo "Populating " . $g->getId() . "...\n";
self::forGroup($g->getId());
}
echo "Done!\n";
}
public function execute() {
$this->files = array();
$groups = MessageGroups::singleton()->getGroups();
$target = $this->getOption( 'path' );
foreach ( $groups as $group ) {
if ( !$group instanceof ExtensionMessageGroup ) continue;
if ( $target && $group->getPath() !== $target ) continue;
$this->addPaths( $group->getMessageFile( 'en' ) );
$this->addPaths( $group->getAliasFile( 'en' ) );
$this->addPaths( $group->getMagicFile( 'en' ) );
}
$files = array_keys( $this->files );
$this->output( trim( implode( "\n", $files ) . "\n" ) );
}
public function execute()
{
$groups = MessageGroups::singleton()->getGroups();
/** @var MessageGroup $group */
foreach ($groups as $group) {
if (!$group instanceof WikiPageMessageGroup) {
continue;
}
// Get all translation subpages and refresh each one of them
$page = TranslatablePage::newFromTitle($group->getTitle());
$translationPages = $page->getTranslationPages();
foreach ($translationPages as $subpage) {
$job = TranslateRenderJob::newJob($subpage);
$job->run();
}
}
}
public function testTranslationPageRestrictions()
{
$superUser = new MockSuperUser();
$title = Title::newFromText('Translatable page');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('<translate>Hello</translate>', $title);
$status = $page->doEditContent($content, 'New page', 0, false, $superUser);
$revision = $status->value['revision']->getId();
$translatablePage = TranslatablePage::newFromRevision($title, $revision);
$translatablePage->addMarkedTag($revision);
MessageGroups::singleton()->recache();
$translationPage = Title::newFromText('Translatable page/fi');
TranslateRenderJob::newJob($translationPage)->run();
$this->assertTrue($translationPage->userCan('read', $superUser), 'Users can read existing translation pages');
$this->assertFalse($translationPage->userCan('edit', $superUser), 'Users can not edit existing translation pages');
$translationPage = Title::newFromText('Translatable page/ab');
$this->assertTrue($translationPage->userCan('read', $superUser), 'Users can read non-existing translation pages');
$this->assertFalse($translationPage->userCan('edit', $superUser), 'Users can not edit non-existing translation pages');
}
private function displayNavigation()
{
global $wgOut;
$groupSelector = new XmlSelect('group', 'group-select');
// pull groups
$groups = MessageGroups::singleton()->getGroups();
foreach ($groups as $group) {
if (!$group->isMeta()) {
continue;
}
$groupSelector->addOption($group->getLabel(), $group->getId());
}
$fields = array();
$fields['transstats-choose-group'] = $groupSelector->getHTML();
$fields['transstats-breakdown'] = Xml::check('breakdown', false);
$out = Xml::openElement('form');
$out .= Xml::buildForm($fields);
$out .= Xml::submitButton(wfMsg('transstats-submit'));
$out .= Xml::closeElement('form');
$wgOut->addHTML($out);
}
public function execute()
{
MessageGroups::singleton()->recache();
MessageIndex::singleton()->rebuild();
}
/**
* If the list of group ids contains wildcards, this function will match
* them against the list of all supported groups and return matched group
* names.
* @param $ids \list{String}
* @return \list{String}
*/
protected function expandWildcards($ids)
{
$hasWild = false;
foreach ($ids as $id) {
if (strpos($id, '*') !== false) {
$hasWild = true;
break;
}
}
if (!$hasWild) {
return $ids;
}
$matcher = new StringMatcher('', $ids);
$all = array();
foreach (MessageGroups::singleton()->getGroups() as $id => $_) {
if ($matcher->match($id)) {
$all[] = $id;
}
}
return $all;
}
public function execute()
{
$output = $this->getOption('output', 'default');
// Select an output engine
switch ($output) {
case 'wiki':
$out = new WikiStatsOutput();
break;
case 'text':
$out = new TextStatsOutput();
break;
case 'csv':
$out = new CsvStatsOutput();
break;
default:
$out = new TranslateStatsOutput();
}
$skipLanguages = array();
if ($this->hasOption('skiplanguages')) {
$skipLanguages = array_map('trim', explode(',', $this->getOption('skiplanguages')));
}
$reportScore = false;
// Check if score should be reported and prepare weights
$most = $this->getOption('most');
$weights = array();
if ($most && isset($this->localisedWeights[$most])) {
$reportScore = true;
foreach ($this->localisedWeights[$most] as $weight) {
$weights[] = $weight;
}
}
// check if l10n should be done
$l10n = false;
if (($output === 'wiki' || $output === 'default') && !$this->hasOption('nol10n')) {
$l10n = true;
}
$wmfscore = $this->hasOption('wmfscore');
// Get groups from input
$groups = array();
if ($reportScore) {
$reqGroups = array_keys($this->localisedWeights[$most]);
} elseif ($wmfscore) {
$reqGroups = array_keys($this->localisedWeights['wikimedia']);
} else {
$reqGroups = array_map('trim', explode(',', $this->getOption('groups')));
}
// List of all groups
$allGroups = MessageGroups::singleton()->getGroups();
// Get list of valid groups
foreach ($reqGroups as $id) {
// Page translation group ids use spaces which are not nice on command line
$id = str_replace('_', ' ', $id);
if (isset($allGroups[$id])) {
$groups[$id] = $allGroups[$id];
} else {
$this->output("Unknown group: {$id}");
}
}
if ($wmfscore) {
// Override/set parameters
$out = new CsvStatsOutput();
$reportScore = true;
$weights = array();
foreach ($this->localisedWeights['wikimedia'] as $weight) {
$weights[] = $weight;
}
$wmfscores = array();
}
if (!count($groups)) {
$this->error('No groups given', true);
}
// List of all languages.
$languages = Language::fetchLanguageNames(false);
// Default sorting order by language code, users can sort wiki output.
ksort($languages);
if ($this->hasOption('legenddetail')) {
$out->addFreeText("{{" . $this->getOption('legenddetail') . "}}\n");
}
$totalWeight = 0;
if ($reportScore) {
if ($wmfscore) {
foreach ($this->localisedWeights['wikimedia'] as $weight) {
$totalWeight += $weight;
}
} else {
foreach ($this->localisedWeights[$most] as $weight) {
$totalWeight += $weight;
}
}
}
$showContinent = $this->getOption('continent');
if (!$wmfscore) {
// Output headers
$out->heading();
$out->blockstart();
if ($most) {
$out->element($l10n ? "{{int:translate-gs-pos}}" : 'Pos.', true);
}
$out->element($l10n ? "{{int:translate-gs-code}}" : 'Code', true);
$out->element($l10n ? "{{int:translate-page-language}}" : 'Language', true);
if ($showContinent) {
$out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true);
}
if ($most && $this->hasOption('speakers')) {
$out->element($l10n ? "{{int:translate-gs-speakers}}" : 'Speakers', true);
}
if ($reportScore) {
$out->element(($l10n ? "{{int:translate-gs-score}}" : 'Score') . ' (' . $totalWeight . ')', true);
}
/**
* @var $g MessageGroup
*/
foreach ($groups as $g) {
// Add unprocessed description of group as heading
if ($reportScore) {
$gid = $g->getId();
$heading = $g->getLabel() . " (" . $this->localisedWeights[$most][$gid] . ")";
} else {
$heading = $g->getLabel();
}
$out->element($heading, true);
if (!$reportScore && $this->hasOption('fuzzy')) {
$out->element($l10n ? "{{int:translate-percentage-fuzzy}}" : 'Fuzzy', true);
}
}
$out->blockend();
}
$rows = array();
foreach ($languages as $code => $name) {
// Skip list
if (in_array($code, $skipLanguages)) {
continue;
}
$rows[$code] = array();
}
foreach ($groups as $groupName => $g) {
$stats = MessageGroupStats::forGroup($groupName);
// Perform the statistic calculations on every language
foreach ($languages as $code => $name) {
// Skip list
if (!$most && in_array($code, $skipLanguages)) {
continue;
}
// Do not calculate if we do not need it for anything.
if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') {
continue;
}
// If --most is set, skip all other
if ($most && !isset($this->mostSpokenLanguages[$code])) {
continue;
}
$total = $stats[$code][MessageGroupStats::TOTAL];
$translated = $stats[$code][MessageGroupStats::TRANSLATED];
$fuzzy = $stats[$code][MessageGroupStats::FUZZY];
$rows[$code][] = array(false, $translated, $total);
if ($this->hasOption('fuzzy')) {
$rows[$code][] = array(true, $fuzzy, $total);
}
}
unset($collection);
}
// init summary array
$summarise = false;
if ($this->hasOption('summary')) {
$summarise = true;
$summary = array();
}
foreach ($languages as $code => $name) {
// Skip list
if (!$most && in_array($code, $skipLanguages)) {
continue;
}
// Skip unneeded
if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') {
continue;
}
// If --most is set, skip all other
if ($most && !isset($this->mostSpokenLanguages[$code])) {
continue;
}
$columns = $rows[$code];
$allZero = true;
foreach ($columns as $fields) {
if (intval($fields[1]) !== 0) {
$allZero = false;
}
}
// Skip dummy languages if requested
if ($allZero && $this->hasOption('skipzero')) {
continue;
}
// Output the the row
if (!$wmfscore) {
$out->blockstart();
}
// Fill language position field
if ($most) {
$out->element($this->mostSpokenLanguages[$code][0]);
}
// Fill language name field
if (!$wmfscore) {
// Fill language code field
$out->element($code);
if ($l10n && function_exists('efI18nTagsInit')) {
$out->element("{{#languagename:" . $code . "}}");
} else {
$out->element($name);
}
}
// Fill continent field
if ($showContinent) {
if ($this->mostSpokenLanguages[$code][2] == 'multiple') {
$continent = $l10n ? "{{int:translate-gs-multiple}}" : 'Multiple';
} else {
$continent = $l10n ? "{{int:timezoneregion-" . $this->mostSpokenLanguages[$code][2] . "}}" : ucfirst($this->mostSpokenLanguages[$code][2]);
}
$out->element($continent);
}
// Fill speakers field
if ($most && $this->hasOption('speakers')) {
$out->element(number_format($this->mostSpokenLanguages[$code][1]));
}
// Fill the score field
if ($reportScore) {
// Keep count
$i = 0;
// Start with 0 points
$score = 0;
foreach ($columns as $fields) {
list(, $upper, $total) = $fields;
// Weigh the score and add it to the current score
$score += $weights[$i] * $upper / $total;
$i++;
}
// Report a round numbers
$score = number_format($score, 0);
if ($summarise) {
$continent = $this->mostSpokenLanguages[$code][2];
if (isset($summary[$continent])) {
$newcount = $summary[$continent][0] + 1;
$newscore = $summary[$continent][1] + (int) $score;
} else {
$newcount = 1;
$newscore = $score;
}
$summary[$continent] = array($newcount, $newscore);
}
if ($wmfscore) {
// Multiple variants can be used for the same wiki.
// Store the scores in an array and output them later
// when they can be averaged.
if (isset($this->wikimediaCodeMap[$code])) {
$wmfcode = $this->wikimediaCodeMap[$code];
} else {
$codeparts = explode('-', $code);
$wmfcode = $codeparts[0];
}
if (isset($wmfscores[$wmfcode])) {
$count = $wmfscores[$wmfcode]['count'] + 1;
$tmpWmfScore = (int) $wmfscores[$wmfcode]['score'];
$tmpWmfCount = (int) $wmfscores[$wmfcode]['count'];
$score = ($tmpWmfCount * $tmpWmfScore + (int) $score) / $count;
$wmfscores[$wmfcode] = array('score' => $score, 'count' => $count);
} else {
$wmfscores[$wmfcode] = array('score' => $score, 'count' => 1);
}
} else {
$out->element($score);
}
}
// Fill fields for groups
if (!$wmfscore) {
foreach ($columns as $fields) {
list($invert, $upper, $total) = $fields;
$c = $out->formatPercent($upper, $total, $invert);
$out->element($c);
}
$out->blockend();
}
}
$out->footer();
if ($reportScore && $this->hasOption('summary')) {
if ($reportScore && $this->hasOption('legendsummary')) {
$out->addFreeText("{{" . $this->getOption('legendsummary') . "}}\n");
}
$out->summaryheading();
$out->blockstart();
$out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true);
$out->element($l10n ? "{{int:translate-gs-count}}" : 'Count', true);
$out->element($l10n ? "{{int:translate-gs-avgscore}}" : 'Avg. score', true);
$out->blockend();
ksort($summary);
$totals = array(0, 0);
foreach ($summary as $key => $values) {
$out->blockstart();
if ($key == 'multiple') {
$out->element($l10n ? "{{int:translate-gs-multiple}}" : 'Multiple');
} else {
$out->element($l10n ? "{{int:timezoneregion-" . $key . "}}" : ucfirst($key));
}
$out->element($values[0]);
$out->element(number_format($values[1] / $values[0]));
$out->blockend();
$totals[0] += $values[0];
$totals[1] += $values[1];
}
$out->blockstart();
$out->element($l10n ? "{{int:translate-gs-total}}" : 'Total');
$out->element($totals[0]);
$out->element(number_format($totals[1] / $totals[0]));
$out->blockend();
$out->footer();
}
// Custom output
if ($wmfscore) {
ksort($wmfscores);
foreach ($wmfscores as $code => $stats) {
echo $code . ';' . number_format($stats['score']) . ";\n";
}
}
}
/**
* This function does the heavy duty of marking a page.
* - Updates the source page with section markers.
* - Updates translate_sections table
* - Updates revtags table
* - Setups renderjobs to update the translation pages
* - Invalidates caches
* @param TranslatablePage $page
* @param array $sections
* @return array|bool
*/
public function markForTranslation(TranslatablePage $page, array $sections)
{
// Add the section markers to the source page
$wikiPage = WikiPage::factory($page->getTitle());
$content = ContentHandler::makeContent($page->getParse()->getSourcePageText(), $page->getTitle());
$status = $wikiPage->doEditContent($content, $this->msg('tpt-mark-summary')->inContentLanguage()->text(), EDIT_FORCE_BOT | EDIT_UPDATE);
if (!$status->isOK()) {
return array('tpt-edit-failed', $status->getWikiText());
}
$newrevision = $status->value['revision'];
// In theory it is either null or Revision object,
// never revision object with null id, but who knows
if ($newrevision instanceof Revision) {
$newrevision = $newrevision->getId();
}
if ($newrevision === null) {
// Probably a no-change edit, so no new revision was assigned.
// Get the latest revision manually
$newrevision = $page->getTitle()->getLatestRevId();
}
$inserts = array();
$changed = array();
$maxid = intval(TranslateMetadata::get($page->getMessageGroupId(), 'maxid'));
$pageId = $page->getTitle()->getArticleID();
/**
* @var TPSection $s
*/
foreach (array_values($sections) as $index => $s) {
$maxid = max($maxid, intval($s->name));
$changed[] = $s->name;
if ($this->getRequest()->getCheck("tpt-sect-{$s->id}-action-nofuzzy")) {
// This will be checked by getTranslationUnitJobs
$s->type = 'old';
}
$inserts[] = array('trs_page' => $pageId, 'trs_key' => $s->name, 'trs_text' => $s->getText(), 'trs_order' => $index);
}
$dbw = wfGetDB(DB_MASTER);
$dbw->delete('translate_sections', array('trs_page' => $page->getTitle()->getArticleID()), __METHOD__);
$dbw->insert('translate_sections', $inserts, __METHOD__);
TranslateMetadata::set($page->getMessageGroupId(), 'maxid', $maxid);
$page->addMarkedTag($newrevision);
MessageGroups::singleton()->recache();
$jobs = self::getRenderJobs($page);
JobQueueGroup::singleton()->push($jobs);
$jobs = self::getTranslationUnitJobs($page, $sections);
JobQueueGroup::singleton()->push($jobs);
// Logging
$this->handlePriorityLanguages($this->getRequest(), $page);
$entry = new ManualLogEntry('pagetranslation', 'mark');
$entry->setPerformer($this->getUser());
$entry->setTarget($page->getTitle());
$entry->setParameters(array('revision' => $newrevision, 'changed' => count($changed)));
$logid = $entry->insert();
$entry->publish($logid);
// Clear more caches
$page->getTitle()->invalidateCache();
MessageIndexRebuildJob::newJob()->insert();
return false;
}
protected function performAction()
{
$jobs = array();
$target = $this->title;
$base = $this->title->getPrefixedText();
$translationPages = $this->getTranslationPages();
$user = $this->getUser();
foreach ($translationPages as $old) {
$jobs[$old->getPrefixedText()] = TranslateDeleteJob::newJob($old, $base, !$this->singleLanguage(), $user);
}
$sectionPages = $this->getSectionPages();
foreach ($sectionPages as $old) {
$jobs[$old->getPrefixedText()] = TranslateDeleteJob::newJob($old, $base, !$this->singleLanguage(), $user);
}
if (!$this->doSubpages) {
$subpages = $this->getSubpages();
foreach ($subpages as $old) {
if (TranslatablePage::isTranslationPage($old)) {
continue;
}
$jobs[$old->getPrefixedText()] = TranslateDeleteJob::newJob($old, $base, !$this->singleLanguage(), $user);
}
}
JobQueueGroup::singleton()->push($jobs);
$cache = wfGetCache(CACHE_DB);
$cache->set(wfMemcKey('pt-base', $target->getPrefixedText()), array_keys($jobs), 60 * 60 * 6);
if (!$this->singleLanguage()) {
$this->page->unmarkTranslatablePage();
}
$this->clearMetadata();
MessageGroups::singleton()->recache();
MessageIndexRebuildJob::newJob()->insert();
$this->getOutput()->addWikiMsg('pt-deletepage-started');
}
function checkThreshold( $group, $languages, $threshold ) {
if ( $threshold === false ) {
return $languages;
}
$qualify = array();
$g = MessageGroups::singleton()->getGroup( $group );
$collection = $g->initCollection( 'en' );
foreach ( $languages as $code ) {
$collection->resetForNewLanguage( $code );
// Initialise messages
$collection->filter( 'ignored' );
$collection->filter( 'optional' );
// Store the count of real messages for later calculation.
$total = count( $collection );
$collection->filter( 'translated', false );
$translated = count( $collection );
if ( $translated / $total > $threshold / 100 ) {
$qualify[] = $code;
}
}
return $qualify;
}
// @todo FIXME: Code duplication with sync-group.php
if (isset($options['group'])) {
// Explode parameter
$groupIds = explode(',', trim($options['group']));
// Get groups and add groups to array
foreach ($groupIds as $groupId) {
$group = MessageGroups::getGroup($groupId);
if ($group !== null) {
$groups[$groupId] = $group;
} else {
STDERR("Invalid group {$groupId}");
}
}
} else {
// Apparently using option groupprefix. Find groups that match.
$allGroups = MessageGroups::singleton()->getGroups();
// Add matching groups to groups array.
foreach ($allGroups as $groupId => $messageGroup) {
if (strpos($groupId, $options['groupprefix']) === 0 && !$messageGroup->isMeta()) {
$groups[$groupId] = $messageGroup;
}
}
}
foreach ($groups as $groupId => $group) {
if (!$group instanceof MessageGroup) {
STDERR("Invalid group: " . $groupId);
exit(1);
}
STDERR('Exporting ' . $groupId);
$langs = $reqLangs;
if ($threshold) {
public function __construct( $code ) {
parent::__construct( $code );
$this->data['core'] = array(
'label' => 'MediaWiki Core',
'var' => 'magicWords',
'file' => Language::getMessagesFileName( '%CODE%' ),
'code' => false,
);
global $wgTranslateExtensionDirectory;
$groups = MessageGroups::singleton()->getGroups();
foreach ( $groups as $g ) {
if ( !$g instanceof ExtensionMessageGroup ) {
continue;
}
$file = $g->getMagicFile();
if ( $file === null ) {
continue;
}
$file = "$wgTranslateExtensionDirectory/$file";
if ( file_exists( $file ) ) {
$this->data[$g->getId()] = array(
'label' => $g->getLabel(),
'var' => 'magicWords',
'file' => $file,
'code' => $code,
);
}
}
}
function getGroups()
{
$groups = wfMsgForContent('translate-languagestats-groups');
if ($groups) {
// Make the group names clean
// Should contain one valid group name per line
// All invalid group names should be ignored
// Return all group names if there are no valid group names at all
// FIXME: implement the above here
$cleanGroups = '';
if ($cleanGroups) {
return $cleanGroups;
}
}
return MessageGroups::singleton()->getGroups();
}
protected static function forLanguageInternal( $code, $stats = array() ) {
$res = self::selectRowsIdLang( null, $code );
$stats = self::extractResults( $res, $stats );
$groups = MessageGroups::singleton()->getGroups();
foreach ( $groups as $id => $group ) {
if ( isset( $stats[$id][$code] ) ) continue;
$stats[$id][$code] = self::forItemInternal( $stats, $group, $code );
}
return $stats;
}
/**
* Constructs a JavaScript enhanced group selector.
* @return JsSelectToInput
*/
protected function groupSelector() {
$groups = MessageGroups::singleton()->getGroups();
foreach ( $groups as $key => $group ) {
if ( !$group->exists() ) {
unset( $groups[$key] );
continue;
}
}
ksort( $groups );
$selector = new XmlSelect( 'mw-group-selector', 'mw-group-selector' );
foreach ( $groups as $code => $name ) {
$selector->addOption( $name->getLabel(), $code );
}
$jsSelect = new JsSelectToInput( $selector );
$jsSelect->setSourceId( 'mw-group-selector' );
return $jsSelect;
}
public function testSearchableTTMServer()
{
global $wgTranslateTestTTMServer;
if (!$wgTranslateTestTTMServer) {
$this->markTestSkipped('No test TTMServer available');
}
$server = TTMServer::factory($wgTranslateTestTTMServer);
$solarium = $server->getSolarium();
// Empty it
$update = $solarium->createUpdate();
$update->addDeleteQuery('*:*');
$update->addCommit();
$solarium->update($update);
// Check that it is empty indeed
$select = $solarium->createSelect();
$select->setRows(0);
$select->setQuery('*:*');
$result = $solarium->select($select);
$this->assertEquals(0, $result->getNumFound());
// Add one definition
$title = Title::newFromText('MediaWiki:one/en');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('1', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setRows(1);
$select->setQuery('*:*');
$result = $solarium->select($select);
$this->assertEquals(1, $result->getNumFound());
$doc = null;
foreach ($result as $doc) {
// @todo FIXME Empty foreach statement.
}
$this->assertEquals(wfWikiId(), $doc->wiki);
$this->assertEquals('en', $doc->language);
$this->assertEquals('1', $doc->content);
$this->assertEquals(array('ttmserver-test'), $doc->group);
// Add one translation
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('yksi', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setRows(1);
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(1, $result->getNumFound());
$doc = null;
foreach ($result as $doc) {
// @todo FIXME Empty foreach statement.
}
$this->assertEquals('yksi', $doc->content);
$this->assertEquals(array('ttmserver-test'), $doc->group);
// Update definition
$title = Title::newFromText('MediaWiki:one/en');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('1-updated', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:en');
$result = $solarium->select($select);
$this->assertEquals(2, $result->getNumFound(), 'Old and new definition exists');
// Translation is fuzzied
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('!!FUZZY!!yksi', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(0, $result->getNumFound());
// Translation is udpated
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('yksi-päiv', $title);
$page->doEditContent($content, __METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(1, $result->getNumFound());
$doc = null;
foreach ($result as $doc) {
// @todo FIXME Empty foreach statement.
}
$this->assertEquals('yksi-päiv', $doc->content);
// And now the messages should be orphaned
global $wgHooks;
$wgHooks['TranslatePostInitGroups'] = array();
MessageGroups::singleton()->recache();
MessageIndex::singleton()->rebuild();
self::runJobs();
$select = $solarium->createSelect();
$select->setQuery('*:*');
$result = $solarium->select($select);
$this->assertEquals(2, $result->getNumFound(), 'One definition and one translation exists');
foreach ($result as $doc) {
$this->assertEquals(null, $doc->group, 'Messages are orphaned');
}
// And message deletion
$title = Title::newFromText('MediaWiki:one/fi');
$page = WikiPage::factory($title);
$page->doDeleteArticle(__METHOD__);
$select = $solarium->createSelect();
$select->setQuery('language:fi');
$result = $solarium->select($select);
$this->assertEquals(0, $result->getNumFound());
}
exit( 1 );
}
if ( isset( $options['help'] ) ) {
showUsage();
}
if ( !isset( $options['group'] ) ) {
STDERR( "ESG1: Message group id must be supplied with group parameter." );
exit( 1 );
}
$group = MessageGroups::getGroup( $options['group'] );
if ( $group === null ) {
if ( $options['group'] === '*' ) {
$mg = MessageGroups::singleton();
$groups = $mg->getGroups();
} else {
STDERR( "ESG2: Invalid message group was given." );
exit( 1 );
}
} else {
$groups = array( $group );
}
if ( !isset( $options['lang'] ) || strval( $options['lang'] ) === '' ) {
STDERR( "ESG3: List of language codes must be supplied with lang parameter." );
exit( 1 );
}
$start = isset( $options['start'] ) ? strtotime( $options['start'] ) : false;
public function __construct($code)
{
parent::__construct($code);
$this->data['core'] = array('label' => 'MediaWiki Core', 'var' => 'magicWords', 'file' => Language::getMessagesFileName(self::PLACEHOLDER), 'code' => false);
$groups = MessageGroups::singleton()->getGroups();
foreach ($groups as $g) {
if (!$g instanceof MediaWikiExtensionMessageGroup) {
continue;
}
$conf = $g->getConfiguration();
if (!isset($conf['FILES']['magicFileSource'])) {
continue;
}
$file = $g->replaceVariables($conf['FILES']['magicFileSource'], 'en');
if (file_exists($file)) {
$this->data[$g->getId()] = array('label' => $g->getLabel(), 'var' => 'magicWords', 'file' => $file, 'code' => $code);
}
}
}
