protected function getData()
{
$params = $this->extractRequestParams();
$group = MessageGroups::getGroup($params['group']);
if (!$group) {
$this->dieUsageMsg(array('missingparam', 'mcgroup'));
} elseif (MessageGroups::isDynamic($group)) {
$this->dieUsage('Dynamic message groups are not supported here', 'invalidparam');
}
return MessageGroupStats::forGroup($group->getId());
}
public function execute()
{
$params = $this->extractRequestParams();
$group = MessageGroups::getGroup($params['group']);
if (!$group) {
$this->dieUsageMsg(array('missingparam', 'mcgroup'));
}
MessageGroupStats::setTimeLimit($params['timelimit']);
$cache = MessageGroupStats::forGroup($group->getId());
$result = $this->getResult();
foreach ($cache as $code => $stats) {
if ($code < $params['offset']) {
continue;
}
list($total, $translated, $fuzzy) = $stats;
if ($total === null) {
$this->setContinueEnumParameter('offset', $code);
break;
}
$data = array('code' => $code, 'total' => $total, 'translated' => $translated, 'fuzzy' => $fuzzy);
$result->addValue(array('query', $this->getModuleName()), null, $data);
}
$result->setIndexedTagName_internal(array('query', $this->getModuleName()), 'messagegroupstats');
}
public function execute()
{
$target = $this->getOption('target');
if (!is_writable($target)) {
$this->error("Target directory is not writable ({$target}).", 1);
}
$threshold = $this->getOption('threshold');
$noFuzzy = $this->hasOption('no-fuzzy');
$noLocation = '';
if ($this->hasOption('no-location')) {
$noLocation = '--no-location ';
}
$skip = array();
if ($this->hasOption('skip')) {
$skip = array_map('trim', explode(',', $this->getOption('skip')));
}
$reqLangs = TranslateUtils::parseLanguageCodes($this->getOption('lang'));
$reqLangs = array_flip($reqLangs);
foreach ($skip as $skipLang) {
unset($reqLangs[$skipLang]);
}
$reqLangs = array_flip($reqLangs);
$codemapOnly = $this->hasOption('codemaponly');
$groupIds = explode(',', trim($this->getOption('group')));
$groupIds = MessageGroups::expandWildcards($groupIds);
$groups = MessageGroups::getGroupsById($groupIds);
/** @var FileBasedMessageGroup $group */
foreach ($groups as $groupId => $group) {
if ($group->isMeta()) {
$this->output("Skipping meta message group {$groupId}.\n");
unset($groups[$groupId]);
continue;
}
if (!$group instanceof FileBasedMessageGroup) {
$this->output("EE2: Unexportable message group {$groupId}.\n");
unset($groups[$groupId]);
continue;
}
}
if (!count($groups)) {
$this->error("EE1: No valid message groups identified.", 1);
}
$changeFilter = false;
$hours = $this->getOption('hours');
if ($hours) {
$namespaces = array();
/** @var FileBasedMessageGroup $group */
foreach ($groups as $group) {
$namespaces[$group->getNamespace()] = true;
}
$namespaces = array_keys($namespaces);
$bots = true;
$changeFilter = array();
$rows = TranslateUtils::translationChanges($hours, $bots, $namespaces);
foreach ($rows as $row) {
$title = Title::makeTitle($row->rc_namespace, $row->rc_title);
$handle = new MessageHandle($title);
$code = $handle->getCode();
if (!$code) {
continue;
}
$groupIds = $handle->getGroupIds();
foreach ($groupIds as $groupId) {
$changeFilter[$groupId][$code] = true;
}
}
}
$skipGroups = array();
if ($this->hasOption('skipgroup')) {
$skipGroups = array_map('trim', explode(',', $this->getOption('skipgroup')));
}
foreach ($groups as $groupId => $group) {
if (in_array($groupId, $skipGroups)) {
$this->output("Group {$groupId} is in skipgroup.\n");
continue;
}
// No changes to this group at all
if (is_array($changeFilter) && !isset($changeFilter[$groupId])) {
$this->output("No recent changes to {$groupId}.\n");
continue;
}
$langs = $reqLangs;
if ($codemapOnly) {
foreach ($langs as $index => $code) {
if ($group->mapCode($code) === $code) {
unset($langs[$index]);
}
}
}
if ($threshold) {
$stats = MessageGroupStats::forGroup($groupId);
foreach ($langs as $index => $code) {
if (!isset($stats[$code])) {
unset($langs[$index]);
continue;
}
$total = $stats[$code][MessageGroupStats::TOTAL];
$translated = $stats[$code][MessageGroupStats::TRANSLATED];
if ($translated / $total * 100 < $threshold) {
unset($langs[$index]);
}
}
}
// Filter out unchanged languages from requested languages
if (is_array($changeFilter)) {
$langs = array_intersect($langs, array_keys($changeFilter[$groupId]));
}
if (!count($langs)) {
continue;
}
$this->output("Exporting {$groupId}...\n");
$ffs = $group->getFFS();
$ffs->setWritePath($target);
$sourceLanguage = $group->getSourceLanguage();
$collection = $group->initCollection($sourceLanguage);
$definitionFile = false;
if ($this->hasOption('ppgettext') && $ffs instanceof GettextFFS) {
global $wgMaxShellMemory, $wgTranslateGroupRoot;
// Need more shell memory for msgmerge.
$wgMaxShellMemory = 402400;
$path = $group->getSourceFilePath($sourceLanguage);
$definitionFile = str_replace($wgTranslateGroupRoot, $this->getOption('ppgettext'), $path);
}
$whitelist = $group->getTranslatableLanguages();
foreach ($langs as $lang) {
// Do not export languges that are blacklisted (or not whitelisted).
// Also check that whitelist is not null, which means that all
// languages are allowed for translation and export.
if (is_array($whitelist) && !isset($whitelist[$lang])) {
continue;
}
$collection->resetForNewLanguage($lang);
$collection->loadTranslations();
// Don't export ignored, unless it is the source language
// or message documentation
global $wgTranslateDocumentationLanguageCode;
if ($lang !== $wgTranslateDocumentationLanguageCode && $lang !== $sourceLanguage) {
$collection->filter('ignored');
}
if ($noFuzzy) {
$collection->filter('fuzzy');
}
$ffs->write($collection);
// Do post processing if requested.
if ($definitionFile) {
if (is_file($definitionFile)) {
$targetFileName = $ffs->getWritePath() . "/" . $group->getTargetFilename($collection->code);
$cmd = "msgmerge --quiet " . $noLocation . "--output-file=" . $targetFileName . ' ' . $targetFileName . ' ' . $definitionFile;
wfShellExec($cmd, $ret);
// Report on errors.
if ($ret) {
$this->error("ERROR: {$ret}");
}
} else {
$this->error("{$definitionFile} does not exist.", 1);
}
}
}
}
}
public function execute()
{
$output = $this->getOption('output', 'default');
// Select an output engine
switch ($output) {
case 'wiki':
$out = new WikiStatsOutput();
break;
case 'text':
$out = new TextStatsOutput();
break;
case 'csv':
$out = new CsvStatsOutput();
break;
default:
$out = new TranslateStatsOutput();
}
$skipLanguages = array();
if ($this->hasOption('skiplanguages')) {
$skipLanguages = array_map('trim', explode(',', $this->getOption('skiplanguages')));
}
$reportScore = false;
// Check if score should be reported and prepare weights
$most = $this->getOption('most');
$weights = array();
if ($most && isset($this->localisedWeights[$most])) {
$reportScore = true;
foreach ($this->localisedWeights[$most] as $weight) {
$weights[] = $weight;
}
}
// check if l10n should be done
$l10n = false;
if (($output === 'wiki' || $output === 'default') && !$this->hasOption('nol10n')) {
$l10n = true;
}
$wmfscore = $this->hasOption('wmfscore');
// Get groups from input
$groups = array();
if ($reportScore) {
$reqGroups = array_keys($this->localisedWeights[$most]);
} elseif ($wmfscore) {
$reqGroups = array_keys($this->localisedWeights['wikimedia']);
} else {
$reqGroups = array_map('trim', explode(',', $this->getOption('groups')));
}
// List of all groups
$allGroups = MessageGroups::singleton()->getGroups();
// Get list of valid groups
foreach ($reqGroups as $id) {
// Page translation group ids use spaces which are not nice on command line
$id = str_replace('_', ' ', $id);
if (isset($allGroups[$id])) {
$groups[$id] = $allGroups[$id];
} else {
$this->output("Unknown group: {$id}");
}
}
if ($wmfscore) {
// Override/set parameters
$out = new CsvStatsOutput();
$reportScore = true;
$weights = array();
foreach ($this->localisedWeights['wikimedia'] as $weight) {
$weights[] = $weight;
}
$wmfscores = array();
}
if (!count($groups)) {
$this->error('No groups given', true);
}
// List of all languages.
$languages = Language::fetchLanguageNames(false);
// Default sorting order by language code, users can sort wiki output.
ksort($languages);
if ($this->hasOption('legenddetail')) {
$out->addFreeText("{{" . $this->getOption('legenddetail') . "}}\n");
}
$totalWeight = 0;
if ($reportScore) {
if ($wmfscore) {
foreach ($this->localisedWeights['wikimedia'] as $weight) {
$totalWeight += $weight;
}
} else {
foreach ($this->localisedWeights[$most] as $weight) {
$totalWeight += $weight;
}
}
}
$showContinent = $this->getOption('continent');
if (!$wmfscore) {
// Output headers
$out->heading();
$out->blockstart();
if ($most) {
$out->element($l10n ? "{{int:translate-gs-pos}}" : 'Pos.', true);
}
$out->element($l10n ? "{{int:translate-gs-code}}" : 'Code', true);
$out->element($l10n ? "{{int:translate-page-language}}" : 'Language', true);
if ($showContinent) {
$out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true);
}
if ($most && $this->hasOption('speakers')) {
$out->element($l10n ? "{{int:translate-gs-speakers}}" : 'Speakers', true);
}
if ($reportScore) {
$out->element(($l10n ? "{{int:translate-gs-score}}" : 'Score') . ' (' . $totalWeight . ')', true);
}
/**
* @var $g MessageGroup
*/
foreach ($groups as $g) {
// Add unprocessed description of group as heading
if ($reportScore) {
$gid = $g->getId();
$heading = $g->getLabel() . " (" . $this->localisedWeights[$most][$gid] . ")";
} else {
$heading = $g->getLabel();
}
$out->element($heading, true);
if (!$reportScore && $this->hasOption('fuzzy')) {
$out->element($l10n ? "{{int:translate-percentage-fuzzy}}" : 'Fuzzy', true);
}
}
$out->blockend();
}
$rows = array();
foreach ($languages as $code => $name) {
// Skip list
if (in_array($code, $skipLanguages)) {
continue;
}
$rows[$code] = array();
}
foreach ($groups as $groupName => $g) {
$stats = MessageGroupStats::forGroup($groupName);
// Perform the statistic calculations on every language
foreach ($languages as $code => $name) {
// Skip list
if (!$most && in_array($code, $skipLanguages)) {
continue;
}
// Do not calculate if we do not need it for anything.
if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') {
continue;
}
// If --most is set, skip all other
if ($most && !isset($this->mostSpokenLanguages[$code])) {
continue;
}
$total = $stats[$code][MessageGroupStats::TOTAL];
$translated = $stats[$code][MessageGroupStats::TRANSLATED];
$fuzzy = $stats[$code][MessageGroupStats::FUZZY];
$rows[$code][] = array(false, $translated, $total);
if ($this->hasOption('fuzzy')) {
$rows[$code][] = array(true, $fuzzy, $total);
}
}
unset($collection);
}
// init summary array
$summarise = false;
if ($this->hasOption('summary')) {
$summarise = true;
$summary = array();
}
foreach ($languages as $code => $name) {
// Skip list
if (!$most && in_array($code, $skipLanguages)) {
continue;
}
// Skip unneeded
if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') {
continue;
}
// If --most is set, skip all other
if ($most && !isset($this->mostSpokenLanguages[$code])) {
continue;
}
$columns = $rows[$code];
$allZero = true;
foreach ($columns as $fields) {
if (intval($fields[1]) !== 0) {
$allZero = false;
}
}
// Skip dummy languages if requested
if ($allZero && $this->hasOption('skipzero')) {
continue;
}
// Output the the row
if (!$wmfscore) {
$out->blockstart();
}
// Fill language position field
if ($most) {
$out->element($this->mostSpokenLanguages[$code][0]);
}
// Fill language name field
if (!$wmfscore) {
// Fill language code field
$out->element($code);
if ($l10n && function_exists('efI18nTagsInit')) {
$out->element("{{#languagename:" . $code . "}}");
} else {
$out->element($name);
}
}
// Fill continent field
if ($showContinent) {
if ($this->mostSpokenLanguages[$code][2] == 'multiple') {
$continent = $l10n ? "{{int:translate-gs-multiple}}" : 'Multiple';
} else {
$continent = $l10n ? "{{int:timezoneregion-" . $this->mostSpokenLanguages[$code][2] . "}}" : ucfirst($this->mostSpokenLanguages[$code][2]);
}
$out->element($continent);
}
// Fill speakers field
if ($most && $this->hasOption('speakers')) {
$out->element(number_format($this->mostSpokenLanguages[$code][1]));
}
// Fill the score field
if ($reportScore) {
// Keep count
$i = 0;
// Start with 0 points
$score = 0;
foreach ($columns as $fields) {
list(, $upper, $total) = $fields;
// Weigh the score and add it to the current score
$score += $weights[$i] * $upper / $total;
$i++;
}
// Report a round numbers
$score = number_format($score, 0);
if ($summarise) {
$continent = $this->mostSpokenLanguages[$code][2];
if (isset($summary[$continent])) {
$newcount = $summary[$continent][0] + 1;
$newscore = $summary[$continent][1] + (int) $score;
} else {
$newcount = 1;
$newscore = $score;
}
$summary[$continent] = array($newcount, $newscore);
}
if ($wmfscore) {
// Multiple variants can be used for the same wiki.
// Store the scores in an array and output them later
// when they can be averaged.
if (isset($this->wikimediaCodeMap[$code])) {
$wmfcode = $this->wikimediaCodeMap[$code];
} else {
$codeparts = explode('-', $code);
$wmfcode = $codeparts[0];
}
if (isset($wmfscores[$wmfcode])) {
$count = $wmfscores[$wmfcode]['count'] + 1;
$tmpWmfScore = (int) $wmfscores[$wmfcode]['score'];
$tmpWmfCount = (int) $wmfscores[$wmfcode]['count'];
$score = ($tmpWmfCount * $tmpWmfScore + (int) $score) / $count;
$wmfscores[$wmfcode] = array('score' => $score, 'count' => $count);
} else {
$wmfscores[$wmfcode] = array('score' => $score, 'count' => 1);
}
} else {
$out->element($score);
}
}
// Fill fields for groups
if (!$wmfscore) {
foreach ($columns as $fields) {
list($invert, $upper, $total) = $fields;
$c = $out->formatPercent($upper, $total, $invert);
$out->element($c);
}
$out->blockend();
}
}
$out->footer();
if ($reportScore && $this->hasOption('summary')) {
if ($reportScore && $this->hasOption('legendsummary')) {
$out->addFreeText("{{" . $this->getOption('legendsummary') . "}}\n");
}
$out->summaryheading();
$out->blockstart();
$out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true);
$out->element($l10n ? "{{int:translate-gs-count}}" : 'Count', true);
$out->element($l10n ? "{{int:translate-gs-avgscore}}" : 'Avg. score', true);
$out->blockend();
ksort($summary);
$totals = array(0, 0);
foreach ($summary as $key => $values) {
$out->blockstart();
if ($key == 'multiple') {
$out->element($l10n ? "{{int:translate-gs-multiple}}" : 'Multiple');
} else {
$out->element($l10n ? "{{int:timezoneregion-" . $key . "}}" : ucfirst($key));
}
$out->element($values[0]);
$out->element(number_format($values[1] / $values[0]));
$out->blockend();
$totals[0] += $values[0];
$totals[1] += $values[1];
}
$out->blockstart();
$out->element($l10n ? "{{int:translate-gs-total}}" : 'Total');
$out->element($totals[0]);
$out->element(number_format($totals[1] / $totals[0]));
$out->blockend();
$out->footer();
}
// Custom output
if ($wmfscore) {
ksort($wmfscores);
foreach ($wmfscores as $code => $stats) {
echo $code . ';' . number_format($stats['score']) . ";\n";
}
}
}
// Add matching groups to groups array.
foreach ($allGroups as $groupId => $messageGroup) {
if (strpos($groupId, $options['groupprefix']) === 0 && !$messageGroup->isMeta()) {
$groups[$groupId] = $messageGroup;
}
}
}
foreach ($groups as $groupId => $group) {
if (!$group instanceof MessageGroup) {
STDERR("Invalid group: " . $groupId);
exit(1);
}
STDERR('Exporting ' . $groupId);
$langs = $reqLangs;
if ($threshold) {
$stats = MessageGroupStats::forGroup($groupId);
foreach ($langs as $index => $code) {
if (!isset($stats[$code])) {
unset($langs[$index]);
}
list($total, $translated, ) = $stats[$code];
if ($translated / $total * 100 < $threshold) {
unset($langs[$index]);
}
}
}
if ($group instanceof FileBasedMessageGroup) {
$ffs = $group->getFFS();
$ffs->setWritePath($options['target']);
$collection = $group->initCollection('en');
$definitionFile = false;
/**
*
* @return array
*/
public function getTranslationPercentages()
{
// Calculate percentages for the available translations
$group = $this->getMessageGroup();
if (!$group instanceof WikiPageMessageGroup) {
return array();
}
$titles = $this->getTranslationPages();
$temp = MessageGroupStats::forGroup($this->getMessageGroupId());
$stats = array();
foreach ($titles as $t) {
$handle = new MessageHandle($t);
$code = $handle->getCode();
// Sometimes we want to display 0.00 for pages for which translation
// hasn't started yet.
$stats[$code] = 0.0;
if (isset($temp[$code]) && $temp[$code][MessageGroupStats::TOTAL] > 0) {
$total = $temp[$code][MessageGroupStats::TOTAL];
$translated = $temp[$code][MessageGroupStats::TRANSLATED];
$percentage = $translated / $total;
$stats[$code] = sprintf('%.2f', $percentage);
}
}
// Content language is always up-to-date
$stats[$this->getSourceLanguageCode()] = 1.0;
return $stats;
}
/**
* Overwriten from SpecialLanguageStats
*
* @return string
*/
function getTable()
{
$table = $this->table;
$this->addWorkflowStatesColumn();
$out = '';
if ($this->purge) {
MessageGroupStats::clearGroup($this->target);
}
MessageGroupStats::setTimeLimit($this->timelimit);
$cache = MessageGroupStats::forGroup($this->target);
$languages = array_keys(TranslateUtils::getLanguageNames($this->getLanguage()->getCode()));
sort($languages);
$this->filterPriorityLangs($languages, $this->target, $cache);
foreach ($languages as $code) {
if ($table->isBlacklisted($this->target, $code) !== null) {
continue;
}
$out .= $this->makeRow($code, $cache);
}
if ($out) {
$table->setMainColumnHeader($this->msg('translate-mgs-column-language'));
$out = $table->createHeader() . "\n" . $out;
$out .= Html::closeElement('tbody');
$out .= Html::openElement('tfoot');
$out .= $table->makeTotalRow($this->msg('translate-mgs-totals'), $this->totals);
$out .= Html::closeElement('tfoot');
$out .= Html::closeElement('table');
return $out;
} else {
$this->nothing = true;
return '';
}
}
protected function exportGroup(MessageGroup $group, $multi = false)
{
// Make sure all existing connections are dead,
// we can't use them in forked children.
LBFactory::destroyInstance();
$server = TTMServer::primary();
$id = $group->getId();
$sourceLanguage = $group->getSourceLanguage();
if ($multi) {
$stats = MessageGroupStats::forGroup($id);
$this->statusLine("Loaded stats for {$id}\n");
} else {
$this->statusLine("Loading stats... ", 4);
$stats = MessageGroupStats::forGroup($id);
$this->output("done!", 4);
$this->statusLine("Inserting sources: ", 5);
}
$collection = $group->initCollection($sourceLanguage);
$collection->filter('ignored');
$collection->filter('optional');
$collection->initMessages();
$sids = array();
$counter = 0;
foreach ($collection->keys() as $mkey => $title) {
$def = $collection[$mkey]->definition();
$sids[$mkey] = $server->insertSource($title, $sourceLanguage, $def);
if (++$counter % $this->mBatchSize === 0 && !$multi) {
wfWaitForSlaves(10);
$this->output('.', 5);
}
}
$total = count($sids);
if ($multi) {
$this->statusLine("Inserted {$total} source entries for {$id}\n");
} else {
$this->output("{$total} entries", 5);
$this->statusLine("Inserting translations...", 6);
}
$dbw = $server->getDB(DB_MASTER);
foreach ($stats as $targetLanguage => $numbers) {
if ($targetLanguage === $sourceLanguage) {
continue;
}
if ($numbers[MessageGroupStats::TRANSLATED] === 0) {
continue;
}
if (!$multi) {
$this->output(sprintf("%19s ", $targetLanguage), $targetLanguage);
}
$collection->resetForNewLanguage($targetLanguage);
$collection->filter('ignored');
$collection->filter('optional');
$collection->filter('translated', false);
$collection->loadTranslations();
$inserts = array();
foreach ($collection->keys() as $mkey => $title) {
$inserts[] = array('tmt_sid' => $sids[$mkey], 'tmt_lang' => $targetLanguage, 'tmt_text' => $collection[$mkey]->translation());
}
do {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$dbw->insert('translate_tmt', $batch, __METHOD__);
if (!$multi) {
$this->output('.', $targetLanguage);
}
wfWaitForSlaves(10);
} while (count($inserts));
}
if ($multi) {
$this->statusLine("Inserted translations for {$id}\n");
}
}
protected function exportGroup(MessageGroup $group, $config)
{
$server = TTMServer::factory($config);
$server->setLogger($this);
$id = $group->getId();
$sourceLanguage = $group->getSourceLanguage();
$stats = MessageGroupStats::forGroup($id);
$collection = $group->initCollection($sourceLanguage);
$collection->filter('ignored');
$collection->initMessages();
$server->beginBatch();
$inserts = array();
foreach ($collection->keys() as $mkey => $title) {
$handle = new MessageHandle($title);
$inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->definition());
}
while ($inserts !== array()) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertDefinitions($batch);
}
$inserts = array();
foreach ($stats as $targetLanguage => $numbers) {
if ($targetLanguage === $sourceLanguage) {
continue;
}
if ($numbers[MessageGroupStats::TRANSLATED] === 0) {
continue;
}
$collection->resetForNewLanguage($targetLanguage);
$collection->filter('ignored');
$collection->filter('translated', false);
$collection->loadTranslations();
foreach ($collection->keys() as $mkey => $title) {
$handle = new MessageHandle($title);
$inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->translation());
}
while (count($inserts) >= $this->mBatchSize) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertTranslations($batch);
}
}
while ($inserts !== array()) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertTranslations($batch);
}
$server->endBatch();
}
$out->element( ( $l10n ? "{{int:translate-percentage-fuzzy}}" : 'Fuzzy' ), true );
}
}
$out->blockend();
}
$rows = array();
foreach ( $languages as $code => $name ) {
// Skip list
if ( in_array( $code, $skipLanguages ) ) continue;
$rows[$code] = array();
}
foreach ( $groups as $groupName => $g ) {
$stats = MessageGroupStats::forGroup( $groupName );
// Perform the statistic calculations on every language
foreach ( $languages as $code => $name ) {
// Skip list
if ( !isset( $options['most'] ) && in_array( $code, $skipLanguages ) ) {
continue;
}
// Do not calculate if we do not need it for anything.
if ( $wmfscore && isset( $wikimediaCodeMap[$code] ) && $wikimediaCodeMap[$code] == '' ) {
continue;
}
// If --most is set, skip all other
if ( isset( $options['most'] ) && !isset( $mostSpokenLanguages[$code] ) ) {
