protected function getData() { $params = $this->extractRequestParams(); $group = MessageGroups::getGroup($params['group']); if (!$group) { $this->dieUsageMsg(array('missingparam', 'mcgroup')); } elseif (MessageGroups::isDynamic($group)) { $this->dieUsage('Dynamic message groups are not supported here', 'invalidparam'); } return MessageGroupStats::forGroup($group->getId()); }
public function execute() { $params = $this->extractRequestParams(); $group = MessageGroups::getGroup($params['group']); if (!$group) { $this->dieUsageMsg(array('missingparam', 'mcgroup')); } MessageGroupStats::setTimeLimit($params['timelimit']); $cache = MessageGroupStats::forGroup($group->getId()); $result = $this->getResult(); foreach ($cache as $code => $stats) { if ($code < $params['offset']) { continue; } list($total, $translated, $fuzzy) = $stats; if ($total === null) { $this->setContinueEnumParameter('offset', $code); break; } $data = array('code' => $code, 'total' => $total, 'translated' => $translated, 'fuzzy' => $fuzzy); $result->addValue(array('query', $this->getModuleName()), null, $data); } $result->setIndexedTagName_internal(array('query', $this->getModuleName()), 'messagegroupstats'); }
public function execute() { $target = $this->getOption('target'); if (!is_writable($target)) { $this->error("Target directory is not writable ({$target}).", 1); } $threshold = $this->getOption('threshold'); $noFuzzy = $this->hasOption('no-fuzzy'); $noLocation = ''; if ($this->hasOption('no-location')) { $noLocation = '--no-location '; } $skip = array(); if ($this->hasOption('skip')) { $skip = array_map('trim', explode(',', $this->getOption('skip'))); } $reqLangs = TranslateUtils::parseLanguageCodes($this->getOption('lang')); $reqLangs = array_flip($reqLangs); foreach ($skip as $skipLang) { unset($reqLangs[$skipLang]); } $reqLangs = array_flip($reqLangs); $codemapOnly = $this->hasOption('codemaponly'); $groupIds = explode(',', trim($this->getOption('group'))); $groupIds = MessageGroups::expandWildcards($groupIds); $groups = MessageGroups::getGroupsById($groupIds); /** @var FileBasedMessageGroup $group */ foreach ($groups as $groupId => $group) { if ($group->isMeta()) { $this->output("Skipping meta message group {$groupId}.\n"); unset($groups[$groupId]); continue; } if (!$group instanceof FileBasedMessageGroup) { $this->output("EE2: Unexportable message group {$groupId}.\n"); unset($groups[$groupId]); continue; } } if (!count($groups)) { $this->error("EE1: No valid message groups identified.", 1); } $changeFilter = false; $hours = $this->getOption('hours'); if ($hours) { $namespaces = array(); /** @var FileBasedMessageGroup $group */ foreach ($groups as $group) { $namespaces[$group->getNamespace()] = true; } $namespaces = array_keys($namespaces); $bots = true; $changeFilter = array(); $rows = TranslateUtils::translationChanges($hours, $bots, $namespaces); foreach ($rows as $row) { $title = Title::makeTitle($row->rc_namespace, $row->rc_title); $handle = new MessageHandle($title); $code = $handle->getCode(); if (!$code) { continue; } $groupIds = $handle->getGroupIds(); foreach ($groupIds as $groupId) { $changeFilter[$groupId][$code] = true; } } } $skipGroups = array(); if ($this->hasOption('skipgroup')) { $skipGroups = array_map('trim', explode(',', $this->getOption('skipgroup'))); } foreach ($groups as $groupId => $group) { if (in_array($groupId, $skipGroups)) { $this->output("Group {$groupId} is in skipgroup.\n"); continue; } // No changes to this group at all if (is_array($changeFilter) && !isset($changeFilter[$groupId])) { $this->output("No recent changes to {$groupId}.\n"); continue; } $langs = $reqLangs; if ($codemapOnly) { foreach ($langs as $index => $code) { if ($group->mapCode($code) === $code) { unset($langs[$index]); } } } if ($threshold) { $stats = MessageGroupStats::forGroup($groupId); foreach ($langs as $index => $code) { if (!isset($stats[$code])) { unset($langs[$index]); continue; } $total = $stats[$code][MessageGroupStats::TOTAL]; $translated = $stats[$code][MessageGroupStats::TRANSLATED]; if ($translated / $total * 100 < $threshold) { unset($langs[$index]); } } } // Filter out unchanged languages from requested languages if (is_array($changeFilter)) { $langs = array_intersect($langs, array_keys($changeFilter[$groupId])); } if (!count($langs)) { continue; } $this->output("Exporting {$groupId}...\n"); $ffs = $group->getFFS(); $ffs->setWritePath($target); $sourceLanguage = $group->getSourceLanguage(); $collection = $group->initCollection($sourceLanguage); $definitionFile = false; if ($this->hasOption('ppgettext') && $ffs instanceof GettextFFS) { global $wgMaxShellMemory, $wgTranslateGroupRoot; // Need more shell memory for msgmerge. $wgMaxShellMemory = 402400; $path = $group->getSourceFilePath($sourceLanguage); $definitionFile = str_replace($wgTranslateGroupRoot, $this->getOption('ppgettext'), $path); } $whitelist = $group->getTranslatableLanguages(); foreach ($langs as $lang) { // Do not export languges that are blacklisted (or not whitelisted). // Also check that whitelist is not null, which means that all // languages are allowed for translation and export. if (is_array($whitelist) && !isset($whitelist[$lang])) { continue; } $collection->resetForNewLanguage($lang); $collection->loadTranslations(); // Don't export ignored, unless it is the source language // or message documentation global $wgTranslateDocumentationLanguageCode; if ($lang !== $wgTranslateDocumentationLanguageCode && $lang !== $sourceLanguage) { $collection->filter('ignored'); } if ($noFuzzy) { $collection->filter('fuzzy'); } $ffs->write($collection); // Do post processing if requested. if ($definitionFile) { if (is_file($definitionFile)) { $targetFileName = $ffs->getWritePath() . "/" . $group->getTargetFilename($collection->code); $cmd = "msgmerge --quiet " . $noLocation . "--output-file=" . $targetFileName . ' ' . $targetFileName . ' ' . $definitionFile; wfShellExec($cmd, $ret); // Report on errors. if ($ret) { $this->error("ERROR: {$ret}"); } } else { $this->error("{$definitionFile} does not exist.", 1); } } } } }
public function execute() { $output = $this->getOption('output', 'default'); // Select an output engine switch ($output) { case 'wiki': $out = new WikiStatsOutput(); break; case 'text': $out = new TextStatsOutput(); break; case 'csv': $out = new CsvStatsOutput(); break; default: $out = new TranslateStatsOutput(); } $skipLanguages = array(); if ($this->hasOption('skiplanguages')) { $skipLanguages = array_map('trim', explode(',', $this->getOption('skiplanguages'))); } $reportScore = false; // Check if score should be reported and prepare weights $most = $this->getOption('most'); $weights = array(); if ($most && isset($this->localisedWeights[$most])) { $reportScore = true; foreach ($this->localisedWeights[$most] as $weight) { $weights[] = $weight; } } // check if l10n should be done $l10n = false; if (($output === 'wiki' || $output === 'default') && !$this->hasOption('nol10n')) { $l10n = true; } $wmfscore = $this->hasOption('wmfscore'); // Get groups from input $groups = array(); if ($reportScore) { $reqGroups = array_keys($this->localisedWeights[$most]); } elseif ($wmfscore) { $reqGroups = array_keys($this->localisedWeights['wikimedia']); } else { $reqGroups = array_map('trim', explode(',', $this->getOption('groups'))); } // List of all groups $allGroups = MessageGroups::singleton()->getGroups(); // Get list of valid groups foreach ($reqGroups as $id) { // Page translation group ids use spaces which are not nice on command line $id = str_replace('_', ' ', $id); if (isset($allGroups[$id])) { $groups[$id] = $allGroups[$id]; } else { $this->output("Unknown group: {$id}"); } } if ($wmfscore) { // Override/set parameters $out = new CsvStatsOutput(); $reportScore = true; $weights = array(); foreach ($this->localisedWeights['wikimedia'] as $weight) { $weights[] = $weight; } $wmfscores = array(); } if (!count($groups)) { $this->error('No groups given', true); } // List of all languages. $languages = Language::fetchLanguageNames(false); // Default sorting order by language code, users can sort wiki output. ksort($languages); if ($this->hasOption('legenddetail')) { $out->addFreeText("{{" . $this->getOption('legenddetail') . "}}\n"); } $totalWeight = 0; if ($reportScore) { if ($wmfscore) { foreach ($this->localisedWeights['wikimedia'] as $weight) { $totalWeight += $weight; } } else { foreach ($this->localisedWeights[$most] as $weight) { $totalWeight += $weight; } } } $showContinent = $this->getOption('continent'); if (!$wmfscore) { // Output headers $out->heading(); $out->blockstart(); if ($most) { $out->element($l10n ? "{{int:translate-gs-pos}}" : 'Pos.', true); } $out->element($l10n ? "{{int:translate-gs-code}}" : 'Code', true); $out->element($l10n ? "{{int:translate-page-language}}" : 'Language', true); if ($showContinent) { $out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true); } if ($most && $this->hasOption('speakers')) { $out->element($l10n ? "{{int:translate-gs-speakers}}" : 'Speakers', true); } if ($reportScore) { $out->element(($l10n ? "{{int:translate-gs-score}}" : 'Score') . ' (' . $totalWeight . ')', true); } /** * @var $g MessageGroup */ foreach ($groups as $g) { // Add unprocessed description of group as heading if ($reportScore) { $gid = $g->getId(); $heading = $g->getLabel() . " (" . $this->localisedWeights[$most][$gid] . ")"; } else { $heading = $g->getLabel(); } $out->element($heading, true); if (!$reportScore && $this->hasOption('fuzzy')) { $out->element($l10n ? "{{int:translate-percentage-fuzzy}}" : 'Fuzzy', true); } } $out->blockend(); } $rows = array(); foreach ($languages as $code => $name) { // Skip list if (in_array($code, $skipLanguages)) { continue; } $rows[$code] = array(); } foreach ($groups as $groupName => $g) { $stats = MessageGroupStats::forGroup($groupName); // Perform the statistic calculations on every language foreach ($languages as $code => $name) { // Skip list if (!$most && in_array($code, $skipLanguages)) { continue; } // Do not calculate if we do not need it for anything. if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') { continue; } // If --most is set, skip all other if ($most && !isset($this->mostSpokenLanguages[$code])) { continue; } $total = $stats[$code][MessageGroupStats::TOTAL]; $translated = $stats[$code][MessageGroupStats::TRANSLATED]; $fuzzy = $stats[$code][MessageGroupStats::FUZZY]; $rows[$code][] = array(false, $translated, $total); if ($this->hasOption('fuzzy')) { $rows[$code][] = array(true, $fuzzy, $total); } } unset($collection); } // init summary array $summarise = false; if ($this->hasOption('summary')) { $summarise = true; $summary = array(); } foreach ($languages as $code => $name) { // Skip list if (!$most && in_array($code, $skipLanguages)) { continue; } // Skip unneeded if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') { continue; } // If --most is set, skip all other if ($most && !isset($this->mostSpokenLanguages[$code])) { continue; } $columns = $rows[$code]; $allZero = true; foreach ($columns as $fields) { if (intval($fields[1]) !== 0) { $allZero = false; } } // Skip dummy languages if requested if ($allZero && $this->hasOption('skipzero')) { continue; } // Output the the row if (!$wmfscore) { $out->blockstart(); } // Fill language position field if ($most) { $out->element($this->mostSpokenLanguages[$code][0]); } // Fill language name field if (!$wmfscore) { // Fill language code field $out->element($code); if ($l10n && function_exists('efI18nTagsInit')) { $out->element("{{#languagename:" . $code . "}}"); } else { $out->element($name); } } // Fill continent field if ($showContinent) { if ($this->mostSpokenLanguages[$code][2] == 'multiple') { $continent = $l10n ? "{{int:translate-gs-multiple}}" : 'Multiple'; } else { $continent = $l10n ? "{{int:timezoneregion-" . $this->mostSpokenLanguages[$code][2] . "}}" : ucfirst($this->mostSpokenLanguages[$code][2]); } $out->element($continent); } // Fill speakers field if ($most && $this->hasOption('speakers')) { $out->element(number_format($this->mostSpokenLanguages[$code][1])); } // Fill the score field if ($reportScore) { // Keep count $i = 0; // Start with 0 points $score = 0; foreach ($columns as $fields) { list(, $upper, $total) = $fields; // Weigh the score and add it to the current score $score += $weights[$i] * $upper / $total; $i++; } // Report a round numbers $score = number_format($score, 0); if ($summarise) { $continent = $this->mostSpokenLanguages[$code][2]; if (isset($summary[$continent])) { $newcount = $summary[$continent][0] + 1; $newscore = $summary[$continent][1] + (int) $score; } else { $newcount = 1; $newscore = $score; } $summary[$continent] = array($newcount, $newscore); } if ($wmfscore) { // Multiple variants can be used for the same wiki. // Store the scores in an array and output them later // when they can be averaged. if (isset($this->wikimediaCodeMap[$code])) { $wmfcode = $this->wikimediaCodeMap[$code]; } else { $codeparts = explode('-', $code); $wmfcode = $codeparts[0]; } if (isset($wmfscores[$wmfcode])) { $count = $wmfscores[$wmfcode]['count'] + 1; $tmpWmfScore = (int) $wmfscores[$wmfcode]['score']; $tmpWmfCount = (int) $wmfscores[$wmfcode]['count']; $score = ($tmpWmfCount * $tmpWmfScore + (int) $score) / $count; $wmfscores[$wmfcode] = array('score' => $score, 'count' => $count); } else { $wmfscores[$wmfcode] = array('score' => $score, 'count' => 1); } } else { $out->element($score); } } // Fill fields for groups if (!$wmfscore) { foreach ($columns as $fields) { list($invert, $upper, $total) = $fields; $c = $out->formatPercent($upper, $total, $invert); $out->element($c); } $out->blockend(); } } $out->footer(); if ($reportScore && $this->hasOption('summary')) { if ($reportScore && $this->hasOption('legendsummary')) { $out->addFreeText("{{" . $this->getOption('legendsummary') . "}}\n"); } $out->summaryheading(); $out->blockstart(); $out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true); $out->element($l10n ? "{{int:translate-gs-count}}" : 'Count', true); $out->element($l10n ? "{{int:translate-gs-avgscore}}" : 'Avg. score', true); $out->blockend(); ksort($summary); $totals = array(0, 0); foreach ($summary as $key => $values) { $out->blockstart(); if ($key == 'multiple') { $out->element($l10n ? "{{int:translate-gs-multiple}}" : 'Multiple'); } else { $out->element($l10n ? "{{int:timezoneregion-" . $key . "}}" : ucfirst($key)); } $out->element($values[0]); $out->element(number_format($values[1] / $values[0])); $out->blockend(); $totals[0] += $values[0]; $totals[1] += $values[1]; } $out->blockstart(); $out->element($l10n ? "{{int:translate-gs-total}}" : 'Total'); $out->element($totals[0]); $out->element(number_format($totals[1] / $totals[0])); $out->blockend(); $out->footer(); } // Custom output if ($wmfscore) { ksort($wmfscores); foreach ($wmfscores as $code => $stats) { echo $code . ';' . number_format($stats['score']) . ";\n"; } } }
// Add matching groups to groups array. foreach ($allGroups as $groupId => $messageGroup) { if (strpos($groupId, $options['groupprefix']) === 0 && !$messageGroup->isMeta()) { $groups[$groupId] = $messageGroup; } } } foreach ($groups as $groupId => $group) { if (!$group instanceof MessageGroup) { STDERR("Invalid group: " . $groupId); exit(1); } STDERR('Exporting ' . $groupId); $langs = $reqLangs; if ($threshold) { $stats = MessageGroupStats::forGroup($groupId); foreach ($langs as $index => $code) { if (!isset($stats[$code])) { unset($langs[$index]); } list($total, $translated, ) = $stats[$code]; if ($translated / $total * 100 < $threshold) { unset($langs[$index]); } } } if ($group instanceof FileBasedMessageGroup) { $ffs = $group->getFFS(); $ffs->setWritePath($options['target']); $collection = $group->initCollection('en'); $definitionFile = false;
/** * * @return array */ public function getTranslationPercentages() { // Calculate percentages for the available translations $group = $this->getMessageGroup(); if (!$group instanceof WikiPageMessageGroup) { return array(); } $titles = $this->getTranslationPages(); $temp = MessageGroupStats::forGroup($this->getMessageGroupId()); $stats = array(); foreach ($titles as $t) { $handle = new MessageHandle($t); $code = $handle->getCode(); // Sometimes we want to display 0.00 for pages for which translation // hasn't started yet. $stats[$code] = 0.0; if (isset($temp[$code]) && $temp[$code][MessageGroupStats::TOTAL] > 0) { $total = $temp[$code][MessageGroupStats::TOTAL]; $translated = $temp[$code][MessageGroupStats::TRANSLATED]; $percentage = $translated / $total; $stats[$code] = sprintf('%.2f', $percentage); } } // Content language is always up-to-date $stats[$this->getSourceLanguageCode()] = 1.0; return $stats; }
/** * Overwriten from SpecialLanguageStats * * @return string */ function getTable() { $table = $this->table; $this->addWorkflowStatesColumn(); $out = ''; if ($this->purge) { MessageGroupStats::clearGroup($this->target); } MessageGroupStats::setTimeLimit($this->timelimit); $cache = MessageGroupStats::forGroup($this->target); $languages = array_keys(TranslateUtils::getLanguageNames($this->getLanguage()->getCode())); sort($languages); $this->filterPriorityLangs($languages, $this->target, $cache); foreach ($languages as $code) { if ($table->isBlacklisted($this->target, $code) !== null) { continue; } $out .= $this->makeRow($code, $cache); } if ($out) { $table->setMainColumnHeader($this->msg('translate-mgs-column-language')); $out = $table->createHeader() . "\n" . $out; $out .= Html::closeElement('tbody'); $out .= Html::openElement('tfoot'); $out .= $table->makeTotalRow($this->msg('translate-mgs-totals'), $this->totals); $out .= Html::closeElement('tfoot'); $out .= Html::closeElement('table'); return $out; } else { $this->nothing = true; return ''; } }
protected function exportGroup(MessageGroup $group, $multi = false) { // Make sure all existing connections are dead, // we can't use them in forked children. LBFactory::destroyInstance(); $server = TTMServer::primary(); $id = $group->getId(); $sourceLanguage = $group->getSourceLanguage(); if ($multi) { $stats = MessageGroupStats::forGroup($id); $this->statusLine("Loaded stats for {$id}\n"); } else { $this->statusLine("Loading stats... ", 4); $stats = MessageGroupStats::forGroup($id); $this->output("done!", 4); $this->statusLine("Inserting sources: ", 5); } $collection = $group->initCollection($sourceLanguage); $collection->filter('ignored'); $collection->filter('optional'); $collection->initMessages(); $sids = array(); $counter = 0; foreach ($collection->keys() as $mkey => $title) { $def = $collection[$mkey]->definition(); $sids[$mkey] = $server->insertSource($title, $sourceLanguage, $def); if (++$counter % $this->mBatchSize === 0 && !$multi) { wfWaitForSlaves(10); $this->output('.', 5); } } $total = count($sids); if ($multi) { $this->statusLine("Inserted {$total} source entries for {$id}\n"); } else { $this->output("{$total} entries", 5); $this->statusLine("Inserting translations...", 6); } $dbw = $server->getDB(DB_MASTER); foreach ($stats as $targetLanguage => $numbers) { if ($targetLanguage === $sourceLanguage) { continue; } if ($numbers[MessageGroupStats::TRANSLATED] === 0) { continue; } if (!$multi) { $this->output(sprintf("%19s ", $targetLanguage), $targetLanguage); } $collection->resetForNewLanguage($targetLanguage); $collection->filter('ignored'); $collection->filter('optional'); $collection->filter('translated', false); $collection->loadTranslations(); $inserts = array(); foreach ($collection->keys() as $mkey => $title) { $inserts[] = array('tmt_sid' => $sids[$mkey], 'tmt_lang' => $targetLanguage, 'tmt_text' => $collection[$mkey]->translation()); } do { $batch = array_splice($inserts, 0, $this->mBatchSize); $dbw->insert('translate_tmt', $batch, __METHOD__); if (!$multi) { $this->output('.', $targetLanguage); } wfWaitForSlaves(10); } while (count($inserts)); } if ($multi) { $this->statusLine("Inserted translations for {$id}\n"); } }
protected function exportGroup(MessageGroup $group, $config) { $server = TTMServer::factory($config); $server->setLogger($this); $id = $group->getId(); $sourceLanguage = $group->getSourceLanguage(); $stats = MessageGroupStats::forGroup($id); $collection = $group->initCollection($sourceLanguage); $collection->filter('ignored'); $collection->initMessages(); $server->beginBatch(); $inserts = array(); foreach ($collection->keys() as $mkey => $title) { $handle = new MessageHandle($title); $inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->definition()); } while ($inserts !== array()) { $batch = array_splice($inserts, 0, $this->mBatchSize); $server->batchInsertDefinitions($batch); } $inserts = array(); foreach ($stats as $targetLanguage => $numbers) { if ($targetLanguage === $sourceLanguage) { continue; } if ($numbers[MessageGroupStats::TRANSLATED] === 0) { continue; } $collection->resetForNewLanguage($targetLanguage); $collection->filter('ignored'); $collection->filter('translated', false); $collection->loadTranslations(); foreach ($collection->keys() as $mkey => $title) { $handle = new MessageHandle($title); $inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->translation()); } while (count($inserts) >= $this->mBatchSize) { $batch = array_splice($inserts, 0, $this->mBatchSize); $server->batchInsertTranslations($batch); } } while ($inserts !== array()) { $batch = array_splice($inserts, 0, $this->mBatchSize); $server->batchInsertTranslations($batch); } $server->endBatch(); }
$out->element( ( $l10n ? "{{int:translate-percentage-fuzzy}}" : 'Fuzzy' ), true ); } } $out->blockend(); } $rows = array(); foreach ( $languages as $code => $name ) { // Skip list if ( in_array( $code, $skipLanguages ) ) continue; $rows[$code] = array(); } foreach ( $groups as $groupName => $g ) { $stats = MessageGroupStats::forGroup( $groupName ); // Perform the statistic calculations on every language foreach ( $languages as $code => $name ) { // Skip list if ( !isset( $options['most'] ) && in_array( $code, $skipLanguages ) ) { continue; } // Do not calculate if we do not need it for anything. if ( $wmfscore && isset( $wikimediaCodeMap[$code] ) && $wikimediaCodeMap[$code] == '' ) { continue; } // If --most is set, skip all other if ( isset( $options['most'] ) && !isset( $mostSpokenLanguages[$code] ) ) {