protected function getData()
{
$params = $this->extractRequestParams();
$group = MessageGroups::getGroup($params['group']);
if (!$group) {
$this->dieUsageMsg(array('missingparam', 'mcgroup'));
} elseif (MessageGroups::isDynamic($group)) {
$this->dieUsage('Dynamic message groups are not supported here', 'invalidparam');
}
return MessageGroupStats::forGroup($group->getId());
}
/**
* @param string $group
* @param string $language
* @param array[]|null $stats
*
* @return self
*/
public static function getNew($group, $language, array $stats = null)
{
$self = new self();
$self->group = $group;
$self->language = $language;
if (is_array($stats)) {
$self->stats = $stats;
} else {
$self->stats = MessageGroupStats::forItem($group, $language);
}
return $self;
}
public function run()
{
$title = $this->title;
$handle = new MessageHandle($title);
$code = $handle->getCode();
if (!$handle->isValid() && !$code) {
return true;
}
$groups = self::getGroupsWithTransitions($handle);
foreach ($groups as $id => $transitions) {
$group = MessageGroups::getGroup($id);
$stats = MessageGroupStats::forItem($id, $code);
$state = self::getNewState($stats, $transitions);
if ($state) {
ApiGroupReview::changeState($group, $code, $state, FuzzyBot::getUser());
}
}
return true;
}
public function execute()
{
$params = $this->extractRequestParams();
MessageGroupStats::setTimeLimit($params['timelimit']);
$cache = $this->getData();
$result = $this->getResult();
foreach ($cache as $item => $stats) {
if ($item < $params['offset']) {
continue;
}
if ($stats[MessageGroupStats::TOTAL] === null) {
$this->setContinueEnumParameter('offset', $item);
break;
}
$data = $this->makeItem($item, $stats);
$result->addValue(array('query', $this->getModuleName()), null, $data);
}
if (defined('ApiResult::META_CONTENT')) {
$result->addIndexedTagName(array('query', $this->getModuleName()), 'stats');
} else {
$result->setIndexedTagName_internal(array('query', $this->getModuleName()), 'stats');
}
}
public function execute()
{
$params = $this->extractRequestParams();
$group = MessageGroups::getGroup($params['group']);
if (!$group) {
$this->dieUsageMsg(array('missingparam', 'mcgroup'));
}
MessageGroupStats::setTimeLimit($params['timelimit']);
$cache = MessageGroupStats::forGroup($group->getId());
$result = $this->getResult();
foreach ($cache as $code => $stats) {
if ($code < $params['offset']) {
continue;
}
list($total, $translated, $fuzzy) = $stats;
if ($total === null) {
$this->setContinueEnumParameter('offset', $code);
break;
}
$data = array('code' => $code, 'total' => $total, 'translated' => $translated, 'fuzzy' => $fuzzy);
$result->addValue(array('query', $this->getModuleName()), null, $data);
}
$result->setIndexedTagName_internal(array('query', $this->getModuleName()), 'messagegroupstats');
}
protected function getData()
{
$params = $this->extractRequestParams();
return MessageGroupStats::forLanguage($params['language']);
}
// Add matching groups to groups array.
foreach ($allGroups as $groupId => $messageGroup) {
if (strpos($groupId, $options['groupprefix']) === 0 && !$messageGroup->isMeta()) {
$groups[$groupId] = $messageGroup;
}
}
}
foreach ($groups as $groupId => $group) {
if (!$group instanceof MessageGroup) {
STDERR("Invalid group: " . $groupId);
exit(1);
}
STDERR('Exporting ' . $groupId);
$langs = $reqLangs;
if ($threshold) {
$stats = MessageGroupStats::forGroup($groupId);
foreach ($langs as $index => $code) {
if (!isset($stats[$code])) {
unset($langs[$index]);
}
list($total, $translated, ) = $stats[$code];
if ($translated / $total * 100 < $threshold) {
unset($langs[$index]);
}
}
}
if ($group instanceof FileBasedMessageGroup) {
$ffs = $group->getFFS();
$ffs->setWritePath($options['target']);
$collection = $group->initCollection('en');
$definitionFile = false;
public function execute()
{
$output = $this->getOption('output', 'default');
// Select an output engine
switch ($output) {
case 'wiki':
$out = new WikiStatsOutput();
break;
case 'text':
$out = new TextStatsOutput();
break;
case 'csv':
$out = new CsvStatsOutput();
break;
default:
$out = new TranslateStatsOutput();
}
$skipLanguages = array();
if ($this->hasOption('skiplanguages')) {
$skipLanguages = array_map('trim', explode(',', $this->getOption('skiplanguages')));
}
$reportScore = false;
// Check if score should be reported and prepare weights
$most = $this->getOption('most');
$weights = array();
if ($most && isset($this->localisedWeights[$most])) {
$reportScore = true;
foreach ($this->localisedWeights[$most] as $weight) {
$weights[] = $weight;
}
}
// check if l10n should be done
$l10n = false;
if (($output === 'wiki' || $output === 'default') && !$this->hasOption('nol10n')) {
$l10n = true;
}
$wmfscore = $this->hasOption('wmfscore');
// Get groups from input
$groups = array();
if ($reportScore) {
$reqGroups = array_keys($this->localisedWeights[$most]);
} elseif ($wmfscore) {
$reqGroups = array_keys($this->localisedWeights['wikimedia']);
} else {
$reqGroups = array_map('trim', explode(',', $this->getOption('groups')));
}
// List of all groups
$allGroups = MessageGroups::singleton()->getGroups();
// Get list of valid groups
foreach ($reqGroups as $id) {
// Page translation group ids use spaces which are not nice on command line
$id = str_replace('_', ' ', $id);
if (isset($allGroups[$id])) {
$groups[$id] = $allGroups[$id];
} else {
$this->output("Unknown group: {$id}");
}
}
if ($wmfscore) {
// Override/set parameters
$out = new CsvStatsOutput();
$reportScore = true;
$weights = array();
foreach ($this->localisedWeights['wikimedia'] as $weight) {
$weights[] = $weight;
}
$wmfscores = array();
}
if (!count($groups)) {
$this->error('No groups given', true);
}
// List of all languages.
$languages = Language::fetchLanguageNames(false);
// Default sorting order by language code, users can sort wiki output.
ksort($languages);
if ($this->hasOption('legenddetail')) {
$out->addFreeText("{{" . $this->getOption('legenddetail') . "}}\n");
}
$totalWeight = 0;
if ($reportScore) {
if ($wmfscore) {
foreach ($this->localisedWeights['wikimedia'] as $weight) {
$totalWeight += $weight;
}
} else {
foreach ($this->localisedWeights[$most] as $weight) {
$totalWeight += $weight;
}
}
}
$showContinent = $this->getOption('continent');
if (!$wmfscore) {
// Output headers
$out->heading();
$out->blockstart();
if ($most) {
$out->element($l10n ? "{{int:translate-gs-pos}}" : 'Pos.', true);
}
$out->element($l10n ? "{{int:translate-gs-code}}" : 'Code', true);
$out->element($l10n ? "{{int:translate-page-language}}" : 'Language', true);
if ($showContinent) {
$out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true);
}
if ($most && $this->hasOption('speakers')) {
$out->element($l10n ? "{{int:translate-gs-speakers}}" : 'Speakers', true);
}
if ($reportScore) {
$out->element(($l10n ? "{{int:translate-gs-score}}" : 'Score') . ' (' . $totalWeight . ')', true);
}
/**
* @var $g MessageGroup
*/
foreach ($groups as $g) {
// Add unprocessed description of group as heading
if ($reportScore) {
$gid = $g->getId();
$heading = $g->getLabel() . " (" . $this->localisedWeights[$most][$gid] . ")";
} else {
$heading = $g->getLabel();
}
$out->element($heading, true);
if (!$reportScore && $this->hasOption('fuzzy')) {
$out->element($l10n ? "{{int:translate-percentage-fuzzy}}" : 'Fuzzy', true);
}
}
$out->blockend();
}
$rows = array();
foreach ($languages as $code => $name) {
// Skip list
if (in_array($code, $skipLanguages)) {
continue;
}
$rows[$code] = array();
}
foreach ($groups as $groupName => $g) {
$stats = MessageGroupStats::forGroup($groupName);
// Perform the statistic calculations on every language
foreach ($languages as $code => $name) {
// Skip list
if (!$most && in_array($code, $skipLanguages)) {
continue;
}
// Do not calculate if we do not need it for anything.
if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') {
continue;
}
// If --most is set, skip all other
if ($most && !isset($this->mostSpokenLanguages[$code])) {
continue;
}
$total = $stats[$code][MessageGroupStats::TOTAL];
$translated = $stats[$code][MessageGroupStats::TRANSLATED];
$fuzzy = $stats[$code][MessageGroupStats::FUZZY];
$rows[$code][] = array(false, $translated, $total);
if ($this->hasOption('fuzzy')) {
$rows[$code][] = array(true, $fuzzy, $total);
}
}
unset($collection);
}
// init summary array
$summarise = false;
if ($this->hasOption('summary')) {
$summarise = true;
$summary = array();
}
foreach ($languages as $code => $name) {
// Skip list
if (!$most && in_array($code, $skipLanguages)) {
continue;
}
// Skip unneeded
if ($wmfscore && isset($this->wikimediaCodeMap[$code]) && $this->wikimediaCodeMap[$code] == '') {
continue;
}
// If --most is set, skip all other
if ($most && !isset($this->mostSpokenLanguages[$code])) {
continue;
}
$columns = $rows[$code];
$allZero = true;
foreach ($columns as $fields) {
if (intval($fields[1]) !== 0) {
$allZero = false;
}
}
// Skip dummy languages if requested
if ($allZero && $this->hasOption('skipzero')) {
continue;
}
// Output the the row
if (!$wmfscore) {
$out->blockstart();
}
// Fill language position field
if ($most) {
$out->element($this->mostSpokenLanguages[$code][0]);
}
// Fill language name field
if (!$wmfscore) {
// Fill language code field
$out->element($code);
if ($l10n && function_exists('efI18nTagsInit')) {
$out->element("{{#languagename:" . $code . "}}");
} else {
$out->element($name);
}
}
// Fill continent field
if ($showContinent) {
if ($this->mostSpokenLanguages[$code][2] == 'multiple') {
$continent = $l10n ? "{{int:translate-gs-multiple}}" : 'Multiple';
} else {
$continent = $l10n ? "{{int:timezoneregion-" . $this->mostSpokenLanguages[$code][2] . "}}" : ucfirst($this->mostSpokenLanguages[$code][2]);
}
$out->element($continent);
}
// Fill speakers field
if ($most && $this->hasOption('speakers')) {
$out->element(number_format($this->mostSpokenLanguages[$code][1]));
}
// Fill the score field
if ($reportScore) {
// Keep count
$i = 0;
// Start with 0 points
$score = 0;
foreach ($columns as $fields) {
list(, $upper, $total) = $fields;
// Weigh the score and add it to the current score
$score += $weights[$i] * $upper / $total;
$i++;
}
// Report a round numbers
$score = number_format($score, 0);
if ($summarise) {
$continent = $this->mostSpokenLanguages[$code][2];
if (isset($summary[$continent])) {
$newcount = $summary[$continent][0] + 1;
$newscore = $summary[$continent][1] + (int) $score;
} else {
$newcount = 1;
$newscore = $score;
}
$summary[$continent] = array($newcount, $newscore);
}
if ($wmfscore) {
// Multiple variants can be used for the same wiki.
// Store the scores in an array and output them later
// when they can be averaged.
if (isset($this->wikimediaCodeMap[$code])) {
$wmfcode = $this->wikimediaCodeMap[$code];
} else {
$codeparts = explode('-', $code);
$wmfcode = $codeparts[0];
}
if (isset($wmfscores[$wmfcode])) {
$count = $wmfscores[$wmfcode]['count'] + 1;
$tmpWmfScore = (int) $wmfscores[$wmfcode]['score'];
$tmpWmfCount = (int) $wmfscores[$wmfcode]['count'];
$score = ($tmpWmfCount * $tmpWmfScore + (int) $score) / $count;
$wmfscores[$wmfcode] = array('score' => $score, 'count' => $count);
} else {
$wmfscores[$wmfcode] = array('score' => $score, 'count' => 1);
}
} else {
$out->element($score);
}
}
// Fill fields for groups
if (!$wmfscore) {
foreach ($columns as $fields) {
list($invert, $upper, $total) = $fields;
$c = $out->formatPercent($upper, $total, $invert);
$out->element($c);
}
$out->blockend();
}
}
$out->footer();
if ($reportScore && $this->hasOption('summary')) {
if ($reportScore && $this->hasOption('legendsummary')) {
$out->addFreeText("{{" . $this->getOption('legendsummary') . "}}\n");
}
$out->summaryheading();
$out->blockstart();
$out->element($l10n ? "{{int:translate-gs-continent}}" : 'Continent', true);
$out->element($l10n ? "{{int:translate-gs-count}}" : 'Count', true);
$out->element($l10n ? "{{int:translate-gs-avgscore}}" : 'Avg. score', true);
$out->blockend();
ksort($summary);
$totals = array(0, 0);
foreach ($summary as $key => $values) {
$out->blockstart();
if ($key == 'multiple') {
$out->element($l10n ? "{{int:translate-gs-multiple}}" : 'Multiple');
} else {
$out->element($l10n ? "{{int:timezoneregion-" . $key . "}}" : ucfirst($key));
}
$out->element($values[0]);
$out->element(number_format($values[1] / $values[0]));
$out->blockend();
$totals[0] += $values[0];
$totals[1] += $values[1];
}
$out->blockstart();
$out->element($l10n ? "{{int:translate-gs-total}}" : 'Total');
$out->element($totals[0]);
$out->element(number_format($totals[1] / $totals[0]));
$out->blockend();
$out->footer();
}
// Custom output
if ($wmfscore) {
ksort($wmfscores);
foreach ($wmfscores as $code => $stats) {
echo $code . ';' . number_format($stats['score']) . ";\n";
}
}
}
/**
* @param $group
* @param $cache
* @param $parent bool
* @return string
*/
protected function makeGroupRow( $group, $cache, $parent = false ) {
$groupId = $group->getId();
if ( $this->table->isBlacklisted( $groupId, $this->target ) !== null ) {
return '';
}
# These are hidden, and the code in MessageGroupStats makes sure that
# these are not counted in the aggregate groups they may belong.
if ( MessageGroups::getPriority( $group ) === 'discouraged' ) {
return '';
}
$stats = $cache[$groupId];
list( $total, $translated, $fuzzy ) = $stats;
if ( $total === null ) {
$this->incomplete = true;
$extra = array();
} else {
if ( $this->noComplete && $fuzzy === 0 && $translated === $total ) {
return '';
}
if ( $this->noEmpty && $translated === 0 && $fuzzy === 0 ) {
return '';
}
if ( $translated === $total ) {
$extra = array( 'task' => 'reviewall' );
} else {
$extra = array();
}
}
if ( !$group instanceof AggregateMessageGroup ) {
$this->totals = MessageGroupStats::multiAdd( $this->totals, $stats );
}
$rowParams = array();
$rowParams['data-groupid'] = $groupId;
if ( is_string( $parent ) ) {
$rowParams['data-parentgroups'] = $parent;
} elseif ( $parent === true ) {
$rowParams['data-ismeta'] = '1';
}
$out = "\t" . Html::openElement( 'tr', $rowParams );
$out .= "\n\t\t" . Html::rawElement( 'td', array(),
$this->table->makeGroupLink( $group, $this->target, $extra ) );
$out .= $this->table->makeNumberColumns( $fuzzy, $translated, $total );
$out .= $this->getWorkflowStateCell( $groupId );
$out .= "\n\t" . Html::closeElement( 'tr' ) . "\n";
return $out;
}
/**
*
* @return array
*/
public function getTranslationPercentages()
{
// Calculate percentages for the available translations
$group = $this->getMessageGroup();
if (!$group instanceof WikiPageMessageGroup) {
return array();
}
$titles = $this->getTranslationPages();
$temp = MessageGroupStats::forGroup($this->getMessageGroupId());
$stats = array();
foreach ($titles as $t) {
$handle = new MessageHandle($t);
$code = $handle->getCode();
// Sometimes we want to display 0.00 for pages for which translation
// hasn't started yet.
$stats[$code] = 0.0;
if (isset($temp[$code]) && $temp[$code][MessageGroupStats::TOTAL] > 0) {
$total = $temp[$code][MessageGroupStats::TOTAL];
$translated = $temp[$code][MessageGroupStats::TRANSLATED];
$percentage = $translated / $total;
$stats[$code] = sprintf('%.2f', $percentage);
}
}
// Content language is always up-to-date
$stats[$this->getSourceLanguageCode()] = 1.0;
return $stats;
}
/**
* Actually creates the table for single message group, unless it
* is blacklisted or hidden by filters.
* @param MessageGroup $group
* @param array $cache
* @param MessageGroup $parent
* @return string
*/
protected function makeGroupRow(MessageGroup $group, array $cache, MessageGroup $parent = null)
{
$groupId = $group->getId();
if ($this->table->isBlacklisted($groupId, $this->target) !== null) {
return '';
}
$stats = $cache[$groupId];
$total = $stats[MessageGroupStats::TOTAL];
$translated = $stats[MessageGroupStats::TRANSLATED];
$fuzzy = $stats[MessageGroupStats::FUZZY];
// Quick checks to see whether filters apply
if ($this->noComplete && $fuzzy === 0 && $translated === $total) {
return '';
}
if ($this->noEmpty && $translated === 0 && $fuzzy === 0) {
return '';
}
// Calculation of summary row values
if (!$group instanceof AggregateMessageGroup) {
if (!isset($this->statsCounted[$groupId])) {
$this->totals = MessageGroupStats::multiAdd($this->totals, $stats);
$this->statsCounted[$groupId] = true;
}
}
$state = $this->getWorkflowStateValue($groupId);
$params = $stats;
$params[] = $state;
$params[] = $groupId;
$params[] = $this->getLanguage()->getCode();
$params[] = $this->target;
$cachekey = wfMemcKey(__METHOD__, implode('-', $params));
$cacheval = wfGetCache(CACHE_ANYTHING)->get($cachekey);
if (!$this->purge && is_string($cacheval)) {
return $cacheval;
}
$extra = array();
if ($total === null) {
$this->incomplete = true;
} elseif ($translated === $total) {
$extra = array('action' => 'proofread');
}
$rowParams = array();
$rowParams['data-groupid'] = $groupId;
$rowParams['class'] = get_class($group);
if ($parent) {
$rowParams['data-parentgroup'] = $parent->getId();
}
$out = "\t" . Html::openElement('tr', $rowParams);
$out .= "\n\t\t" . Html::rawElement('td', array(), $this->table->makeGroupLink($group, $this->target, $extra));
$out .= $this->table->makeNumberColumns($stats);
$out .= $this->getWorkflowStateCell($groupId, $state);
$out .= "\n\t" . Html::closeElement('tr') . "\n";
wfGetCache(CACHE_ANYTHING)->set($cachekey, $out, 3600 * 24);
return $out;
}
$out->element( ( $l10n ? "{{int:translate-percentage-fuzzy}}" : 'Fuzzy' ), true );
}
}
$out->blockend();
}
$rows = array();
foreach ( $languages as $code => $name ) {
// Skip list
if ( in_array( $code, $skipLanguages ) ) continue;
$rows[$code] = array();
}
foreach ( $groups as $groupName => $g ) {
$stats = MessageGroupStats::forGroup( $groupName );
// Perform the statistic calculations on every language
foreach ( $languages as $code => $name ) {
// Skip list
if ( !isset( $options['most'] ) && in_array( $code, $skipLanguages ) ) {
continue;
}
// Do not calculate if we do not need it for anything.
if ( $wmfscore && isset( $wikimediaCodeMap[$code] ) && $wikimediaCodeMap[$code] == '' ) {
continue;
}
// If --most is set, skip all other
if ( isset( $options['most'] ) && !isset( $mostSpokenLanguages[$code] ) ) {
/**
* Runs message checks, adds tp:transver tags and updates statistics.
* Hook: ArticleSaveComplete
*/
public static function onSave( $article, $user, $text, $summary,
$minor, $_, $_, $flags, $revision
) {
$title = $article->getTitle();
$handle = new MessageHandle( $title );
if ( !$handle->isValid() ) {
return true;
}
// Update it.
if ( $revision === null ) {
$rev = $article->getTitle()->getLatestRevId();
} else {
$rev = $revision->getID();
}
$fuzzy = self::checkNeedsFuzzy( $handle, $text );
self::updateFuzzyTag( $title, $rev, $fuzzy );
MessageGroupStats::clear( $handle );
if ( $fuzzy === false ) {
wfRunHooks( 'Translate:newTranslation', array( $handle, $rev, $text, $user ) );
}
return true;
}
/**
* Set the maximum time statistics are calculated.
* If the time limit is exceeded, the missing
* entries will be null.
* @param $limit float time in seconds
*/
public static function setTimeLimit( $limit ) {
self::$timeStart = microtime( true );
self::$limit = $limit;
}
/**
* Purge message group stats when set of keys have changed.
* @param $old array
* @param $new array
*/
protected function clearMessageGroupStats(array $old, array $new)
{
$changes = array();
foreach (array_diff_assoc($new, $old) as $groups) {
foreach ((array) $groups as $group) {
$changes[$group] = true;
}
}
foreach (array_diff_assoc($old, $new) as $groups) {
foreach ((array) $groups as $group) {
$changes[$group] = true;
}
}
MessageGroupStats::clearGroup(array_keys($changes));
}
protected function exportGroup(MessageGroup $group, $config)
{
$server = TTMServer::factory($config);
$server->setLogger($this);
$id = $group->getId();
$sourceLanguage = $group->getSourceLanguage();
$stats = MessageGroupStats::forGroup($id);
$collection = $group->initCollection($sourceLanguage);
$collection->filter('ignored');
$collection->initMessages();
$server->beginBatch();
$inserts = array();
foreach ($collection->keys() as $mkey => $title) {
$handle = new MessageHandle($title);
$inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->definition());
}
while ($inserts !== array()) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertDefinitions($batch);
}
$inserts = array();
foreach ($stats as $targetLanguage => $numbers) {
if ($targetLanguage === $sourceLanguage) {
continue;
}
if ($numbers[MessageGroupStats::TRANSLATED] === 0) {
continue;
}
$collection->resetForNewLanguage($targetLanguage);
$collection->filter('ignored');
$collection->filter('translated', false);
$collection->loadTranslations();
foreach ($collection->keys() as $mkey => $title) {
$handle = new MessageHandle($title);
$inserts[] = array($handle, $sourceLanguage, $collection[$mkey]->translation());
}
while (count($inserts) >= $this->mBatchSize) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertTranslations($batch);
}
}
while ($inserts !== array()) {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$server->batchInsertTranslations($batch);
}
$server->endBatch();
}
public function execute($parameters)
{
$this->setHeaders();
$user = $this->getUser();
$request = $this->getRequest();
$target = $request->getText('target', $parameters);
$revision = $request->getInt('revision', 0);
$action = $request->getVal('do');
$out = $this->getOutput();
TranslateUtils::addSpecialHelpLink($out, 'Help:Extension:Translate/Page_translation_example');
// No specific page or invalid input
$title = Title::newFromText($target);
if (!$title) {
if ($target !== '') {
$out->addWikiMsg('tpt-badtitle');
} else {
$this->listPages();
}
return;
}
// Check permissions
if (!$user->isAllowed('pagetranslation')) {
throw new PermissionsError('pagetranslation');
}
// Check permissions
if ($request->wasPosted() && !$user->matchEditToken($request->getText('token'))) {
throw new PermissionsError('pagetranslation');
}
// We are processing some specific page
if (!$title->exists()) {
$out->addWikiMsg('tpt-nosuchpage', $title->getPrefixedText());
return;
}
if ($action === 'discourage' || $action === 'encourage') {
$id = TranslatablePage::getMessageGroupIdFromTitle($title);
$current = MessageGroups::getPriority($id);
if ($action === 'encourage') {
$new = '';
} else {
$new = 'discouraged';
}
if ($new !== $current) {
MessageGroups::setPriority($id, $new);
$entry = new ManualLogEntry('pagetranslation', $action);
$entry->setPerformer($user);
$entry->setTarget($title);
$logid = $entry->insert();
$entry->publish($logid);
}
$this->listPages();
$group = MessageGroups::getGroup($id);
$parents = MessageGroups::getSharedGroups($group);
MessageGroupStats::clearGroup($parents);
return;
}
if ($action === 'unlink') {
if (!$request->wasPosted()) {
$this->showUnlinkConfirmation($title);
return;
} else {
$page = TranslatablePage::newFromTitle($title);
$content = ContentHandler::makeContent(self::getStrippedSourcePageText($page->getParse()), $title);
$status = WikiPage::factory($title)->doEditContent($content, $this->msg('tpt-unlink-summary')->inContentLanguage()->text(), EDIT_FORCE_BOT | EDIT_UPDATE);
if (!$status->isOK()) {
$out->wrapWikiMsg('<div class="errorbox">$1</div>', array('tpt-edit-failed', $status->getWikiText()));
return;
}
$page = TranslatablePage::newFromTitle($title);
$this->unmarkPage($page, $user);
$out->wrapWikiMsg('<div class="successbox">$1</div>', array('tpt-unmarked', $title->getPrefixedText()));
$this->listPages();
return;
}
}
if ($action === 'unmark') {
$page = TranslatablePage::newFromTitle($title);
$this->unmarkPage($page, $user);
$out->wrapWikiMsg('<div class="successbox">$1</div>', array('tpt-unmarked', $title->getPrefixedText()));
$this->listPages();
return;
}
if ($revision === 0) {
// Get the latest revision
$revision = intval($title->getLatestRevID());
}
$page = TranslatablePage::newFromRevision($title, $revision);
if (!$page instanceof TranslatablePage) {
$out->wrapWikiMsg('<div class="errorbox">$1</div>', array('tpt-notsuitable', $title->getPrefixedText(), $revision));
return;
}
if ($revision !== intval($title->getLatestRevID())) {
// We do want to notify the reviewer if the underlying page changes during review
$target = $title->getFullUrl(array('oldid' => $revision));
$link = "<span class='plainlinks'>[{$target} {$revision}]</span>";
$out->wrapWikiMsg('<div class="warningbox">$1</div>', array('tpt-oldrevision', $title->getPrefixedText(), $link));
$this->listPages();
return;
}
$lastrev = $page->getMarkedTag();
if ($lastrev !== false && $lastrev === $revision) {
$out->wrapWikiMsg('<div class="warningbox">$1</div>', array('tpt-already-marked'));
$this->listPages();
return;
}
// This will modify the sections to include name property
$error = false;
$sections = $this->checkInput($page, $error);
// Non-fatal error which prevents saving
if ($error === false && $request->wasPosted()) {
// Check if user wants to translate title
// If not, remove it from the list of sections
if (!$request->getCheck('translatetitle')) {
$sections = array_filter($sections, function ($s) {
return $s->id !== 'Page display title';
});
}
$err = $this->markForTranslation($page, $sections);
if ($err) {
call_user_func_array(array($out, 'addWikiMsg'), $err);
} else {
$this->showSuccess($page);
$this->listPages();
}
return;
}
$this->showPage($page, $sections);
}
/**
* @param $code
* @param $cache
* @return string
*/
protected function makeRow($code, $cache)
{
$stats = $cache[$code];
$total = $stats[MessageGroupStats::TOTAL];
$translated = $stats[MessageGroupStats::TRANSLATED];
$fuzzy = $stats[MessageGroupStats::FUZZY];
if ($total === null) {
$this->incomplete = true;
$extra = array();
} else {
if ($this->noComplete && $fuzzy === 0 && $translated === $total) {
return '';
}
if ($this->noEmpty && $translated === 0 && $fuzzy === 0) {
return '';
}
// Skip below 2% if "don't show without translations" is checked.
if ($this->noEmpty && $translated / $total < 0.02) {
return '';
}
if ($translated === $total) {
$extra = array('action' => 'proofread');
} else {
$extra = array();
}
}
$this->totals = MessageGroupStats::multiAdd($this->totals, $stats);
$out = "\t" . Html::openElement('tr');
$out .= "\n\t\t" . $this->getMainColumnCell($code, $extra);
$out .= $this->table->makeNumberColumns($stats);
$state = $this->getWorkflowStateValue($code);
$out .= $this->getWorkflowStateCell($code, $state);
$out .= "\n\t" . Html::closeElement('tr') . "\n";
return $out;
}
/**
* @param $code
* @param $cache
* @return string
*/
protected function makeRow($code, $cache)
{
$stats = $cache[$code];
list($total, $translated, $fuzzy) = $stats;
if ($total === null) {
$this->incomplete = true;
$extra = array();
} else {
if ($this->noComplete && $fuzzy === 0 && $translated === $total) {
return '';
}
if ($this->noEmpty && $translated === 0 && $fuzzy === 0) {
return '';
}
if ($translated === $total) {
$extra = array('task' => 'reviewall');
} else {
$extra = array();
}
}
$this->totals = MessageGroupStats::multiAdd($this->totals, $stats);
$out = "\t" . Html::openElement('tr');
$out .= "\n\t\t" . $this->getMainColumnCell($code, $extra);
$out .= $this->table->makeNumberColumns($fuzzy, $translated, $total);
$out .= $this->getWorkflowStateCell($code);
$out .= "\n\t" . Html::closeElement('tr') . "\n";
return $out;
}
protected function exportGroup(MessageGroup $group, $multi = false)
{
// Make sure all existing connections are dead,
// we can't use them in forked children.
LBFactory::destroyInstance();
$server = TTMServer::primary();
$id = $group->getId();
$sourceLanguage = $group->getSourceLanguage();
if ($multi) {
$stats = MessageGroupStats::forGroup($id);
$this->statusLine("Loaded stats for {$id}\n");
} else {
$this->statusLine("Loading stats... ", 4);
$stats = MessageGroupStats::forGroup($id);
$this->output("done!", 4);
$this->statusLine("Inserting sources: ", 5);
}
$collection = $group->initCollection($sourceLanguage);
$collection->filter('ignored');
$collection->filter('optional');
$collection->initMessages();
$sids = array();
$counter = 0;
foreach ($collection->keys() as $mkey => $title) {
$def = $collection[$mkey]->definition();
$sids[$mkey] = $server->insertSource($title, $sourceLanguage, $def);
if (++$counter % $this->mBatchSize === 0 && !$multi) {
wfWaitForSlaves(10);
$this->output('.', 5);
}
}
$total = count($sids);
if ($multi) {
$this->statusLine("Inserted {$total} source entries for {$id}\n");
} else {
$this->output("{$total} entries", 5);
$this->statusLine("Inserting translations...", 6);
}
$dbw = $server->getDB(DB_MASTER);
foreach ($stats as $targetLanguage => $numbers) {
if ($targetLanguage === $sourceLanguage) {
continue;
}
if ($numbers[MessageGroupStats::TRANSLATED] === 0) {
continue;
}
if (!$multi) {
$this->output(sprintf("%19s ", $targetLanguage), $targetLanguage);
}
$collection->resetForNewLanguage($targetLanguage);
$collection->filter('ignored');
$collection->filter('optional');
$collection->filter('translated', false);
$collection->loadTranslations();
$inserts = array();
foreach ($collection->keys() as $mkey => $title) {
$inserts[] = array('tmt_sid' => $sids[$mkey], 'tmt_lang' => $targetLanguage, 'tmt_text' => $collection[$mkey]->translation());
}
do {
$batch = array_splice($inserts, 0, $this->mBatchSize);
$dbw->insert('translate_tmt', $batch, __METHOD__);
if (!$multi) {
$this->output('.', $targetLanguage);
}
wfWaitForSlaves(10);
} while (count($inserts));
}
if ($multi) {
$this->statusLine("Inserted translations for {$id}\n");
}
}
/**
* Purge message group stats when set of keys have changed.
* @param array $old
* @param array $new
*/
protected function clearMessageGroupStats(array $old, array $new)
{
$changes = array();
foreach ($new as $key => $groups) {
// Using != here on purpose to ignore order of items
if (!isset($old[$key])) {
$changes[$key] = array(array(), (array) $groups);
} elseif ($groups != $old[$key]) {
$changes[$key] = array((array) $old[$key], (array) $groups);
}
}
foreach ($old as $key => $groups) {
if (!isset($new[$key])) {
$changes[$key] = array((array) $groups, array());
}
// We already checked for diffs above
}
$changedGroups = array();
foreach ($changes as $data) {
foreach ($data[0] as $group) {
$changedGroups[$group] = true;
}
foreach ($data[1] as $group) {
$changedGroups[$group] = true;
}
}
MessageGroupStats::clearGroup(array_keys($changedGroups));
foreach ($changes as $key => $data) {
list($ns, $pagename) = explode(':', $key, 2);
$title = Title::makeTitle($ns, $pagename);
$handle = new MessageHandle($title);
list($oldGroups, $newGroups) = $data;
Hooks::run('TranslateEventMessageMembershipChange', array($handle, $oldGroups, $newGroups));
}
}
public function execute()
{
$target = $this->getOption('target');
if (!is_writable($target)) {
$this->error("Target directory is not writable ({$target}).", 1);
}
$threshold = $this->getOption('threshold');
$noFuzzy = $this->hasOption('no-fuzzy');
$noLocation = '';
if ($this->hasOption('no-location')) {
$noLocation = '--no-location ';
}
$skip = array();
if ($this->hasOption('skip')) {
$skip = array_map('trim', explode(',', $this->getOption('skip')));
}
$reqLangs = TranslateUtils::parseLanguageCodes($this->getOption('lang'));
$reqLangs = array_flip($reqLangs);
foreach ($skip as $skipLang) {
unset($reqLangs[$skipLang]);
}
$reqLangs = array_flip($reqLangs);
$codemapOnly = $this->hasOption('codemaponly');
$groupIds = explode(',', trim($this->getOption('group')));
$groupIds = MessageGroups::expandWildcards($groupIds);
$groups = MessageGroups::getGroupsById($groupIds);
/** @var FileBasedMessageGroup $group */
foreach ($groups as $groupId => $group) {
if ($group->isMeta()) {
$this->output("Skipping meta message group {$groupId}.\n");
unset($groups[$groupId]);
continue;
}
if (!$group instanceof FileBasedMessageGroup) {
$this->output("EE2: Unexportable message group {$groupId}.\n");
unset($groups[$groupId]);
continue;
}
}
if (!count($groups)) {
$this->error("EE1: No valid message groups identified.", 1);
}
$changeFilter = false;
$hours = $this->getOption('hours');
if ($hours) {
$namespaces = array();
/** @var FileBasedMessageGroup $group */
foreach ($groups as $group) {
$namespaces[$group->getNamespace()] = true;
}
$namespaces = array_keys($namespaces);
$bots = true;
$changeFilter = array();
$rows = TranslateUtils::translationChanges($hours, $bots, $namespaces);
foreach ($rows as $row) {
$title = Title::makeTitle($row->rc_namespace, $row->rc_title);
$handle = new MessageHandle($title);
$code = $handle->getCode();
if (!$code) {
continue;
}
$groupIds = $handle->getGroupIds();
foreach ($groupIds as $groupId) {
$changeFilter[$groupId][$code] = true;
}
}
}
$skipGroups = array();
if ($this->hasOption('skipgroup')) {
$skipGroups = array_map('trim', explode(',', $this->getOption('skipgroup')));
}
foreach ($groups as $groupId => $group) {
if (in_array($groupId, $skipGroups)) {
$this->output("Group {$groupId} is in skipgroup.\n");
continue;
}
// No changes to this group at all
if (is_array($changeFilter) && !isset($changeFilter[$groupId])) {
$this->output("No recent changes to {$groupId}.\n");
continue;
}
$langs = $reqLangs;
if ($codemapOnly) {
foreach ($langs as $index => $code) {
if ($group->mapCode($code) === $code) {
unset($langs[$index]);
}
}
}
if ($threshold) {
$stats = MessageGroupStats::forGroup($groupId);
foreach ($langs as $index => $code) {
if (!isset($stats[$code])) {
unset($langs[$index]);
continue;
}
$total = $stats[$code][MessageGroupStats::TOTAL];
$translated = $stats[$code][MessageGroupStats::TRANSLATED];
if ($translated / $total * 100 < $threshold) {
unset($langs[$index]);
}
}
}
// Filter out unchanged languages from requested languages
if (is_array($changeFilter)) {
$langs = array_intersect($langs, array_keys($changeFilter[$groupId]));
}
if (!count($langs)) {
continue;
}
$this->output("Exporting {$groupId}...\n");
$ffs = $group->getFFS();
$ffs->setWritePath($target);
$sourceLanguage = $group->getSourceLanguage();
$collection = $group->initCollection($sourceLanguage);
$definitionFile = false;
if ($this->hasOption('ppgettext') && $ffs instanceof GettextFFS) {
global $wgMaxShellMemory, $wgTranslateGroupRoot;
// Need more shell memory for msgmerge.
$wgMaxShellMemory = 402400;
$path = $group->getSourceFilePath($sourceLanguage);
$definitionFile = str_replace($wgTranslateGroupRoot, $this->getOption('ppgettext'), $path);
}
$whitelist = $group->getTranslatableLanguages();
foreach ($langs as $lang) {
// Do not export languges that are blacklisted (or not whitelisted).
// Also check that whitelist is not null, which means that all
// languages are allowed for translation and export.
if (is_array($whitelist) && !isset($whitelist[$lang])) {
continue;
}
$collection->resetForNewLanguage($lang);
$collection->loadTranslations();
// Don't export ignored, unless it is the source language
// or message documentation
global $wgTranslateDocumentationLanguageCode;
if ($lang !== $wgTranslateDocumentationLanguageCode && $lang !== $sourceLanguage) {
$collection->filter('ignored');
}
if ($noFuzzy) {
$collection->filter('fuzzy');
}
$ffs->write($collection);
// Do post processing if requested.
if ($definitionFile) {
if (is_file($definitionFile)) {
$targetFileName = $ffs->getWritePath() . "/" . $group->getTargetFilename($collection->code);
$cmd = "msgmerge --quiet " . $noLocation . "--output-file=" . $targetFileName . ' ' . $targetFileName . ' ' . $definitionFile;
wfShellExec($cmd, $ret);
// Report on errors.
if ($ret) {
$this->error("ERROR: {$ret}");
}
} else {
$this->error("{$definitionFile} does not exist.", 1);
}
}
}
}
}
public function execute( $parameters ) {
$this->setHeaders();
global $wgRequest, $wgOut, $wgUser;
$this->user = $wgUser;
$request = $wgRequest;
$target = $wgRequest->getText( 'target', $parameters );
$revision = $wgRequest->getInt( 'revision', 0 );
$action = $request->getVal( 'do' );
// No specific page or invalid input
$title = Title::newFromText( $target );
if ( !$title ) {
if ( $target !== '' ) {
$wgOut->addWikiMsg( 'tpt-badtitle' );
} else {
$this->listPages();
}
return;
}
// Check permissions
if ( !$this->user->isAllowed( 'pagetranslation' ) ) {
$wgOut->permissionRequired( 'pagetranslation' );
return;
}
// Check permissions
if ( $wgRequest->wasPosted() && !$this->user->matchEditToken( $wgRequest->getText( 'token' ) ) ) {
self::superDebug( __METHOD__, "token failure", $this->user );
$wgOut->permissionRequired( 'pagetranslation' );
return;
}
// We are processing some specific page
if ( !$title->exists() ) {
$wgOut->addWikiMsg( 'tpt-nosuchpage', $title->getPrefixedText() );
return;
}
if ( $action === 'discourage' || $action === 'encourage' ) {
$id = TranslatablePage::getMessageGroupIdFromTitle( $title );
$dbw = wfGetDB( DB_MASTER );
$table = 'translate_groupreviews';
$row = array(
'tgr_group' => $id,
'tgr_lang' => '*priority',
'tgr_state' => 'discouraged',
);
if ( $action === 'encourage' ) {
$dbw->delete( $table, $row, __METHOD__ );
} else {
$index = array( 'tgr_group', 'tgr_lang' );
$dbw->replace( $table, array( $index ), $row, __METHOD__ );
}
$this->listPages();
$group = MessageGroups::getGroup( $id );
$parents = MessageGroups::getParentGroups( $group );
MessageGroupStats::clearGroup( $parents );
return;
}
if ( $action === 'unmark' ) {
$page = TranslatablePage::newFromTitle( $title );
$page->removeTags();
$page->getTitle()->invalidateCache();
$logger = new LogPage( 'pagetranslation' );
$params = array( 'user' => $wgUser->getName() );
$logger->addEntry( 'unmark', $page->getTitle(), null, array( serialize( $params ) ) );
$wgOut->addWikiMsg( 'tpt-unmarked', $title->getPrefixedText() );
self::superDebug( __METHOD__, "unmarked page", $this->user, $title );
return;
}
if ( $revision === 0 ) {
// Get the latest revision
$revision = intval( $title->getLatestRevID() );
}
$page = TranslatablePage::newFromRevision( $title, $revision );
if ( !$page instanceof TranslatablePage ) {
$wgOut->addWikiMsg( 'tpt-notsuitable', $title->getPrefixedText(), $revision );
return;
}
if ( $revision !== intval( $title->getLatestRevID() ) ) {
// We do want to notify the reviewer if the underlying page changes during review
$target = $title->getFullUrl( array( 'oldid' => $revision ) );
$link = "<span class='plainlinks'>[$target $revision]</span>";
$wgOut->addWikiMsg( 'tpt-oldrevision', $title->getPrefixedText(), $link );
self::superDebug( __METHOD__, "revision mismatch while marking", $this->user, $title, $revision, intval( $title->getLatestRevID() ) );
return;
}
$lastrev = $page->getMarkedTag();
if ( $lastrev !== false && $lastrev === $revision ) {
$wgOut->addWikiMsg( 'tpt-already-marked' );
$this->listPages();
return;
}
// This will modify the sections to include name property
$error = false;
$sections = $this->checkInput( $page, $error );
// Non-fatal error which prevents saving
if ( $error === false && $wgRequest->wasPosted() ) {
$err = $this->markForTranslation( $page, $sections );
if ( $err ) {
call_user_func_array( array( $wgOut, 'addWikiMsg' ), $err );
} else {
$this->showSuccess( $page );
$this->listPages();
}
return;
}
self::superDebug( __METHOD__, "marking page", $this->user, $title, $revision );
$this->showPage( $page, $sections );
}
