public function execute() { $groupIds = explode(',', trim($this->getOption('group'))); $groupIds = MessageGroups::expandWildcards($groupIds); $groups = MessageGroups::getGroupsById($groupIds); if (!count($groups)) { $this->error("ESG2: No valid message groups identified.", 1); } $start = $this->getOption('start') ? strtotime($this->getOption('start')) : false; $end = $this->getOption('end') ? strtotime($this->getOption('end')) : false; $this->output("Conflict times: " . wfTimestamp(TS_ISO_8601, $start) . " - " . wfTimestamp(TS_ISO_8601, $end) . "\n"); $codes = array_filter(array_map('trim', explode(',', $this->getOption('lang')))); $supportedCodes = array_keys(TranslateUtils::getLanguageNames('en')); ksort($supportedCodes); if ($codes[0] === '*') { $codes = $supportedCodes; } /** @var FileBasedMessageGroup $group */ foreach ($groups as $groupId => &$group) { if ($group->isMeta()) { $this->output("Skipping meta message group {$groupId}.\n"); continue; } $this->output("{$group->getLabel()} ", $group); foreach ($codes as $code) { // No sync possible for unsupported language codes. if (!in_array($code, $supportedCodes)) { $this->output("Unsupported code " . $code . ": skipping.\n"); continue; } $file = $group->getSourceFilePath($code); if (!$file) { continue; } if (!file_exists($file)) { continue; } $cs = new ChangeSyncer($group, $this); $cs->setProgressCallback(array($this, 'myOutput')); $cs->interactive = !$this->hasOption('noask'); $cs->nocolor = $this->hasOption('nocolor'); $cs->norc = $this->hasOption('norc'); # @todo FIXME: Make this auto detect. # Guess last modified date of the file from either git, svn or filesystem if ($this->hasOption('git')) { $ts = $cs->getTimestampsFromGit($file); } else { $ts = $cs->getTimestampsFromSvn($file); } if (!$ts) { $ts = $cs->getTimestampsFromFs($file); } $this->output("Modify time for {$code}: " . wfTimestamp(TS_ISO_8601, $ts) . "\n"); $cs->checkConflicts($code, $start, $end, $ts); } unset($group); } // Print timestamp if the user wants to store it $this->output(wfTimestamp(TS_RFC2822) . "\n"); }
/** * Finds changes in external sources compared to wiki state. * * The returned array is as following: * - First level is indexed by language code * - Second level is indexed by change type: * - - addition (new message in the file) * - - deletion (message in wiki not present in the file) * - - change (difference in content) * - Third level is a list of changes * - Fourth level is change properties * - - key (the message key) * - - content (the message content in external source, null for deletions) * * @param FileBasedMessageGroup $group * @param array|string $languages * @throws MWException * @return array array[language code][change type] = change. */ public function processGroup(FileBasedMessageGroup $group, $languages) { $this->changes = array(); if ($languages === self::ALL_LANGUAGES) { $languages = $group->getTranslatableLanguages(); // This means all languages if ($languages === null) { $languages = TranslateUtils::getLanguageNames('en'); } $languages = array_keys($languages); } elseif (!is_array($languages)) { throw new MWException('Invalid input given for $languages'); } // Process the source language before others $sourceLanguage = $group->getSourceLanguage(); $index = array_search($sourceLanguage, $languages); if ($index !== false) { unset($languages[$index]); $this->processLanguage($group, $sourceLanguage); } foreach ($languages as $code) { $this->processLanguage($group, $code); } return $this->changes; }
function run() { global $wgTranslateDocumentationLanguageCode; $title = $this->title; $params = $this->params; $user = FuzzyBot::getUser(); $flags = EDIT_DEFER_UPDATES | EDIT_FORCE_BOT; $wikiPage = WikiPage::factory($title); $summary = wfMessage('translate-manage-import-summary')->inContentLanguage()->plain(); $content = ContentHandler::makeContent($params['content'], $title); $wikiPage->doEditContent($content, $summary, $flags, false, $user); // NOTE: message documentation is excluded from fuzzying! if ($params['fuzzy']) { $handle = new MessageHandle($title); $key = $handle->getKey(); $languages = TranslateUtils::getLanguageNames('en'); unset($languages[$wgTranslateDocumentationLanguageCode]); $languages = array_keys($languages); $dbw = wfGetDB(DB_MASTER); $fields = array('page_id', 'page_latest'); $conds = array('page_namespace' => $title->getNamespace()); $pages = array(); foreach ($languages as $code) { $otherTitle = Title::makeTitleSafe($title->getNamespace(), "{$key}/{$code}"); $pages[$otherTitle->getDBKey()] = true; } unset($pages[$title->getDBKey()]); if (count($pages) === 0) { return true; } $conds['page_title'] = array_keys($pages); $res = $dbw->select('page', $fields, $conds, __METHOD__); $inserts = array(); foreach ($res as $row) { $inserts[] = array('rt_type' => RevTag::getType('fuzzy'), 'rt_page' => $row->page_id, 'rt_revision' => $row->page_latest); } $dbw->replace('revtag', array(array('rt_type', 'rt_page', 'rt_revision')), $inserts, __METHOD__); } return true; }
/** * Returns a localised language name. * @param string $code Language code. * @param string $language Language code of language the the name should be in. * @return string Best-effort localisation of wanted language name. */ public static function getLanguageName($code, $language = 'en') { $languages = TranslateUtils::getLanguageNames($language); if (isset($languages[$code])) { return $languages[$code]; } else { return $code; } }
/** * Get all the translatable languages for a group, considering the whitelisting * and blacklisting. * @return array|null The language codes as array keys. */ public function getTranslatableLanguages() { $groupConfiguration = $this->getConfiguration(); if (!isset($groupConfiguration['LANGUAGES'])) { // No LANGUAGES section in the configuration. return null; } $lists = $groupConfiguration['LANGUAGES']; $codes = array(); // The list of languages to return if (isset($lists['blacklist'])) { $blacklist = $lists['blacklist']; if (is_array($blacklist)) { $codes = array_flip(array_keys(TranslateUtils::getLanguageNames('en'))); foreach ($blacklist as $code) { unset($codes[$code]); } } else { // All languages blacklisted. This is very rare but not impossible. $codes = array(); } } $whitelist = array(); if (isset($lists['whitelist'])) { $whitelist = $lists['whitelist']; if ($whitelist === "*") { // All languages whitelisted return null; } } foreach ($whitelist as $code) { $codes[$code] = true; } return $codes; }
/** * Returns array of errors in the form parameters. */ protected function getFormErrors() { $errors = array(); $codes = TranslateUtils::getLanguageNames('en'); if (!$this->options['language'] || !isset($codes[$this->options['language']])) { $errors['language'] = $this->msg('translate-page-no-such-language')->text(); $this->options['language'] = $this->defaults['language']; } if (!$this->group instanceof MessageGroup) { $errors['group'] = $this->msg('translate-page-no-such-group')->text(); $this->options['group'] = $this->defaults['group']; } else { $languages = $this->group->getTranslatableLanguages(); if ($languages !== null && !isset($languages[$this->options['language']])) { $errors['language'] = $this->msg('translate-language-disabled')->text(); } } return $errors; }
/** * @param MessageGroup $group * @param array[] $stats * @return array[] */ protected static function forGroupInternal($group, $stats = array()) { $id = $group->getId(); $res = self::selectRowsIdLang($id, null); $stats = self::extractResults($res, $stats); # Go over each language filling missing entries $languages = array_keys(TranslateUtils::getLanguageNames('en')); // This is for calculating things in correct order sort($languages); foreach ($languages as $code) { if (isset($stats[$id][$code])) { continue; } $stats[$id][$code] = self::forItemInternal($stats, $group, $code); } // This is for sorting the values added later in correct order foreach (array_keys($stats) as $key) { ksort($stats[$key]); } return $stats; }
/** * @param $code * @param $params * @return string */ protected function getMainColumnCell($code, $params) { if (!isset($this->names)) { $this->names = TranslateUtils::getLanguageNames($this->getLanguage()->getCode()); $this->translate = SpecialPage::getTitleFor('Translate'); } $queryParameters = $params + array('group' => $this->target, 'language' => $code); if (isset($this->names[$code])) { $text = htmlspecialchars("{$code}: {$this->names[$code]}"); } else { $text = htmlspecialchars($code); } $link = Linker::link($this->translate, $text, array(), $queryParameters); return Html::rawElement('td', array(), $link); }
public function execute() { global $wgTranslateMessageNamespaces; ini_set('memory_limit', -1); // How many messages per language to use. // Language is skipped if it has less than 1000 translations. $messages = 5000; $languages = TranslateUtils::getLanguageNames('en'); $cache = wfGetCache(CACHE_DB); $key = wfMemckey(__METHOD__, $messages); $pages = $cache->get($key); if (!is_array($pages)) { $dbr = wfGetDB(DB_SLAVE); $conds = array(); $conds[] = 'page_title' . $dbr->buildLike($dbr->anyString(), '/', $dbr->anyString()); $conds['page_namespace'] = $wgTranslateMessageNamespaces; echo "Before query\n"; $res = $dbr->select(array('page'), array('page_title, page_id'), $conds, __METHOD__); echo "After query\n"; $total = $res->numRows(); $index = 0; foreach ($res as $row) { $index++; $code = substr($row->page_title, strrpos($row->page_title, '/') + 1); if (isset($languages[$code])) { $pages[$code][] = $row->page_id; } if ($index % 10000 === 0) { $progress = number_format($index / $total * 100, 2); echo "{$progress}%\n"; } } echo "\n"; foreach (array_keys($pages) as $code) { if (count($pages[$code]) > $messages) { $pages[$code] = array_slice($pages[$code], 0, $messages); } $pages[$code] = implode('|', $pages[$code]); } echo "After code map\n"; ksort($pages); echo "After sort map\n"; $cache->set($key, $pages, 3600 * 24); echo "After set map\n"; } unset($pages['qqq']); unset($pages['de-formal']); unset($pages['nl-informal']); unset($pages['en-gb']); $pids = array(); $threads = 2; foreach ($pages as $code => $pageids) { $pid = $threads > 1 ? pcntl_fork() : -1; if ($pid === 0) { // Child, reseed because there is no bug in PHP: // http://bugs.php.net/bug.php?id=42465 mt_srand(getmypid()); $this->analyzeLanguage($code, $pageids); exit; } elseif ($pid === -1) { // Fork failed or one thread, do it serialized $this->analyzeLanguage($code, $pageids); } else { // Main thread $pids[] = $pid; } // If we hit the thread limit, wait for any child to finish. if (count($pids) >= $threads) { $status = 0; $pid = pcntl_wait($status); unset($pids[$pid]); } } foreach ($pids as $pid) { $status = 0; pcntl_waitpid($pid, $status); } $this->output("Combining languages\n"); $huge = array(); foreach (glob("temp-*.json") as $file) { $contents = file_get_contents($file); $json = FormatJson::decode($contents, true); $huge = array_merge($json, $huge); $huge['data'] = array_merge($json['data'], $huge['data']); } $json = FormatJson::encode($huge, true, FormatJson::ALL_OK); file_put_contents('translatewiki.net.json', $json); }
protected function setUp() { parent::setUp(); $this->group = MessageGroupBase::factory($this->groupConfiguration); $this->codes = array_flip(array_keys(TranslateUtils::getLanguageNames('en'))); }
/** * Returns the default target language for messages. * @return Language */ protected function getTargetLanguage() { $ui = $this->getLanguage(); $source = $this->getSourceLanguage(); if ($ui->getCode() !== $source->getCode()) { return $ui; } $options = FormatJson::decode($this->getUser()->getOption('translate-sandbox'), true); $supported = TranslateUtils::getLanguageNames('en'); if (isset($options['languages'])) { foreach ($options['languages'] as $code) { if (!isset($supported[$code])) { continue; } if ($code !== $source->getCode()) { return Language::factory($code); } } } // User has not chosen any valid language. Pick the source. return Language::factory($source->getCode()); }
/** * @param $code * @param $params * @return string */ protected function getMainColumnCell($code, $params) { if (!isset($this->names)) { global $wgLang; $this->names = TranslateUtils::getLanguageNames($wgLang->getCode()); $this->translate = SpecialPage::getTitleFor('Translate'); } $queryParameters = $params + array('group' => $this->target, 'language' => $code); $text = htmlspecialchars("{$code}: {$this->names[$code]}"); $linker = class_exists('DummyLinker') ? new DummyLinker() : new Linker(); $link = $linker->link($this->translate, $text, array(), $queryParameters); return Html::rawElement('td', array(), $link); }