public function execute()
{
$groupIds = explode(',', trim($this->getOption('group')));
$groupIds = MessageGroups::expandWildcards($groupIds);
$groups = MessageGroups::getGroupsById($groupIds);
if (!count($groups)) {
$this->error("ESG2: No valid message groups identified.", 1);
}
$start = $this->getOption('start') ? strtotime($this->getOption('start')) : false;
$end = $this->getOption('end') ? strtotime($this->getOption('end')) : false;
$this->output("Conflict times: " . wfTimestamp(TS_ISO_8601, $start) . " - " . wfTimestamp(TS_ISO_8601, $end) . "\n");
$codes = array_filter(array_map('trim', explode(',', $this->getOption('lang'))));
$supportedCodes = array_keys(TranslateUtils::getLanguageNames('en'));
ksort($supportedCodes);
if ($codes[0] === '*') {
$codes = $supportedCodes;
}
/** @var FileBasedMessageGroup $group */
foreach ($groups as $groupId => &$group) {
if ($group->isMeta()) {
$this->output("Skipping meta message group {$groupId}.\n");
continue;
}
$this->output("{$group->getLabel()} ", $group);
foreach ($codes as $code) {
// No sync possible for unsupported language codes.
if (!in_array($code, $supportedCodes)) {
$this->output("Unsupported code " . $code . ": skipping.\n");
continue;
}
$file = $group->getSourceFilePath($code);
if (!$file) {
continue;
}
if (!file_exists($file)) {
continue;
}
$cs = new ChangeSyncer($group, $this);
$cs->setProgressCallback(array($this, 'myOutput'));
$cs->interactive = !$this->hasOption('noask');
$cs->nocolor = $this->hasOption('nocolor');
$cs->norc = $this->hasOption('norc');
# @todo FIXME: Make this auto detect.
# Guess last modified date of the file from either git, svn or filesystem
if ($this->hasOption('git')) {
$ts = $cs->getTimestampsFromGit($file);
} else {
$ts = $cs->getTimestampsFromSvn($file);
}
if (!$ts) {
$ts = $cs->getTimestampsFromFs($file);
}
$this->output("Modify time for {$code}: " . wfTimestamp(TS_ISO_8601, $ts) . "\n");
$cs->checkConflicts($code, $start, $end, $ts);
}
unset($group);
}
// Print timestamp if the user wants to store it
$this->output(wfTimestamp(TS_RFC2822) . "\n");
}
/**
* Finds changes in external sources compared to wiki state.
*
* The returned array is as following:
* - First level is indexed by language code
* - Second level is indexed by change type:
* - - addition (new message in the file)
* - - deletion (message in wiki not present in the file)
* - - change (difference in content)
* - Third level is a list of changes
* - Fourth level is change properties
* - - key (the message key)
* - - content (the message content in external source, null for deletions)
*
* @param FileBasedMessageGroup $group
* @param array|string $languages
* @throws MWException
* @return array array[language code][change type] = change.
*/
public function processGroup(FileBasedMessageGroup $group, $languages)
{
$this->changes = array();
if ($languages === self::ALL_LANGUAGES) {
$languages = $group->getTranslatableLanguages();
// This means all languages
if ($languages === null) {
$languages = TranslateUtils::getLanguageNames('en');
}
$languages = array_keys($languages);
} elseif (!is_array($languages)) {
throw new MWException('Invalid input given for $languages');
}
// Process the source language before others
$sourceLanguage = $group->getSourceLanguage();
$index = array_search($sourceLanguage, $languages);
if ($index !== false) {
unset($languages[$index]);
$this->processLanguage($group, $sourceLanguage);
}
foreach ($languages as $code) {
$this->processLanguage($group, $code);
}
return $this->changes;
}
function run()
{
global $wgTranslateDocumentationLanguageCode;
$title = $this->title;
$params = $this->params;
$user = FuzzyBot::getUser();
$flags = EDIT_DEFER_UPDATES | EDIT_FORCE_BOT;
$wikiPage = WikiPage::factory($title);
$summary = wfMessage('translate-manage-import-summary')->inContentLanguage()->plain();
$content = ContentHandler::makeContent($params['content'], $title);
$wikiPage->doEditContent($content, $summary, $flags, false, $user);
// NOTE: message documentation is excluded from fuzzying!
if ($params['fuzzy']) {
$handle = new MessageHandle($title);
$key = $handle->getKey();
$languages = TranslateUtils::getLanguageNames('en');
unset($languages[$wgTranslateDocumentationLanguageCode]);
$languages = array_keys($languages);
$dbw = wfGetDB(DB_MASTER);
$fields = array('page_id', 'page_latest');
$conds = array('page_namespace' => $title->getNamespace());
$pages = array();
foreach ($languages as $code) {
$otherTitle = Title::makeTitleSafe($title->getNamespace(), "{$key}/{$code}");
$pages[$otherTitle->getDBKey()] = true;
}
unset($pages[$title->getDBKey()]);
if (count($pages) === 0) {
return true;
}
$conds['page_title'] = array_keys($pages);
$res = $dbw->select('page', $fields, $conds, __METHOD__);
$inserts = array();
foreach ($res as $row) {
$inserts[] = array('rt_type' => RevTag::getType('fuzzy'), 'rt_page' => $row->page_id, 'rt_revision' => $row->page_latest);
}
$dbw->replace('revtag', array(array('rt_type', 'rt_page', 'rt_revision')), $inserts, __METHOD__);
}
return true;
}
/**
* Returns a localised language name.
* @param string $code Language code.
* @param string $language Language code of language the the name should be in.
* @return string Best-effort localisation of wanted language name.
*/
public static function getLanguageName($code, $language = 'en')
{
$languages = TranslateUtils::getLanguageNames($language);
if (isset($languages[$code])) {
return $languages[$code];
} else {
return $code;
}
}
/**
* Get all the translatable languages for a group, considering the whitelisting
* and blacklisting.
* @return array|null The language codes as array keys.
*/
public function getTranslatableLanguages()
{
$groupConfiguration = $this->getConfiguration();
if (!isset($groupConfiguration['LANGUAGES'])) {
// No LANGUAGES section in the configuration.
return null;
}
$lists = $groupConfiguration['LANGUAGES'];
$codes = array();
// The list of languages to return
if (isset($lists['blacklist'])) {
$blacklist = $lists['blacklist'];
if (is_array($blacklist)) {
$codes = array_flip(array_keys(TranslateUtils::getLanguageNames('en')));
foreach ($blacklist as $code) {
unset($codes[$code]);
}
} else {
// All languages blacklisted. This is very rare but not impossible.
$codes = array();
}
}
$whitelist = array();
if (isset($lists['whitelist'])) {
$whitelist = $lists['whitelist'];
if ($whitelist === "*") {
// All languages whitelisted
return null;
}
}
foreach ($whitelist as $code) {
$codes[$code] = true;
}
return $codes;
}
/**
* Returns array of errors in the form parameters.
*/
protected function getFormErrors()
{
$errors = array();
$codes = TranslateUtils::getLanguageNames('en');
if (!$this->options['language'] || !isset($codes[$this->options['language']])) {
$errors['language'] = $this->msg('translate-page-no-such-language')->text();
$this->options['language'] = $this->defaults['language'];
}
if (!$this->group instanceof MessageGroup) {
$errors['group'] = $this->msg('translate-page-no-such-group')->text();
$this->options['group'] = $this->defaults['group'];
} else {
$languages = $this->group->getTranslatableLanguages();
if ($languages !== null && !isset($languages[$this->options['language']])) {
$errors['language'] = $this->msg('translate-language-disabled')->text();
}
}
return $errors;
}
/**
* @param MessageGroup $group
* @param array[] $stats
* @return array[]
*/
protected static function forGroupInternal($group, $stats = array())
{
$id = $group->getId();
$res = self::selectRowsIdLang($id, null);
$stats = self::extractResults($res, $stats);
# Go over each language filling missing entries
$languages = array_keys(TranslateUtils::getLanguageNames('en'));
// This is for calculating things in correct order
sort($languages);
foreach ($languages as $code) {
if (isset($stats[$id][$code])) {
continue;
}
$stats[$id][$code] = self::forItemInternal($stats, $group, $code);
}
// This is for sorting the values added later in correct order
foreach (array_keys($stats) as $key) {
ksort($stats[$key]);
}
return $stats;
}
/**
* @param $code
* @param $params
* @return string
*/
protected function getMainColumnCell($code, $params)
{
if (!isset($this->names)) {
$this->names = TranslateUtils::getLanguageNames($this->getLanguage()->getCode());
$this->translate = SpecialPage::getTitleFor('Translate');
}
$queryParameters = $params + array('group' => $this->target, 'language' => $code);
if (isset($this->names[$code])) {
$text = htmlspecialchars("{$code}: {$this->names[$code]}");
} else {
$text = htmlspecialchars($code);
}
$link = Linker::link($this->translate, $text, array(), $queryParameters);
return Html::rawElement('td', array(), $link);
}
public function execute()
{
global $wgTranslateMessageNamespaces;
ini_set('memory_limit', -1);
// How many messages per language to use.
// Language is skipped if it has less than 1000 translations.
$messages = 5000;
$languages = TranslateUtils::getLanguageNames('en');
$cache = wfGetCache(CACHE_DB);
$key = wfMemckey(__METHOD__, $messages);
$pages = $cache->get($key);
if (!is_array($pages)) {
$dbr = wfGetDB(DB_SLAVE);
$conds = array();
$conds[] = 'page_title' . $dbr->buildLike($dbr->anyString(), '/', $dbr->anyString());
$conds['page_namespace'] = $wgTranslateMessageNamespaces;
echo "Before query\n";
$res = $dbr->select(array('page'), array('page_title, page_id'), $conds, __METHOD__);
echo "After query\n";
$total = $res->numRows();
$index = 0;
foreach ($res as $row) {
$index++;
$code = substr($row->page_title, strrpos($row->page_title, '/') + 1);
if (isset($languages[$code])) {
$pages[$code][] = $row->page_id;
}
if ($index % 10000 === 0) {
$progress = number_format($index / $total * 100, 2);
echo "{$progress}%\n";
}
}
echo "\n";
foreach (array_keys($pages) as $code) {
if (count($pages[$code]) > $messages) {
$pages[$code] = array_slice($pages[$code], 0, $messages);
}
$pages[$code] = implode('|', $pages[$code]);
}
echo "After code map\n";
ksort($pages);
echo "After sort map\n";
$cache->set($key, $pages, 3600 * 24);
echo "After set map\n";
}
unset($pages['qqq']);
unset($pages['de-formal']);
unset($pages['nl-informal']);
unset($pages['en-gb']);
$pids = array();
$threads = 2;
foreach ($pages as $code => $pageids) {
$pid = $threads > 1 ? pcntl_fork() : -1;
if ($pid === 0) {
// Child, reseed because there is no bug in PHP:
// http://bugs.php.net/bug.php?id=42465
mt_srand(getmypid());
$this->analyzeLanguage($code, $pageids);
exit;
} elseif ($pid === -1) {
// Fork failed or one thread, do it serialized
$this->analyzeLanguage($code, $pageids);
} else {
// Main thread
$pids[] = $pid;
}
// If we hit the thread limit, wait for any child to finish.
if (count($pids) >= $threads) {
$status = 0;
$pid = pcntl_wait($status);
unset($pids[$pid]);
}
}
foreach ($pids as $pid) {
$status = 0;
pcntl_waitpid($pid, $status);
}
$this->output("Combining languages\n");
$huge = array();
foreach (glob("temp-*.json") as $file) {
$contents = file_get_contents($file);
$json = FormatJson::decode($contents, true);
$huge = array_merge($json, $huge);
$huge['data'] = array_merge($json['data'], $huge['data']);
}
$json = FormatJson::encode($huge, true, FormatJson::ALL_OK);
file_put_contents('translatewiki.net.json', $json);
}
protected function setUp()
{
parent::setUp();
$this->group = MessageGroupBase::factory($this->groupConfiguration);
$this->codes = array_flip(array_keys(TranslateUtils::getLanguageNames('en')));
}
/**
* Returns the default target language for messages.
* @return Language
*/
protected function getTargetLanguage()
{
$ui = $this->getLanguage();
$source = $this->getSourceLanguage();
if ($ui->getCode() !== $source->getCode()) {
return $ui;
}
$options = FormatJson::decode($this->getUser()->getOption('translate-sandbox'), true);
$supported = TranslateUtils::getLanguageNames('en');
if (isset($options['languages'])) {
foreach ($options['languages'] as $code) {
if (!isset($supported[$code])) {
continue;
}
if ($code !== $source->getCode()) {
return Language::factory($code);
}
}
}
// User has not chosen any valid language. Pick the source.
return Language::factory($source->getCode());
}
/**
* @param $code
* @param $params
* @return string
*/
protected function getMainColumnCell($code, $params)
{
if (!isset($this->names)) {
global $wgLang;
$this->names = TranslateUtils::getLanguageNames($wgLang->getCode());
$this->translate = SpecialPage::getTitleFor('Translate');
}
$queryParameters = $params + array('group' => $this->target, 'language' => $code);
$text = htmlspecialchars("{$code}: {$this->names[$code]}");
$linker = class_exists('DummyLinker') ? new DummyLinker() : new Linker();
$link = $linker->link($this->translate, $text, array(), $queryParameters);
return Html::rawElement('td', array(), $link);
}
