public function testParsing() { $title = Title::newFromText('MediaWiki:Ugakey/nl'); $page = WikiPage::factory($title); $content = ContentHandler::makeContent('$1 van $2', $title); $status = $page->doEditContent($content, __METHOD__); $value = $status->getValue(); /** * @var Revision $rev */ $rev = $value['revision']; $revision = $rev->getId(); $dbw = wfGetDB(DB_MASTER); $conds = array('rt_page' => $title->getArticleID(), 'rt_type' => RevTag::getType('fuzzy'), 'rt_revision' => $revision); $index = array_keys($conds); $dbw->replace('revtag', array($index), $conds, __METHOD__); $handle = new MessageHandle($title); $this->assertTrue($handle->isValid(), 'Message is known'); $this->assertTrue($handle->isFuzzy(), 'Message is fuzzy after database fuzzying'); // Update the translation without the fuzzy string $content = ContentHandler::makeContent('$1 van $2', $title); $page->doEditContent($content, __METHOD__); $this->assertFalse($handle->isFuzzy(), 'Message is unfuzzy after edit'); $content = ContentHandler::makeContent('!!FUZZY!!$1 van $2', $title); $page->doEditContent($content, __METHOD__); $this->assertTrue($handle->isFuzzy(), 'Message is fuzzy after manual fuzzying'); // Update the translation without the fuzzy string $content = ContentHandler::makeContent('$1 van $2', $title); $page->doEditContent($content, __METHOD__); $this->assertFalse($handle->isFuzzy(), 'Message is unfuzzy after edit'); }
/** * Determines the schema version. * * @return int */ public static function checkSchema() { if ( self::$schema !== false ) { return self::$schema; } else { $dbr = wfGetDB( DB_SLAVE ); if ( $dbr->tableExists( 'revtag_type' ) ) { return self::$schema = 1; } else { return self::$schema = 2; } } }
function run() { global $wgTranslateDocumentationLanguageCode; $title = $this->title; $params = $this->params; $user = FuzzyBot::getUser(); $flags = EDIT_DEFER_UPDATES | EDIT_FORCE_BOT; $wikiPage = WikiPage::factory($title); $summary = wfMessage('translate-manage-import-summary')->inContentLanguage()->plain(); $content = ContentHandler::makeContent($params['content'], $title); $wikiPage->doEditContent($content, $summary, $flags, false, $user); // NOTE: message documentation is excluded from fuzzying! if ($params['fuzzy']) { $handle = new MessageHandle($title); $key = $handle->getKey(); $languages = TranslateUtils::getLanguageNames('en'); unset($languages[$wgTranslateDocumentationLanguageCode]); $languages = array_keys($languages); $dbw = wfGetDB(DB_MASTER); $fields = array('page_id', 'page_latest'); $conds = array('page_namespace' => $title->getNamespace()); $pages = array(); foreach ($languages as $code) { $otherTitle = Title::makeTitleSafe($title->getNamespace(), "{$key}/{$code}"); $pages[$otherTitle->getDBKey()] = true; } unset($pages[$title->getDBKey()]); if (count($pages) === 0) { return true; } $conds['page_title'] = array_keys($pages); $res = $dbw->select('page', $fields, $conds, __METHOD__); $inserts = array(); foreach ($res as $row) { $inserts[] = array('rt_type' => RevTag::getType('fuzzy'), 'rt_page' => $row->page_id, 'rt_revision' => $row->page_latest); } $dbw->replace('revtag', array(array('rt_type', 'rt_page', 'rt_revision')), $inserts, __METHOD__); } return true; }
public function getData() { $db = wfGetDB(DB_MASTER); $conds = array('rt_page' => $this->handle->getTitle()->getArticleID(), 'rt_type' => RevTag::getType('tp:transver')); $options = array('ORDER BY' => 'rt_revision DESC'); $translationRevision = $db->selectField('revtag', 'rt_value', $conds, __METHOD__, $options); if ($translationRevision === false) { throw new TranslationHelperException("No definition revision recorded"); } $definitionTitle = Title::makeTitleSafe($this->handle->getTitle()->getNamespace(), $this->handle->getKey() . '/' . $this->group->getSourceLanguage()); if (!$definitionTitle || !$definitionTitle->exists()) { throw new TranslationHelperException("Definition page doesn't exist"); } // Using newFromId instead of newFromTitle, because the page might have been renamed $oldrev = Revision::newFromId($translationRevision); if (!$oldrev) { throw new TranslationHelperException("Old definition version doesn't exist anymore"); } $oldContent = $oldrev->getContent(); $newContent = $this->getDefinitionContent(); if (!$oldContent) { throw new TranslationHelperException("Old definition version doesn't exist anymore"); } if (!$oldContent instanceof WikitextContent || !$newContent instanceof WikitextContent) { throw new TranslationHelperException('Can only work on Wikitext content'); } if ($oldContent->equals($newContent)) { throw new TranslationHelperException('No changes'); } $diff = new DifferenceEngine($this->context); if (method_exists('DifferenceEngine', 'setTextLanguage')) { $diff->setTextLanguage($this->group->getSourceLanguage()); } $diff->setContent($oldContent, $newContent); $diff->setReducedLineNumbers(); $diff->showDiffStyle(); $html = $diff->getDiff($this->context->msg('tpt-diff-old')->escaped(), $this->context->msg('tpt-diff-new')->escaped()); return array('value_old' => $oldContent->getNativeData(), 'value_new' => $newContent->getNativeData(), 'revisionid_old' => $oldrev->getId(), 'revisionid_new' => $definitionTitle->getLatestRevId(), 'language' => $this->group->getSourceLanguage(), 'html' => $html); }
public function execute() { global $wgTranslateMessageNamespaces; $namespace = $this->getOption('namespace', $wgTranslateMessageNamespaces); if (is_string($namespace)) { if (!MWNamespace::exists($namespace)) { $namespace = MWNamespace::getCanonicalIndex($namespace); if ($namespace === null) { $this->error('Bad namespace', true); } } } $db = wfGetDB(DB_MASTER); $tables = array('page', 'text', 'revision'); $fields = array('page_id', 'page_title', 'page_namespace', 'rev_id', 'old_text', 'old_flags'); $conds = array('page_latest = rev_id', 'old_id = rev_text_id', 'page_namespace' => $namespace); $limit = 100; $offset = 0; while (true) { $inserts = array(); $this->output('.', 0); $options = array('LIMIT' => $limit, 'OFFSET' => $offset); $res = $db->select($tables, $fields, $conds, __METHOD__, $options); if (!$res->numRows()) { break; } foreach ($res as $r) { $text = Revision::getRevisionText($r); if (strpos($text, TRANSLATE_FUZZY) !== false) { $inserts[] = array('rt_page' => $r->page_id, 'rt_revision' => $r->rev_id, 'rt_type' => RevTag::getType('fuzzy')); } } $offset += $limit; $db->replace('revtag', 'rt_type_page_revision', $inserts, __METHOD__); } }
/** * Loads existence and fuzzy state for given list of keys. * @param string[] $keys List of keys in database format. * @param int $dbtype One of DB_* constants. */ protected function loadInfo(array $keys, $dbtype = DB_SLAVE) { if ($this->dbInfo !== null) { return; } $this->dbInfo = array(); if (!count($keys)) { return; } $dbr = wfGetDB($dbtype); $tables = array('page', 'revtag'); $fields = array('page_namespace', 'page_title', 'rt_type'); $conds = $this->getTitleConds($dbr); $joins = array('revtag' => array('LEFT JOIN', array('page_id=rt_page', 'page_latest=rt_revision', 'rt_type' => RevTag::getType('fuzzy')))); $this->dbInfo = $dbr->select($tables, $fields, $conds, __METHOD__, array(), $joins); }
/** * This constructs the list of all groups from multiple different * sources. When possible, a cache dependency is created to automatically * recreate the cache when configuration changes. * @todo Reduce the ways of which messages can be added. Target is just * to have three ways: Yaml files, translatable pages and with the hook. * @todo In conjuction with the above, reduce the number of global * variables like wgTranslate#C and have the message groups specify * their own cache dependencies. */ protected static function loadGroupDefinitions() { global $wgTranslateAddMWExtensionGroups; global $wgEnablePageTranslation, $wgTranslateGroupFiles; global $wgTranslateAC, $wgTranslateEC, $wgTranslateCC; global $wgAutoloadClasses; global $wgTranslateWorkflowStates; $deps = array(); $deps[] = new GlobalDependency('wgTranslateAddMWExtensionGroups'); $deps[] = new GlobalDependency('wgEnablePageTranslation'); $deps[] = new GlobalDependency('wgTranslateGroupFiles'); $deps[] = new GlobalDependency('wgTranslateAC'); $deps[] = new GlobalDependency('wgTranslateEC'); $deps[] = new GlobalDependency('wgTranslateCC'); $deps[] = new GlobalDependency('wgTranslateExtensionDirectory'); $deps[] = new GlobalDependency('wgTranslateWorkflowStates'); $deps[] = new FileDependency(dirname(__FILE__) . '/groups/mediawiki-defines.txt'); $deps[] = new FileDependency(dirname(__FILE__) . '/groups/Wikia/extensions.txt'); $deps[] = new FileDependency(dirname(__FILE__) . '/groups/Toolserver/toolserver-textdomains.txt'); if ($wgTranslateAddMWExtensionGroups) { $a = new PremadeMediawikiExtensionGroups(); $a->addAll(); } if ($wgEnablePageTranslation) { $dbr = wfGetDB(DB_MASTER); $tables = array('page', 'revtag'); $vars = array('page_id', 'page_namespace', 'page_title'); $conds = array('page_id=rt_page', 'rt_type' => RevTag::getType('tp:mark')); $options = array('GROUP BY' => 'rt_page'); $res = $dbr->select($tables, $vars, $conds, __METHOD__, $options); foreach ($res as $r) { $title = Title::makeTitle($r->page_namespace, $r->page_title); $id = TranslatablePage::getMessageGroupIdFromTitle($title); $wgTranslateCC[$id] = new WikiPageMessageGroup($id, $title); $wgTranslateCC[$id]->setLabel($title->getPrefixedText()); } } if ($wgTranslateWorkflowStates) { $wgTranslateCC['translate-workflow-states'] = new WorkflowStatesMessageGroup(); } $autoload = array(); wfRunHooks('TranslatePostInitGroups', array(&$wgTranslateCC, &$deps, &$autoload)); foreach ($wgTranslateGroupFiles as $configFile) { wfDebug($configFile . "\n"); $deps[] = new FileDependency(realpath($configFile)); $fgroups = TranslateYaml::parseGroupFile($configFile); foreach ($fgroups as $id => $conf) { if (!empty($conf['AUTOLOAD']) && is_array($conf['AUTOLOAD'])) { $dir = dirname($configFile); foreach ($conf['AUTOLOAD'] as $class => $file) { // For this request and for caching. $wgAutoloadClasses[$class] = "{$dir}/{$file}"; $autoload[$class] = "{$dir}/{$file}"; } } $group = MessageGroupBase::factory($conf); $wgTranslateCC[$id] = $group; } } $key = wfMemckey('translate-groups'); $value = array('ac' => $wgTranslateAC, 'ec' => $wgTranslateEC, 'cc' => $wgTranslateCC, 'autoload' => $autoload); $wrapper = new DependencyWrapper($value, $deps); $wrapper->storeToCache(self::getCache(), $key, 60 * 60 * 2); wfDebug(__METHOD__ . "-end\n"); }
/** * Check if a title is marked as fuzzy. * @return bool If title is marked fuzzy. */ public function isFuzzy() { $dbr = wfGetDB( DB_SLAVE ); $tables = array( 'page', 'revtag' ); $field = 'rt_type'; $conds = array( 'page_namespace' => $this->title->getNamespace(), 'page_title' => $this->title->getDBkey(), 'rt_type' => RevTag::getType( 'fuzzy' ), 'page_id=rt_page', 'page_latest=rt_revision' ); $res = $dbr->selectField( $tables, $field, $conds, __METHOD__ ); return $res !== false; }
/** * Adds tag which identifies the revision of source message at that time. * This is used to show diff against current version of source message * when updating a translation. * Hook: Translate:newTranslation * @param MessageHandle $handle * @param int $revision * @param string $text * @param User $user * @return bool */ public static function updateTransverTag(MessageHandle $handle, $revision, $text, User $user) { if ($user->isAllowed('bot')) { return false; } $group = $handle->getGroup(); $title = $handle->getTitle(); $name = $handle->getKey() . '/' . $group->getSourceLanguage(); $definitionTitle = Title::makeTitleSafe($title->getNamespace(), $name); if (!$definitionTitle || !$definitionTitle->exists()) { return true; } $definitionRevision = $definitionTitle->getLatestRevID(); $dbw = wfGetDB(DB_MASTER); $conds = array('rt_page' => $title->getArticleID(), 'rt_type' => RevTag::getType('tp:transver'), 'rt_revision' => $revision, 'rt_value' => $definitionRevision); $index = array('rt_type', 'rt_page', 'rt_revision'); $dbw->replace('revtag', array($index), $conds, __METHOD__); return true; }
/** * Get a list of page ids where the latest revision is either tagged or marked */ public static function getTranslatablePages() { // Avoid replication lag issues $dbr = wfGetDB(DB_MASTER); $tables = array('revtag', 'page'); $fields = 'rt_page'; $conds = array('rt_page = page_id', 'rt_revision = page_latest', 'rt_type' => array(RevTag::getType('tp:mark'), RevTag::getType('tp:tag'))); $options = array('GROUP BY' => 'rt_page'); $res = $dbr->select($tables, $fields, $conds, __METHOD__, $options); $results = array(); foreach ($res as $row) { $results[] = $row->rt_page; } return $results; }
protected function buildPageArray($res) { $pages = array(); foreach ($res as $r) { // We have multiple rows for same page, because of different tags if (!isset($pages[$r->page_id])) { $pages[$r->page_id] = array(); $title = Title::newFromRow($r); $pages[$r->page_id]['title'] = $title; $pages[$r->page_id]['latest'] = intval($title->getLatestRevID()); } $tag = RevTag::typeToTag($r->rt_type); $pages[$r->page_id][$tag] = intval($r->rt_revision); } return $pages; }
public static function getTranslatablePages(array &$groups, array &$deps, array &$autoload) { global $wgEnablePageTranslation; $deps[] = new GlobalDependency('wgEnablePageTranslation'); if (!$wgEnablePageTranslation) { return; } $db = wfGetDB(DB_MASTER); $tables = array('page', 'revtag'); $vars = array('page_id', 'page_namespace', 'page_title'); $conds = array('page_id=rt_page', 'rt_type' => RevTag::getType('tp:mark')); $options = array('GROUP BY' => 'rt_page'); $res = $db->select($tables, $vars, $conds, __METHOD__, $options); foreach ($res as $r) { $title = Title::newFromRow($r); $id = TranslatablePage::getMessageGroupIdFromTitle($title); $groups[$id] = new WikiPageMessageGroup($id, $title); $groups[$id]->setLabel($title->getPrefixedText()); } }
protected function checkRevTagTable() { $fixes = array(); $dbr = wfGetDB( DB_SLAVE ); $tags = array( 'tp:mark', 'tp:tag', 'tp:transver', 'fuzzy' ); $pages = $dbr->select( 'revtag', 'rt_page', null, __METHOD__, array( 'GROUP BY' => 'rt_page' ) ); $this->output( "Checking that tags match a valid page...\n\n" ); foreach ( $pages as $row ) { $id = $row->rt_page; $title = Title::newFromID( $id ); $name = $title ? $title->getPrefixedText() : "#$id"; if ( !$title ) { $name = "#$id"; $deleted = $this->findDeletedPage( $id ); if ( $deleted === false ) { $this->output( "Page id $id does not correspond to any page\n" ); $fixes["$name <revtag>"] = array( 'delete tags', 'revtag', array( 'rt_page' => $id ) ); } else { $name .= "<$deleted>"; } } } $this->output( "Checked {$pages->numRows()} pages in the revtag table\n" ); $this->output( "\n\nValidating tags...\n" ); $result = $dbr->select( 'revtag', '*', null, __METHOD__ ); foreach ( $result as $_ ) { if ( !isset( $tags[$_->rt_type] ) ) { $name = $this->idToName( $_->rt_page ); $this->output( "Page $name has unknown tag {$_->rt_type}\n" ); $fixes["$name <revtag:unknown:{$_->rt_type}>"] = array( 'delete tag', 'revtag', array( 'rt_page' => $id, 'rt_type' => $_->rt_type ) ); continue; } elseif ( $_->rt_type === RevTag::getType( 'tp:transver' ) ) { $check = $this->checkTransrevRevision( $rev ); // FIXME: $rev is undefined if ( $check !== true ) { $name = $this->idToName( $_->rt_page ); $this->output( "Page $name has invalid tp:transver: $check\n" ); $fixes["$name <revtag:transver>"] = array( 'delete tag', 'revtag', array( 'rt_page' => $id, 'rt_type' => $_->rt_type ) ); } } } $this->output( "Checked {$result->numRows()} tags in the revtag table\n\n\n" ); return $fixes; }
$limit = max( 1000, intval( $count / 100 ) ); $offset = 0; while ( true ) { $inserts = array(); echo "$offset/$count\n"; $options = array( 'LIMIT' => $limit, 'OFFSET' => $offset ); $res = $db->select( $tables, $fields, $conds, __METHOD__, $options ); if ( !$res->numRows() ) { break; } foreach ( $res as $r ) { $text = Revision::getRevisionText( $r ); if ( strpos( $text, TRANSLATE_FUZZY ) !== false ) { $inserts[] = array( 'rt_page' => $r->page_id, 'rt_revision' => $r->rev_id, 'rt_type' => RevTag::getType( 'fuzzy' ), ); } } $offset += $limit; $db->replace( 'revtag', 'rt_type_page_revision', $inserts, __METHOD__ ); } /// @endcond
function tagFuzzy( $problematic ) { if ( !count( $problematic ) ) { return; } $db = wfGetDB( DB_MASTER ); foreach ( $problematic as $p ) { $title = Title::makeTitleSafe( $p[0], $p[1] ); $titleText = $title->getDBKey(); $res = $db->select( 'page', array( 'page_id', 'page_latest' ), array( 'page_namespace' => $p[0], 'page_title' => $titleText ), __METHOD__ ); $inserts = array(); foreach ( $res as $r ) { $inserts = array( 'rt_page' => $r->page_id, 'rt_revision' => $r->page_latest, 'rt_type' => RevTag::getType( 'fuzzy' ) ); } $db->replace( 'revtag', 'rt_type_page_revision', $inserts, __METHOD__ ); } }
protected static function addSectionTag(Title $title, $revision, $pageRevision) { if ($pageRevision === null) { throw new MWException('Page revision is null'); } $dbw = wfGetDB(DB_MASTER); $conds = array('rt_page' => $title->getArticleId(), 'rt_type' => RevTag::getType('tp:transver'), 'rt_revision' => $revision); $dbw->delete('revtag', $conds, __METHOD__); $conds['rt_value'] = $pageRevision; $dbw->insert('revtag', $conds, __METHOD__); }
protected function getPageDiff() { $this->mustBeKnownMessage(); $group = $this->group; $title = $this->handle->getTitle(); $key = $this->handle->getKey(); if ($group instanceof WikiPageMessageGroup || !$title->exists()) { return null; } $definitionTitle = Title::makeTitleSafe($title->getNamespace(), "{$key}/en"); if (!$definitionTitle || !$definitionTitle->exists()) { return null; } $db = wfGetDB(DB_MASTER); $conds = array('rt_page' => $title->getArticleId(), 'rt_type' => RevTag::getType('tp:transver')); $options = array('ORDER BY' => 'rt_revision DESC'); $latestRevision = $definitionTitle->getLatestRevID(); $translationRevision = $db->selectField('revtag', 'rt_value', $conds, __METHOD__, $options); if ($translationRevision === false) { return null; } // Using newFromId instead of newFromTitle, because the page might have been renamed $oldrev = Revision::newFromId($translationRevision); if (!$oldrev) { // And someone might still have deleted it return null; } $oldtext = $oldrev->getText(); $newtext = Revision::newFromTitle($definitionTitle, $latestRevision)->getText(); if ($oldtext === $newtext) { return null; } $diff = new DifferenceEngine(); if (method_exists('DifferenceEngine', 'setTextLanguage')) { $diff->setTextLanguage($this->group->getSourceLanguage()); } $diff->setText($oldtext, $newtext); $diff->setReducedLineNumbers(); $diff->showDiffStyle(); return $diff->getDiff(wfMsgHtml('tpt-diff-old'), wfMsgHtml('tpt-diff-new')); }
/** * @param $page Article * @param $changed */ public function addFuzzyTags( $page, $changed ) { if ( !count( $changed ) ) { self::superDebug( __METHOD__, 'nochanged', $page->getTitle() ); return; } $titles = array(); $prefix = $page->getTitle()->getPrefixedText(); $db = wfGetDB( DB_MASTER ); foreach ( $changed as $c ) { $title = Title::makeTitleSafe( NS_TRANSLATIONS, "$prefix/$c" ); if ( $title ) { $titles[] = 'page_title ' . $db->buildLike( $title->getDBkey() . '/', $db->anyString() ); } } $titleCond = $db->makeList( $titles, LIST_OR ); $fields = array( 'page_id', 'page_latest' ); $conds = array( 'page_namespace' => NS_TRANSLATIONS, $titleCond ); $res = $db->select( 'page', $fields, $conds, __METHOD__ ); $inserts = array(); // @todo Filter out qqq so it is not marked as fuzzy. foreach ( $res as $r ) { $inserts[] = array( 'rt_page' => $r->page_id, 'rt_type' => RevTag::getType( 'fuzzy' ), 'rt_revision' => $r->page_latest, ); } if ( count( $inserts ) ) { self::superDebug( __METHOD__, 'inserts', $inserts ); $db->replace( 'revtag', array( 'rt_type_page_revision' ), $inserts, __METHOD__ ); } }