public function testParsing()
{
$title = Title::newFromText('MediaWiki:Ugakey/nl');
$page = WikiPage::factory($title);
$content = ContentHandler::makeContent('$1 van $2', $title);
$status = $page->doEditContent($content, __METHOD__);
$value = $status->getValue();
/**
* @var Revision $rev
*/
$rev = $value['revision'];
$revision = $rev->getId();
$dbw = wfGetDB(DB_MASTER);
$conds = array('rt_page' => $title->getArticleID(), 'rt_type' => RevTag::getType('fuzzy'), 'rt_revision' => $revision);
$index = array_keys($conds);
$dbw->replace('revtag', array($index), $conds, __METHOD__);
$handle = new MessageHandle($title);
$this->assertTrue($handle->isValid(), 'Message is known');
$this->assertTrue($handle->isFuzzy(), 'Message is fuzzy after database fuzzying');
// Update the translation without the fuzzy string
$content = ContentHandler::makeContent('$1 van $2', $title);
$page->doEditContent($content, __METHOD__);
$this->assertFalse($handle->isFuzzy(), 'Message is unfuzzy after edit');
$content = ContentHandler::makeContent('!!FUZZY!!$1 van $2', $title);
$page->doEditContent($content, __METHOD__);
$this->assertTrue($handle->isFuzzy(), 'Message is fuzzy after manual fuzzying');
// Update the translation without the fuzzy string
$content = ContentHandler::makeContent('$1 van $2', $title);
$page->doEditContent($content, __METHOD__);
$this->assertFalse($handle->isFuzzy(), 'Message is unfuzzy after edit');
}
/**
* Determines the schema version.
*
* @return int
*/
public static function checkSchema() {
if ( self::$schema !== false ) {
return self::$schema;
} else {
$dbr = wfGetDB( DB_SLAVE );
if ( $dbr->tableExists( 'revtag_type' ) ) {
return self::$schema = 1;
} else {
return self::$schema = 2;
}
}
}
function run()
{
global $wgTranslateDocumentationLanguageCode;
$title = $this->title;
$params = $this->params;
$user = FuzzyBot::getUser();
$flags = EDIT_DEFER_UPDATES | EDIT_FORCE_BOT;
$wikiPage = WikiPage::factory($title);
$summary = wfMessage('translate-manage-import-summary')->inContentLanguage()->plain();
$content = ContentHandler::makeContent($params['content'], $title);
$wikiPage->doEditContent($content, $summary, $flags, false, $user);
// NOTE: message documentation is excluded from fuzzying!
if ($params['fuzzy']) {
$handle = new MessageHandle($title);
$key = $handle->getKey();
$languages = TranslateUtils::getLanguageNames('en');
unset($languages[$wgTranslateDocumentationLanguageCode]);
$languages = array_keys($languages);
$dbw = wfGetDB(DB_MASTER);
$fields = array('page_id', 'page_latest');
$conds = array('page_namespace' => $title->getNamespace());
$pages = array();
foreach ($languages as $code) {
$otherTitle = Title::makeTitleSafe($title->getNamespace(), "{$key}/{$code}");
$pages[$otherTitle->getDBKey()] = true;
}
unset($pages[$title->getDBKey()]);
if (count($pages) === 0) {
return true;
}
$conds['page_title'] = array_keys($pages);
$res = $dbw->select('page', $fields, $conds, __METHOD__);
$inserts = array();
foreach ($res as $row) {
$inserts[] = array('rt_type' => RevTag::getType('fuzzy'), 'rt_page' => $row->page_id, 'rt_revision' => $row->page_latest);
}
$dbw->replace('revtag', array(array('rt_type', 'rt_page', 'rt_revision')), $inserts, __METHOD__);
}
return true;
}
public function getData()
{
$db = wfGetDB(DB_MASTER);
$conds = array('rt_page' => $this->handle->getTitle()->getArticleID(), 'rt_type' => RevTag::getType('tp:transver'));
$options = array('ORDER BY' => 'rt_revision DESC');
$translationRevision = $db->selectField('revtag', 'rt_value', $conds, __METHOD__, $options);
if ($translationRevision === false) {
throw new TranslationHelperException("No definition revision recorded");
}
$definitionTitle = Title::makeTitleSafe($this->handle->getTitle()->getNamespace(), $this->handle->getKey() . '/' . $this->group->getSourceLanguage());
if (!$definitionTitle || !$definitionTitle->exists()) {
throw new TranslationHelperException("Definition page doesn't exist");
}
// Using newFromId instead of newFromTitle, because the page might have been renamed
$oldrev = Revision::newFromId($translationRevision);
if (!$oldrev) {
throw new TranslationHelperException("Old definition version doesn't exist anymore");
}
$oldContent = $oldrev->getContent();
$newContent = $this->getDefinitionContent();
if (!$oldContent) {
throw new TranslationHelperException("Old definition version doesn't exist anymore");
}
if (!$oldContent instanceof WikitextContent || !$newContent instanceof WikitextContent) {
throw new TranslationHelperException('Can only work on Wikitext content');
}
if ($oldContent->equals($newContent)) {
throw new TranslationHelperException('No changes');
}
$diff = new DifferenceEngine($this->context);
if (method_exists('DifferenceEngine', 'setTextLanguage')) {
$diff->setTextLanguage($this->group->getSourceLanguage());
}
$diff->setContent($oldContent, $newContent);
$diff->setReducedLineNumbers();
$diff->showDiffStyle();
$html = $diff->getDiff($this->context->msg('tpt-diff-old')->escaped(), $this->context->msg('tpt-diff-new')->escaped());
return array('value_old' => $oldContent->getNativeData(), 'value_new' => $newContent->getNativeData(), 'revisionid_old' => $oldrev->getId(), 'revisionid_new' => $definitionTitle->getLatestRevId(), 'language' => $this->group->getSourceLanguage(), 'html' => $html);
}
public function execute()
{
global $wgTranslateMessageNamespaces;
$namespace = $this->getOption('namespace', $wgTranslateMessageNamespaces);
if (is_string($namespace)) {
if (!MWNamespace::exists($namespace)) {
$namespace = MWNamespace::getCanonicalIndex($namespace);
if ($namespace === null) {
$this->error('Bad namespace', true);
}
}
}
$db = wfGetDB(DB_MASTER);
$tables = array('page', 'text', 'revision');
$fields = array('page_id', 'page_title', 'page_namespace', 'rev_id', 'old_text', 'old_flags');
$conds = array('page_latest = rev_id', 'old_id = rev_text_id', 'page_namespace' => $namespace);
$limit = 100;
$offset = 0;
while (true) {
$inserts = array();
$this->output('.', 0);
$options = array('LIMIT' => $limit, 'OFFSET' => $offset);
$res = $db->select($tables, $fields, $conds, __METHOD__, $options);
if (!$res->numRows()) {
break;
}
foreach ($res as $r) {
$text = Revision::getRevisionText($r);
if (strpos($text, TRANSLATE_FUZZY) !== false) {
$inserts[] = array('rt_page' => $r->page_id, 'rt_revision' => $r->rev_id, 'rt_type' => RevTag::getType('fuzzy'));
}
}
$offset += $limit;
$db->replace('revtag', 'rt_type_page_revision', $inserts, __METHOD__);
}
}
/**
* Loads existence and fuzzy state for given list of keys.
* @param string[] $keys List of keys in database format.
* @param int $dbtype One of DB_* constants.
*/
protected function loadInfo(array $keys, $dbtype = DB_SLAVE)
{
if ($this->dbInfo !== null) {
return;
}
$this->dbInfo = array();
if (!count($keys)) {
return;
}
$dbr = wfGetDB($dbtype);
$tables = array('page', 'revtag');
$fields = array('page_namespace', 'page_title', 'rt_type');
$conds = $this->getTitleConds($dbr);
$joins = array('revtag' => array('LEFT JOIN', array('page_id=rt_page', 'page_latest=rt_revision', 'rt_type' => RevTag::getType('fuzzy'))));
$this->dbInfo = $dbr->select($tables, $fields, $conds, __METHOD__, array(), $joins);
}
/**
* This constructs the list of all groups from multiple different
* sources. When possible, a cache dependency is created to automatically
* recreate the cache when configuration changes.
* @todo Reduce the ways of which messages can be added. Target is just
* to have three ways: Yaml files, translatable pages and with the hook.
* @todo In conjuction with the above, reduce the number of global
* variables like wgTranslate#C and have the message groups specify
* their own cache dependencies.
*/
protected static function loadGroupDefinitions()
{
global $wgTranslateAddMWExtensionGroups;
global $wgEnablePageTranslation, $wgTranslateGroupFiles;
global $wgTranslateAC, $wgTranslateEC, $wgTranslateCC;
global $wgAutoloadClasses;
global $wgTranslateWorkflowStates;
$deps = array();
$deps[] = new GlobalDependency('wgTranslateAddMWExtensionGroups');
$deps[] = new GlobalDependency('wgEnablePageTranslation');
$deps[] = new GlobalDependency('wgTranslateGroupFiles');
$deps[] = new GlobalDependency('wgTranslateAC');
$deps[] = new GlobalDependency('wgTranslateEC');
$deps[] = new GlobalDependency('wgTranslateCC');
$deps[] = new GlobalDependency('wgTranslateExtensionDirectory');
$deps[] = new GlobalDependency('wgTranslateWorkflowStates');
$deps[] = new FileDependency(dirname(__FILE__) . '/groups/mediawiki-defines.txt');
$deps[] = new FileDependency(dirname(__FILE__) . '/groups/Wikia/extensions.txt');
$deps[] = new FileDependency(dirname(__FILE__) . '/groups/Toolserver/toolserver-textdomains.txt');
if ($wgTranslateAddMWExtensionGroups) {
$a = new PremadeMediawikiExtensionGroups();
$a->addAll();
}
if ($wgEnablePageTranslation) {
$dbr = wfGetDB(DB_MASTER);
$tables = array('page', 'revtag');
$vars = array('page_id', 'page_namespace', 'page_title');
$conds = array('page_id=rt_page', 'rt_type' => RevTag::getType('tp:mark'));
$options = array('GROUP BY' => 'rt_page');
$res = $dbr->select($tables, $vars, $conds, __METHOD__, $options);
foreach ($res as $r) {
$title = Title::makeTitle($r->page_namespace, $r->page_title);
$id = TranslatablePage::getMessageGroupIdFromTitle($title);
$wgTranslateCC[$id] = new WikiPageMessageGroup($id, $title);
$wgTranslateCC[$id]->setLabel($title->getPrefixedText());
}
}
if ($wgTranslateWorkflowStates) {
$wgTranslateCC['translate-workflow-states'] = new WorkflowStatesMessageGroup();
}
$autoload = array();
wfRunHooks('TranslatePostInitGroups', array(&$wgTranslateCC, &$deps, &$autoload));
foreach ($wgTranslateGroupFiles as $configFile) {
wfDebug($configFile . "\n");
$deps[] = new FileDependency(realpath($configFile));
$fgroups = TranslateYaml::parseGroupFile($configFile);
foreach ($fgroups as $id => $conf) {
if (!empty($conf['AUTOLOAD']) && is_array($conf['AUTOLOAD'])) {
$dir = dirname($configFile);
foreach ($conf['AUTOLOAD'] as $class => $file) {
// For this request and for caching.
$wgAutoloadClasses[$class] = "{$dir}/{$file}";
$autoload[$class] = "{$dir}/{$file}";
}
}
$group = MessageGroupBase::factory($conf);
$wgTranslateCC[$id] = $group;
}
}
$key = wfMemckey('translate-groups');
$value = array('ac' => $wgTranslateAC, 'ec' => $wgTranslateEC, 'cc' => $wgTranslateCC, 'autoload' => $autoload);
$wrapper = new DependencyWrapper($value, $deps);
$wrapper->storeToCache(self::getCache(), $key, 60 * 60 * 2);
wfDebug(__METHOD__ . "-end\n");
}
/**
* Check if a title is marked as fuzzy.
* @return bool If title is marked fuzzy.
*/
public function isFuzzy() {
$dbr = wfGetDB( DB_SLAVE );
$tables = array( 'page', 'revtag' );
$field = 'rt_type';
$conds = array(
'page_namespace' => $this->title->getNamespace(),
'page_title' => $this->title->getDBkey(),
'rt_type' => RevTag::getType( 'fuzzy' ),
'page_id=rt_page',
'page_latest=rt_revision'
);
$res = $dbr->selectField( $tables, $field, $conds, __METHOD__ );
return $res !== false;
}
/**
* Adds tag which identifies the revision of source message at that time.
* This is used to show diff against current version of source message
* when updating a translation.
* Hook: Translate:newTranslation
* @param MessageHandle $handle
* @param int $revision
* @param string $text
* @param User $user
* @return bool
*/
public static function updateTransverTag(MessageHandle $handle, $revision, $text, User $user)
{
if ($user->isAllowed('bot')) {
return false;
}
$group = $handle->getGroup();
$title = $handle->getTitle();
$name = $handle->getKey() . '/' . $group->getSourceLanguage();
$definitionTitle = Title::makeTitleSafe($title->getNamespace(), $name);
if (!$definitionTitle || !$definitionTitle->exists()) {
return true;
}
$definitionRevision = $definitionTitle->getLatestRevID();
$dbw = wfGetDB(DB_MASTER);
$conds = array('rt_page' => $title->getArticleID(), 'rt_type' => RevTag::getType('tp:transver'), 'rt_revision' => $revision, 'rt_value' => $definitionRevision);
$index = array('rt_type', 'rt_page', 'rt_revision');
$dbw->replace('revtag', array($index), $conds, __METHOD__);
return true;
}
/**
* Get a list of page ids where the latest revision is either tagged or marked
*/
public static function getTranslatablePages()
{
// Avoid replication lag issues
$dbr = wfGetDB(DB_MASTER);
$tables = array('revtag', 'page');
$fields = 'rt_page';
$conds = array('rt_page = page_id', 'rt_revision = page_latest', 'rt_type' => array(RevTag::getType('tp:mark'), RevTag::getType('tp:tag')));
$options = array('GROUP BY' => 'rt_page');
$res = $dbr->select($tables, $fields, $conds, __METHOD__, $options);
$results = array();
foreach ($res as $row) {
$results[] = $row->rt_page;
}
return $results;
}
protected function buildPageArray($res)
{
$pages = array();
foreach ($res as $r) {
// We have multiple rows for same page, because of different tags
if (!isset($pages[$r->page_id])) {
$pages[$r->page_id] = array();
$title = Title::newFromRow($r);
$pages[$r->page_id]['title'] = $title;
$pages[$r->page_id]['latest'] = intval($title->getLatestRevID());
}
$tag = RevTag::typeToTag($r->rt_type);
$pages[$r->page_id][$tag] = intval($r->rt_revision);
}
return $pages;
}
public static function getTranslatablePages(array &$groups, array &$deps, array &$autoload)
{
global $wgEnablePageTranslation;
$deps[] = new GlobalDependency('wgEnablePageTranslation');
if (!$wgEnablePageTranslation) {
return;
}
$db = wfGetDB(DB_MASTER);
$tables = array('page', 'revtag');
$vars = array('page_id', 'page_namespace', 'page_title');
$conds = array('page_id=rt_page', 'rt_type' => RevTag::getType('tp:mark'));
$options = array('GROUP BY' => 'rt_page');
$res = $db->select($tables, $vars, $conds, __METHOD__, $options);
foreach ($res as $r) {
$title = Title::newFromRow($r);
$id = TranslatablePage::getMessageGroupIdFromTitle($title);
$groups[$id] = new WikiPageMessageGroup($id, $title);
$groups[$id]->setLabel($title->getPrefixedText());
}
}
protected function checkRevTagTable() {
$fixes = array();
$dbr = wfGetDB( DB_SLAVE );
$tags = array( 'tp:mark', 'tp:tag', 'tp:transver', 'fuzzy' );
$pages = $dbr->select( 'revtag', 'rt_page', null, __METHOD__, array( 'GROUP BY' => 'rt_page' ) );
$this->output( "Checking that tags match a valid page...\n\n" );
foreach ( $pages as $row ) {
$id = $row->rt_page;
$title = Title::newFromID( $id );
$name = $title ? $title->getPrefixedText() : "#$id";
if ( !$title ) {
$name = "#$id";
$deleted = $this->findDeletedPage( $id );
if ( $deleted === false ) {
$this->output( "Page id $id does not correspond to any page\n" );
$fixes["$name <revtag>"] = array( 'delete tags', 'revtag', array( 'rt_page' => $id ) );
} else {
$name .= "<$deleted>";
}
}
}
$this->output( "Checked {$pages->numRows()} pages in the revtag table\n" );
$this->output( "\n\nValidating tags...\n" );
$result = $dbr->select( 'revtag', '*', null, __METHOD__ );
foreach ( $result as $_ ) {
if ( !isset( $tags[$_->rt_type] ) ) {
$name = $this->idToName( $_->rt_page );
$this->output( "Page $name has unknown tag {$_->rt_type}\n" );
$fixes["$name <revtag:unknown:{$_->rt_type}>"] =
array( 'delete tag', 'revtag', array( 'rt_page' => $id, 'rt_type' => $_->rt_type ) );
continue;
} elseif ( $_->rt_type === RevTag::getType( 'tp:transver' ) ) {
$check = $this->checkTransrevRevision( $rev ); // FIXME: $rev is undefined
if ( $check !== true ) {
$name = $this->idToName( $_->rt_page );
$this->output( "Page $name has invalid tp:transver: $check\n" );
$fixes["$name <revtag:transver>"] =
array( 'delete tag', 'revtag', array( 'rt_page' => $id, 'rt_type' => $_->rt_type ) );
}
}
}
$this->output( "Checked {$result->numRows()} tags in the revtag table\n\n\n" );
return $fixes;
}
$limit = max( 1000, intval( $count / 100 ) );
$offset = 0;
while ( true ) {
$inserts = array();
echo "$offset/$count\n";
$options = array( 'LIMIT' => $limit, 'OFFSET' => $offset );
$res = $db->select( $tables, $fields, $conds, __METHOD__, $options );
if ( !$res->numRows() ) {
break;
}
foreach ( $res as $r ) {
$text = Revision::getRevisionText( $r );
if ( strpos( $text, TRANSLATE_FUZZY ) !== false ) {
$inserts[] = array(
'rt_page' => $r->page_id,
'rt_revision' => $r->rev_id,
'rt_type' => RevTag::getType( 'fuzzy' ),
);
}
}
$offset += $limit;
$db->replace( 'revtag', 'rt_type_page_revision', $inserts, __METHOD__ );
}
/// @endcond
function tagFuzzy( $problematic ) {
if ( !count( $problematic ) ) {
return;
}
$db = wfGetDB( DB_MASTER );
foreach ( $problematic as $p ) {
$title = Title::makeTitleSafe( $p[0], $p[1] );
$titleText = $title->getDBKey();
$res = $db->select( 'page', array( 'page_id', 'page_latest' ),
array( 'page_namespace' => $p[0], 'page_title' => $titleText ), __METHOD__ );
$inserts = array();
foreach ( $res as $r ) {
$inserts = array(
'rt_page' => $r->page_id,
'rt_revision' => $r->page_latest,
'rt_type' => RevTag::getType( 'fuzzy' )
);
}
$db->replace( 'revtag', 'rt_type_page_revision', $inserts, __METHOD__ );
}
}
protected static function addSectionTag(Title $title, $revision, $pageRevision)
{
if ($pageRevision === null) {
throw new MWException('Page revision is null');
}
$dbw = wfGetDB(DB_MASTER);
$conds = array('rt_page' => $title->getArticleId(), 'rt_type' => RevTag::getType('tp:transver'), 'rt_revision' => $revision);
$dbw->delete('revtag', $conds, __METHOD__);
$conds['rt_value'] = $pageRevision;
$dbw->insert('revtag', $conds, __METHOD__);
}
protected function getPageDiff()
{
$this->mustBeKnownMessage();
$group = $this->group;
$title = $this->handle->getTitle();
$key = $this->handle->getKey();
if ($group instanceof WikiPageMessageGroup || !$title->exists()) {
return null;
}
$definitionTitle = Title::makeTitleSafe($title->getNamespace(), "{$key}/en");
if (!$definitionTitle || !$definitionTitle->exists()) {
return null;
}
$db = wfGetDB(DB_MASTER);
$conds = array('rt_page' => $title->getArticleId(), 'rt_type' => RevTag::getType('tp:transver'));
$options = array('ORDER BY' => 'rt_revision DESC');
$latestRevision = $definitionTitle->getLatestRevID();
$translationRevision = $db->selectField('revtag', 'rt_value', $conds, __METHOD__, $options);
if ($translationRevision === false) {
return null;
}
// Using newFromId instead of newFromTitle, because the page might have been renamed
$oldrev = Revision::newFromId($translationRevision);
if (!$oldrev) {
// And someone might still have deleted it
return null;
}
$oldtext = $oldrev->getText();
$newtext = Revision::newFromTitle($definitionTitle, $latestRevision)->getText();
if ($oldtext === $newtext) {
return null;
}
$diff = new DifferenceEngine();
if (method_exists('DifferenceEngine', 'setTextLanguage')) {
$diff->setTextLanguage($this->group->getSourceLanguage());
}
$diff->setText($oldtext, $newtext);
$diff->setReducedLineNumbers();
$diff->showDiffStyle();
return $diff->getDiff(wfMsgHtml('tpt-diff-old'), wfMsgHtml('tpt-diff-new'));
}
/**
* @param $page Article
* @param $changed
*/
public function addFuzzyTags( $page, $changed ) {
if ( !count( $changed ) ) {
self::superDebug( __METHOD__, 'nochanged', $page->getTitle() );
return;
}
$titles = array();
$prefix = $page->getTitle()->getPrefixedText();
$db = wfGetDB( DB_MASTER );
foreach ( $changed as $c ) {
$title = Title::makeTitleSafe( NS_TRANSLATIONS, "$prefix/$c" );
if ( $title ) {
$titles[] = 'page_title ' . $db->buildLike( $title->getDBkey() . '/', $db->anyString() );
}
}
$titleCond = $db->makeList( $titles, LIST_OR );
$fields = array( 'page_id', 'page_latest' );
$conds = array( 'page_namespace' => NS_TRANSLATIONS, $titleCond );
$res = $db->select( 'page', $fields, $conds, __METHOD__ );
$inserts = array();
// @todo Filter out qqq so it is not marked as fuzzy.
foreach ( $res as $r ) {
$inserts[] = array(
'rt_page' => $r->page_id,
'rt_type' => RevTag::getType( 'fuzzy' ),
'rt_revision' => $r->page_latest,
);
}
if ( count( $inserts ) ) {
self::superDebug( __METHOD__, 'inserts', $inserts );
$db->replace( 'revtag', array( 'rt_type_page_revision' ), $inserts, __METHOD__ );
}
}
