function moveToExternal($cluster, $maxID)
{
$fname = 'moveToExternal';
$dbw =& wfGetDB(DB_MASTER);
print "Moving {$maxID} text rows to external storage\n";
$ext = new ExternalStoreDB();
for ($id = 1; $id <= $maxID; $id++) {
if (!($id % REPORTING_INTERVAL)) {
print "{$id}\n";
wfWaitForSlaves(5);
}
$row = $dbw->selectRow('text', array('old_flags', 'old_text'), array('old_id' => $id, "old_flags NOT LIKE '%external%'"), $fname);
if (!$row) {
# Non-existent or already done
continue;
}
# Resolve stubs
$flags = explode(',', $row->old_flags);
if (in_array('object', $flags) && substr($row->old_text, 0, strlen(STUB_HEADER)) === STUB_HEADER) {
resolveStub($id, $row->old_text, $row->old_flags);
continue;
}
$url = $ext->store($cluster, $row->old_text);
if (!$url) {
print "Error writing to external storage\n";
exit;
}
if ($row->old_flags === '') {
$flags = 'external';
} else {
$flags = "{$row->old_flags},external";
}
$dbw->update('text', array('old_flags' => $flags, 'old_text' => $url), array('old_id' => $id), $fname);
}
}
function addWiki($lang, $site, $dbName)
{
global $IP, $wgLanguageNames, $wgDefaultExternalStore;
$name = $wgLanguageNames[$lang];
$dbw =& wfGetDB(DB_WRITE);
$common = "/home/wikipedia/common";
$maintenance = "{$IP}/maintenance";
print "Creating database {$dbName} for {$lang}.{$site}\n";
# Set up the database
$dbw->query("SET table_type=Innodb");
$dbw->query("CREATE DATABASE {$dbName}");
$dbw->selectDB($dbName);
print "Initialising tables\n";
dbsource("{$maintenance}/tables.sql", $dbw);
dbsource("{$IP}/extensions/OAI/update_table.sql", $dbw);
$dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)");
# Initialise external storage
if ($wgDefaultExternalStore && preg_match('!^DB://(.*)$!', $wgDefaultExternalStore, $m)) {
print "Initialising external storage...\n";
require_once 'ExternalStoreDB.php';
global $wgDBuser, $wgDBpassword, $wgExternalServers;
$cluster = $m[1];
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb =& $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$dbName}");
$extdb->selectDB($dbName);
dbsource("{$maintenance}/storage/blobs.sql", $extdb);
$extdb->immediateCommit();
}
$wgTitle = Title::newMainPage();
$wgArticle = new Article($wgTitle);
$ucsite = ucfirst($site);
$wgArticle->insertNewArticle("\n==This subdomain is reserved for the creation of a {$ucsite} in '''[[:en:{$name}|{$name}]]''' language==\n\nIf you can write in this language and want to collaborate in the creation of this encyclopedia then '''you''' can make it.\n\nGo ahead. Translate this page and start working on your encyclopedia.\n\nFor help, see '''[[m:Help:How to start a new Wikipedia|how to start a new Wikipedia]]'''.\n\n==Sister projects==\n[http://meta.wikipedia.org Meta-Wikipedia] | [http://www.wiktionary.org Wikitonary] | [http://www.wikibooks.org Wikibooks] | [http://www.wikinews.org Wikinews] | [http://www.wikiquote.org Wikiquote] | [http://www.wikisource.org Wikisource]\n\nSee the [http://www.wikipedia.org Wikipedia portal] for other language Wikipedias.\n\n[[aa:]]\n[[af:]]\n[[als:]]\n[[ar:]]\n[[de:]]\n[[en:]]\n[[as:]]\n[[ast:]]\n[[ay:]]\n[[az:]]\n[[be:]]\n[[bg:]]\n[[bn:]]\n[[bo:]]\n[[bs:]]\n[[cs:]]\n[[co:]]\n[[cs:]]\n[[cy:]]\n[[da:]]\n[[el:]]\n[[eo:]]\n[[es:]]\n[[et:]]\n[[eu:]]\n[[fa:]]\n[[fi:]]\n[[fr:]]\n[[fy:]]\n[[ga:]]\n[[gl:]]\n[[gn:]]\n[[gu:]]\n[[he:]]\n[[hi:]]\n[[hr:]]\n[[hy:]]\n[[ia:]]\n[[id:]]\n[[is:]]\n[[it:]]\n[[ja:]]\n[[ka:]]\n[[kk:]]\n[[km:]]\n[[kn:]]\n[[ko:]]\n[[ks:]]\n[[ku:]]\n[[ky:]]\n[[la:]]\n[[ln:]]\n[[lo:]]\n[[lt:]]\n[[lv:]]\n[[hu:]]\n[[mi:]]\n[[mk:]]\n[[ml:]]\n[[mn:]]\n[[mr:]]\n[[ms:]]\n[[mt:]]\n[[my:]]\n[[na:]]\n[[nah:]]\n[[nds:]]\n[[ne:]]\n[[nl:]]\n[[no:]]\n[[oc:]]\n[[om:]]\n[[pa:]]\n[[pl:]]\n[[ps:]]\n[[pt:]]\n[[qu:]]\n[[ro:]]\n[[ru:]]\n[[sa:]]\n[[si:]]\n[[sk:]]\n[[sl:]]\n[[sq:]]\n[[sr:]]\n[[sv:]]\n[[sw:]]\n[[ta:]]\n[[te:]]\n[[tg:]]\n[[th:]]\n[[tk:]]\n[[tl:]]\n[[tr:]]\n[[tt:]]\n[[ug:]]\n[[uk:]]\n[[ur:]]\n[[uz:]]\n[[vi:]]\n[[vo:]]\n[[xh:]]\n[[yo:]]\n[[za:]]\n[[zh:]]\n[[zu:]]\n", '', false, false);
print "Adding to dblists\n";
# Add to dblist
$file = fopen("{$common}/all.dblist", "a");
fwrite($file, "{$dbName}\n");
fclose($file);
# Update the sublists
system("cd {$common} && ./refresh-dblist");
print "Constructing interwiki SQL\n";
# Rebuild interwiki tables
$sql = getRebuildInterwikiSQL();
$tempname = tempnam('/tmp', 'addwiki');
$file = fopen($tempname, 'w');
if (!$file) {
wfDie("Error, unable to open temporary file {$tempname}\n");
}
fwrite($file, $sql);
fclose($file);
print "Sourcing interwiki SQL\n";
dbsource($tempname, $dbw);
unlink($tempname);
print "Script ended. You now want to run sync-common-all to publish *dblist files (check them for duplicates first)\n";
}
function moveToExternal($cluster, $maxID)
{
$fname = 'moveToExternal';
$dbw =& wfGetDB(DB_MASTER);
print "Moving {$maxID} text rows to external storage\n";
$ext = new ExternalStoreDB();
for ($id = 1; $id <= $maxID; $id++) {
if (!($id % REPORTING_INTERVAL)) {
print "{$id}\n";
wfWaitForSlaves(5);
}
$row = $dbw->selectRow('text', array('old_flags', 'old_text'), array('old_id' => $id, "old_flags NOT LIKE '%external%'"), $fname);
if (!$row) {
# Non-existent or already done
continue;
}
# Resolve stubs
$text = $row->old_text;
if ($row->old_flags === '') {
$flags = 'external';
} else {
$flags = "{$row->old_flags},external";
}
if (strpos($flags, 'object') !== false) {
$obj = unserialize($text);
$className = strtolower(get_class($obj));
if ($className == 'historyblobstub') {
resolveStub($id, $row->old_text, $row->old_flags);
continue;
} elseif ($className == 'historyblobcurstub') {
$text = gzdeflate($obj->getText());
$flags = 'utf-8,gzip,external';
} elseif ($className == 'concatenatedgziphistoryblob') {
// Do nothing
} else {
print "Warning: unrecognised object class \"{$className}\"\n";
continue;
}
}
if (strlen($text) < 100) {
// Don't move tiny revisions
continue;
}
#print "Storing " . strlen( $text ) . " bytes to $url\n";
$url = $ext->store($cluster, $text);
if (!$url) {
print "Error writing to external storage\n";
exit;
}
$dbw->update('text', array('old_flags' => $flags, 'old_text' => $url), array('old_id' => $id), $fname);
}
}
public function execute() {
global $wgDefaultExternalStore;
# Setup
$from = $this->getArg( 0 );
$to = $this->getArg( 1 );
$this->output( "Renaming blob tables in ES from $from to $to...\n" );
$this->output( "Sleeping 5 seconds...\n" );
sleep( 5 );
# Initialise external storage
if ( is_array( $wgDefaultExternalStore ) ) {
$stores = $wgDefaultExternalStore;
} elseif ( $wgDefaultExternalStore ) {
$stores = array( $wgDefaultExternalStore );
} else {
$stores = array();
}
if ( count( $stores ) ) {
$this->output( "Initialising external storage...\n" );
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ( $stores as $storeURL ) {
$m = array();
if ( !preg_match( '!^DB://(.*)$!', $storeURL, $m ) ) {
continue;
}
$cluster = $m[1];
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB;
$extdb =& $store->getMaster( $cluster );
$extdb->query( "SET table_type=InnoDB" );
$extdb->query( "CREATE DATABASE {$to}" );
$extdb->query( "ALTER TABLE {$from}.blobs RENAME TO {$to}.blobs" );
$extdb->selectDB( $from );
$extdb->sourceFile( $this->getDir() . '/storage/blobs.sql' );
$extdb->commit();
}
}
$this->output( "done.\n" );
}
/**
* hook for ExternalStoreDB::FetchBlob
*
* @param string $cluster storage identifier
* @param integer $id blob identifier
* @param integer $itemID item identifier when revision text is merged & archived
* @param string $ret returned blob text
*/
function ExternalStoreDBFetchBlobHook($cluster, $id, $itemID, &$ret)
{
global $wgTheSchwartzSecretToken;
wfProfileIn(__METHOD__);
// there's already blob text
if ($ret !== false) {
wfProfileOut(__METHOD__);
return true;
}
// wikia doesn't use $itemID
$url = sprintf("%s?action=fetchblob&store=%s&id=%d&token=%s&format=json", F::app()->wg->FetchBlobApiURL, $cluster, $id, $wgTheSchwartzSecretToken);
$response = json_decode(Http::get($url, "default", array('noProxy' => true)));
if (isset($response->fetchblob)) {
$blob = isset($response->fetchblob->blob) ? $response->fetchblob->blob : false;
$hash = isset($response->fetchblob->hash) ? $response->fetchblob->hash : null;
if ($blob) {
// pack to binary
$blob = pack("H*", $blob);
$hash = md5($blob);
// check md5 sum for binary
if ($hash == $response->fetchblob->hash) {
wfDebug(__METHOD__ . ": md5 sum match\n");
$ret = $blob;
// now store blob in local database but only when it's Poznan's devbox
$isPoznanDevbox = F::app()->wg->DevelEnvironment === true && F::app()->wg->WikiaDatacenter == "poz";
if ($isPoznanDevbox) {
wfDebug(__METHOD__ . ": this is poznań devbox\n");
$store = new ExternalStoreDB();
$dbw = $store->getMaster($cluster);
if ($dbw) {
wfDebug(__METHOD__ . ": storing blob {$id} on local storage {$cluster}\n");
$dbw->insert($store->getTable($dbw), array("blob_id" => $id, "blob_text" => $ret), __METHOD__, array("IGNORE"));
$dbw->commit();
}
}
} else {
wfDebug(__METHOD__ . ": md5 sum not match, {$hash} != {$response->fetchblob}->hash\n");
}
}
} else {
wfDebug(__METHOD__ . ": malformed response from API call\n");
}
wfProfileOut(__METHOD__);
return true;
}
/**
* Compress the text in chunks after concatenating the revisions.
*
* @param int $startId
* @param int $maxChunkSize
* @param string $beginDate
* @param string $endDate
* @param string $extdb
* @param bool|int $maxPageId
* @return bool
*/
private function compressWithConcat($startId, $maxChunkSize, $beginDate, $endDate, $extdb = "", $maxPageId = false)
{
$loadStyle = self::LS_CHUNKED;
$dbr = wfGetDB(DB_SLAVE);
$dbw = wfGetDB(DB_MASTER);
# Set up external storage
if ($extdb != '') {
$storeObj = new ExternalStoreDB();
}
# Get all articles by page_id
if (!$maxPageId) {
$maxPageId = $dbr->selectField('page', 'max(page_id)', '', __METHOD__);
}
$this->output("Starting from {$startId} of {$maxPageId}\n");
$pageConds = array();
/*
if ( $exclude_ns0 ) {
print "Excluding main namespace\n";
$pageConds[] = 'page_namespace<>0';
}
if ( $queryExtra ) {
$pageConds[] = $queryExtra;
}
*/
# For each article, get a list of revisions which fit the criteria
# No recompression, use a condition on old_flags
# Don't compress object type entities, because that might produce data loss when
# overwriting bulk storage concat rows. Don't compress external references, because
# the script doesn't yet delete rows from external storage.
$conds = array('old_flags NOT ' . $dbr->buildLike($dbr->anyString(), 'object', $dbr->anyString()) . ' AND old_flags NOT ' . $dbr->buildLike($dbr->anyString(), 'external', $dbr->anyString()));
if ($beginDate) {
if (!preg_match('/^\\d{14}$/', $beginDate)) {
$this->error("Invalid begin date \"{$beginDate}\"\n");
return false;
}
$conds[] = "rev_timestamp>'" . $beginDate . "'";
}
if ($endDate) {
if (!preg_match('/^\\d{14}$/', $endDate)) {
$this->error("Invalid end date \"{$endDate}\"\n");
return false;
}
$conds[] = "rev_timestamp<'" . $endDate . "'";
}
if ($loadStyle == self::LS_CHUNKED) {
$tables = array('revision', 'text');
$fields = array('rev_id', 'rev_text_id', 'old_flags', 'old_text');
$conds[] = 'rev_text_id=old_id';
$revLoadOptions = 'FOR UPDATE';
} else {
$tables = array('revision');
$fields = array('rev_id', 'rev_text_id');
$revLoadOptions = array();
}
# Don't work with current revisions
# Don't lock the page table for update either -- TS 2006-04-04
#$tables[] = 'page';
#$conds[] = 'page_id=rev_page AND rev_id != page_latest';
for ($pageId = $startId; $pageId <= $maxPageId; $pageId++) {
wfWaitForSlaves();
# Wake up
$dbr->ping();
# Get the page row
$pageRes = $dbr->select('page', array('page_id', 'page_namespace', 'page_title', 'page_latest'), $pageConds + array('page_id' => $pageId), __METHOD__);
if ($pageRes->numRows() == 0) {
continue;
}
$pageRow = $dbr->fetchObject($pageRes);
# Display progress
$titleObj = Title::makeTitle($pageRow->page_namespace, $pageRow->page_title);
$this->output("{$pageId}\t" . $titleObj->getPrefixedDBkey() . " ");
# Load revisions
$revRes = $dbw->select($tables, $fields, array_merge(array('rev_page' => $pageRow->page_id, 'rev_id < ' . $pageRow->page_latest), $conds), __METHOD__, $revLoadOptions);
$revs = array();
foreach ($revRes as $revRow) {
$revs[] = $revRow;
}
if (count($revs) < 2) {
# No revisions matching, no further processing
$this->output("\n");
continue;
}
# For each chunk
$i = 0;
while ($i < count($revs)) {
if ($i < count($revs) - $maxChunkSize) {
$thisChunkSize = $maxChunkSize;
} else {
$thisChunkSize = count($revs) - $i;
}
$chunk = new ConcatenatedGzipHistoryBlob();
$stubs = array();
$dbw->begin(__METHOD__);
$usedChunk = false;
$primaryOldid = $revs[$i]->rev_text_id;
// @codingStandardsIgnoreStart Ignore avoid function calls in a FOR loop test part warning
# Get the text of each revision and add it to the object
for ($j = 0; $j < $thisChunkSize && $chunk->isHappy(); $j++) {
// @codingStandardsIgnoreEnd
$oldid = $revs[$i + $j]->rev_text_id;
# Get text
if ($loadStyle == self::LS_INDIVIDUAL) {
$textRow = $dbw->selectRow('text', array('old_flags', 'old_text'), array('old_id' => $oldid), __METHOD__, 'FOR UPDATE');
$text = Revision::getRevisionText($textRow);
} else {
$text = Revision::getRevisionText($revs[$i + $j]);
}
if ($text === false) {
$this->error("\nError, unable to get text in old_id {$oldid}");
#$dbw->delete( 'old', array( 'old_id' => $oldid ) );
}
if ($extdb == "" && $j == 0) {
$chunk->setText($text);
$this->output('.');
} else {
# Don't make a stub if it's going to be longer than the article
# Stubs are typically about 100 bytes
if (strlen($text) < 120) {
$stub = false;
$this->output('x');
} else {
$stub = new HistoryBlobStub($chunk->addItem($text));
$stub->setLocation($primaryOldid);
$stub->setReferrer($oldid);
$this->output('.');
$usedChunk = true;
}
$stubs[$j] = $stub;
}
}
$thisChunkSize = $j;
# If we couldn't actually use any stubs because the pages were too small, do nothing
if ($usedChunk) {
if ($extdb != "") {
# Move blob objects to External Storage
$stored = $storeObj->store($extdb, serialize($chunk));
if ($stored === false) {
$this->error("Unable to store object");
return false;
}
# Store External Storage URLs instead of Stub placeholders
foreach ($stubs as $stub) {
if ($stub === false) {
continue;
}
# $stored should provide base path to a BLOB
$url = $stored . "/" . $stub->getHash();
$dbw->update('text', array('old_text' => $url, 'old_flags' => 'external,utf-8'), array('old_id' => $stub->getReferrer()));
}
} else {
# Store the main object locally
$dbw->update('text', array('old_text' => serialize($chunk), 'old_flags' => 'object,utf-8'), array('old_id' => $primaryOldid));
# Store the stub objects
for ($j = 1; $j < $thisChunkSize; $j++) {
# Skip if not compressing and don't overwrite the first revision
if ($stubs[$j] !== false && $revs[$i + $j]->rev_text_id != $primaryOldid) {
$dbw->update('text', array('old_text' => serialize($stubs[$j]), 'old_flags' => 'object,utf-8'), array('old_id' => $revs[$i + $j]->rev_text_id));
}
}
}
}
# Done, next
$this->output("/");
$dbw->commit(__METHOD__);
$i += $thisChunkSize;
wfWaitForSlaves();
}
$this->output("\n");
}
return true;
}
function moveToExternal($cluster, $maxID, $minID = 1)
{
$fname = 'moveToExternal';
$dbw = wfGetDB(DB_MASTER);
$dbr = wfGetDB(DB_SLAVE);
$count = $maxID - $minID + 1;
$blockSize = 1000;
$numBlocks = ceil($count / $blockSize);
print "Moving text rows from {$minID} to {$maxID} to external storage\n";
$ext = new ExternalStoreDB();
$numMoved = 0;
$numStubs = 0;
for ($block = 0; $block < $numBlocks; $block++) {
$blockStart = $block * $blockSize + $minID;
$blockEnd = $blockStart + $blockSize - 1;
if (!($block % REPORTING_INTERVAL)) {
print "oldid={$blockStart}, moved={$numMoved}\n";
wfWaitForSlaves(2);
}
$res = $dbr->select('text', array('old_id', 'old_flags', 'old_text'), array("old_id BETWEEN {$blockStart} AND {$blockEnd}", "old_flags NOT LIKE '%external%'"), $fname);
while ($row = $dbr->fetchObject($res)) {
# Resolve stubs
$text = $row->old_text;
$id = $row->old_id;
if ($row->old_flags === '') {
$flags = 'external';
} else {
$flags = "{$row->old_flags},external";
}
if (strpos($flags, 'object') !== false) {
$obj = unserialize($text);
$className = strtolower(get_class($obj));
if ($className == 'historyblobstub') {
#resolveStub( $id, $row->old_text, $row->old_flags );
#$numStubs++;
continue;
} elseif ($className == 'historyblobcurstub') {
$text = gzdeflate($obj->getText());
$flags = 'utf-8,gzip,external';
} elseif ($className == 'concatenatedgziphistoryblob') {
// Do nothing
} else {
print "Warning: unrecognised object class \"{$className}\"\n";
continue;
}
} else {
$className = false;
}
if (strlen($text) < 100 && $className === false) {
// Don't move tiny revisions
continue;
}
#print "Storing " . strlen( $text ) . " bytes to $url\n";
#print "old_id=$id\n";
$url = $ext->store($cluster, $text);
if (!$url) {
print "Error writing to external storage\n";
exit;
}
$dbw->update('text', array('old_flags' => $flags, 'old_text' => $url), array('old_id' => $id), $fname);
$numMoved++;
}
$dbr->freeResult($res);
}
}
public function execute()
{
global $IP, $wgDefaultExternalStore, $wmfVersionNumber;
if (!$wmfVersionNumber) {
// set in CommonSettings.php
$this->error('$wmfVersionNumber is not set, please use MWScript.php wrapper.', true);
}
$lang = $this->getArg(0);
$site = $this->getArg(1);
$dbName = $this->getArg(2);
$domain = $this->getArg(3);
$languageNames = Language::getLanguageNames();
if (!isset($languageNames[$lang])) {
$this->error("Language {$lang} not found in Names.php", true);
}
$name = $languageNames[$lang];
$dbw = wfGetDB(DB_MASTER);
$common = "/home/wikipedia/common";
$this->output("Creating database {$dbName} for {$lang}.{$site} ({$name})\n");
# Set up the database
$dbw->query("SET table_type=Innodb");
$dbw->query("CREATE DATABASE {$dbName}");
$dbw->selectDB($dbName);
$this->output("Initialising tables\n");
$dbw->sourceFile($this->getDir() . '/tables.sql');
$dbw->sourceFile("{$IP}/extensions/OAI/update_table.sql");
$dbw->sourceFile("{$IP}/extensions/AntiSpoof/sql/patch-antispoof.mysql.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_changes.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_log.sql");
$dbw->sourceFile("{$IP}/extensions/TitleKey/titlekey.sql");
$dbw->sourceFile("{$IP}/extensions/Oversight/hidden.sql");
$dbw->sourceFile("{$IP}/extensions/GlobalBlocking/localdb_patches/setup-global_block_whitelist.sql");
$dbw->sourceFile("{$IP}/extensions/AbuseFilter/abusefilter.tables.sql");
$dbw->sourceFile("{$IP}/extensions/PrefStats/patches/PrefStats.sql");
$dbw->sourceFile("{$IP}/extensions/ProofreadPage/ProofreadPage.sql");
$dbw->sourceFile("{$IP}/extensions/ClickTracking/patches/ClickTrackingEvents.sql");
$dbw->sourceFile("{$IP}/extensions/ClickTracking/patches/ClickTracking.sql");
$dbw->sourceFile("{$IP}/extensions/UserDailyContribs/patches/UserDailyContribs.sql");
$dbw->sourceFile("{$IP}/extensions/Math/db/math.sql");
$dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)");
# Initialise external storage
if (is_array($wgDefaultExternalStore)) {
$stores = $wgDefaultExternalStore;
} elseif ($wgDefaultExternalStore) {
$stores = array($wgDefaultExternalStore);
} else {
$stores = array();
}
if (count($stores)) {
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ($stores as $storeURL) {
$m = array();
if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) {
continue;
}
$cluster = $m[1];
$this->output("Initialising external storage {$cluster}...\n");
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb = $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$dbName}");
$extdb->selectDB($dbName);
# Hack x2
$blobsTable = $store->getTable($extdb);
$sedCmd = "sed s/blobs\\\\\\>/{$blobsTable}/ " . $this->getDir() . "/storage/blobs.sql";
$blobsFile = popen($sedCmd, 'r');
$extdb->sourceStream($blobsFile);
pclose($blobsFile);
$extdb->commit();
}
}
$title = Title::newFromText(wfMessage('mainpage')->inLanguage($lang)->useDatabase(false)->plain());
$this->output("Writing main page to " . $title->getPrefixedDBkey() . "\n");
$article = new Article($title);
$ucsite = ucfirst($site);
$article->doEdit($this->getFirstArticle($ucsite, $name), '', EDIT_NEW | EDIT_AUTOSUMMARY);
$this->output("Adding to dblists\n");
# Add to dblist
$file = fopen("{$common}/all.dblist", "a");
fwrite($file, "{$dbName}\n");
fclose($file);
# Update the sublists
shell_exec("cd {$common} && ./refresh-dblist");
# Add to wikiversions.dat
$file = fopen("{$common}/wikiversions.dat", "a");
fwrite($file, "{$dbName} php-{$wmfVersionNumber}\n");
fclose($file);
# Rebuild wikiversions.cdb
shell_exec("cd {$common}/multiversion && ./refreshWikiversionsCDB");
# print "Constructing interwiki SQL\n";
# Rebuild interwiki tables
# passthru( '/home/wikipedia/conf/interwiki/update' );
$time = wfTimestamp(TS_RFC2822);
// These arguments need to be escaped twice: once for echo and once for at
$escDbName = wfEscapeShellArg(wfEscapeShellArg($dbName));
$escTime = wfEscapeShellArg(wfEscapeShellArg($time));
$escUcsite = wfEscapeShellArg(wfEscapeShellArg($ucsite));
$escName = wfEscapeShellArg(wfEscapeShellArg($name));
$escLang = wfEscapeShellArg(wfEscapeShellArg($lang));
$escDomain = wfEscapeShellArg(wfEscapeShellArg($domain));
shell_exec("echo notifyNewProjects {$escDbName} {$escTime} {$escUcsite} {$escName} {$escLang} {$escDomain} | at now + 15 minutes");
$this->output("Script ended. You still have to:\n\t* Add any required settings in InitialiseSettings.php\n\t* Run sync-common-all\n");
}
function addWiki($lang, $site, $dbName)
{
global $IP, $wgLanguageNames, $wgDefaultExternalStore;
$name = $wgLanguageNames[$lang];
$dbw = wfGetDB(DB_WRITE);
$common = "/home/wikipedia/common";
$maintenance = "{$IP}/maintenance";
print "Creating database {$dbName} for {$lang}.{$site}\n";
# Set up the database
$dbw->query("SET table_type=Innodb");
$dbw->query("CREATE DATABASE {$dbName}");
$dbw->selectDB($dbName);
print "Initialising tables\n";
dbsource("{$maintenance}/tables.sql", $dbw);
dbsource("{$IP}/extensions/OAI/update_table.sql", $dbw);
dbsource("{$IP}/extensions/AntiSpoof/mysql/patch-antispoof.sql", $dbw);
dbsource("{$IP}/extensions/CheckUser/cu_changes.sql", $dbw);
$dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)");
# Initialise external storage
if (is_array($wgDefaultExternalStore)) {
$stores = $wgDefaultExternalStore;
} elseif ($stores) {
$stores = array($wgDefaultExternalStore);
} else {
$stores = array();
}
if (count($stores)) {
require_once 'ExternalStoreDB.php';
print "Initialising external storage {$store}...\n";
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ($stores as $storeURL) {
$m = array();
if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) {
continue;
}
$cluster = $m[1];
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb =& $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$dbName}");
$extdb->selectDB($dbName);
dbsource("{$maintenance}/storage/blobs.sql", $extdb);
$extdb->immediateCommit();
}
}
global $wgTitle, $wgArticle;
$wgTitle = Title::newMainPage();
$wgArticle = new Article($wgTitle);
$ucsite = ucfirst($site);
$wgArticle->insertNewArticle(<<<EOT
==This subdomain is reserved for the creation of a [[wikimedia:Our projects|{$ucsite}]] in '''[[w:en:{$name}|{$name}]]''' language==
* Please '''do not start editing''' this new site. This site has a test project on the [[incubator:|Wikimedia Incubator]] (or on the [[betawikiversity:|BetaWikiversity]] or on the [[oldwikisource:|Old Wikisource]]) and it will be imported to here.
* If you would like to help translating the interface to this language, please do not translate here, but go to [[betawiki:|Betawiki]], a special wiki for translating the interface. That way everyone can use it on every wiki using the [[mw:|same software]].
* For information about how to edit and for other general help, see [[m:Help:Contents|Help on Wikimedia's Meta-Wiki]] or [[mw:Help:Contents|Help on MediaWiki.org]].
== Sister projects ==
<span class="plainlinks">
[http://www.wikipedia.org Wikipedia] |
[http://www.wiktionary.org Wiktonary] |
[http://www.wikibooks.org Wikibooks] |
[http://www.wikinews.org Wikinews] |
[http://www.wikiquote.org Wikiquote] |
[http://www.wikisource.org Wikisource]
[http://www.wikiversity.org Wikiversity]
</span>
See Wikimedia's [[m:|Meta-Wiki]] for the coordination of these projects.
[[aa:]]
[[af:]]
[[als:]]
[[ar:]]
[[de:]]
[[en:]]
[[as:]]
[[ast:]]
[[ay:]]
[[az:]]
[[bcl:]]
[[be:]]
[[bg:]]
[[bn:]]
[[bo:]]
[[bs:]]
[[cs:]]
[[co:]]
[[cs:]]
[[cy:]]
[[da:]]
[[el:]]
[[eo:]]
[[es:]]
[[et:]]
[[eu:]]
[[fa:]]
[[fi:]]
[[fr:]]
[[fy:]]
[[ga:]]
[[gl:]]
[[gn:]]
[[gu:]]
[[he:]]
[[hi:]]
[[hr:]]
[[hsb:]]
[[hy:]]
[[ia:]]
[[id:]]
[[is:]]
[[it:]]
[[ja:]]
[[ka:]]
[[kk:]]
[[km:]]
[[kn:]]
[[ko:]]
[[ks:]]
[[ku:]]
[[ky:]]
[[la:]]
[[ln:]]
[[lo:]]
[[lt:]]
[[lv:]]
[[hu:]]
[[mi:]]
[[mk:]]
[[ml:]]
[[mn:]]
[[mr:]]
[[ms:]]
[[mt:]]
[[my:]]
[[na:]]
[[nah:]]
[[nds:]]
[[ne:]]
[[nl:]]
[[no:]]
[[oc:]]
[[om:]]
[[pa:]]
[[pl:]]
[[ps:]]
[[pt:]]
[[qu:]]
[[ro:]]
[[ru:]]
[[sa:]]
[[si:]]
[[sk:]]
[[sl:]]
[[sq:]]
[[sr:]]
[[sv:]]
[[sw:]]
[[ta:]]
[[te:]]
[[tg:]]
[[th:]]
[[tk:]]
[[tl:]]
[[tr:]]
[[tt:]]
[[ug:]]
[[uk:]]
[[ur:]]
[[uz:]]
[[vi:]]
[[vo:]]
[[xh:]]
[[yo:]]
[[za:]]
[[zh:]]
[[zu:]]
EOT
, '', false, false);
print "Adding to dblists\n";
# Add to dblist
$file = fopen("{$common}/all.dblist", "a");
fwrite($file, "{$dbName}\n");
fclose($file);
# Update the sublists
system("cd {$common} && ./refresh-dblist");
print "Constructing interwiki SQL\n";
# Rebuild interwiki tables
$sql = getRebuildInterwikiSQL();
$tempname = tempnam('/tmp', 'addwiki');
$file = fopen($tempname, 'w');
if (!$file) {
wfDie("Error, unable to open temporary file {$tempname}\n");
}
fwrite($file, $sql);
fclose($file);
print "Sourcing interwiki SQL\n";
dbsource($tempname, $dbw);
#unlink( $tempname );
# Create the upload dir
global $wgUploadDirectory;
if (file_exists($wgUploadDirectory)) {
echo "{$wgUploadDirectory} already exists.\n";
} else {
echo "Creating {$wgUploadDirectory}...\n";
mkdir($wgUploadDirectory, 0777);
chmod($wgUploadDirectory, 0777);
}
print "Script ended. You now want to run sync-common-all to publish *dblist files (check them for duplicates first)\n";
}
if (is_array($wgDefaultExternalStore)) {
$stores = $wgDefaultExternalStore;
} elseif ($wgDefaultExternalStore) {
$stores = array($wgDefaultExternalStore);
} else {
$stores = array();
}
if (count($stores)) {
require_once 'ExternalStoreDB.php';
print "Initialising external storage {$store}...\n";
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ($stores as $storeURL) {
$m = array();
if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) {
continue;
}
$cluster = $m[1];
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb =& $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$to}");
$extdb->query("ALTER TABLE {$from}.blobs RENAME TO {$to}.blobs");
$extdb->selectDB($from);
dbsource("{$maintenance}/storage/blobs.sql", $extdb);
$extdb->immediateCommit();
}
}
echo "done.\n";
public function execute()
{
global $IP, $wgLanguageNames, $wgDefaultExternalStore, $wgNoDBParam;
$wgNoDBParam = true;
$lang = $this->getArg(0);
$site = $this->getArg(1);
$dbName = $this->getArg(2);
if (!isset($wgLanguageNames[$lang])) {
$this->error("Language {$lang} not found in \$wgLanguageNames", true);
}
$name = $wgLanguageNames[$lang];
$dbw = wfGetDB(DB_MASTER);
$common = "/home/wikipedia/common";
$this->output("Creating database {$dbName} for {$lang}.{$site} ({$name})\n");
# Set up the database
$dbw->query("SET table_type=Innodb");
$dbw->query("CREATE DATABASE {$dbName}");
$dbw->selectDB($dbName);
$this->output("Initialising tables\n");
$dbw->sourceFile($this->getDir() . '/tables.sql');
$dbw->sourceFile("{$IP}/extensions/OAI/update_table.sql");
$dbw->sourceFile("{$IP}/extensions/AntiSpoof/sql/patch-antispoof.mysql.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_changes.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_log.sql");
$dbw->sourceFile("{$IP}/extensions/TitleKey/titlekey.sql");
$dbw->sourceFile("{$IP}/extensions/Oversight/hidden.sql");
$dbw->sourceFile("{$IP}/extensions/GlobalBlocking/localdb_patches/setup-global_block_whitelist.sql");
$dbw->sourceFile("{$IP}/extensions/AbuseFilter/abusefilter.tables.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/PrefStats/PrefStats.sql");
$dbw->sourceFile("{$IP}/extensions/ProofreadPage/ProofreadPage.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/ClickTracking/ClickTrackingEvents.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/ClickTracking/ClickTracking.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/UserDailyContribs/UserDailyContribs.sql");
$dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)");
# Initialise external storage
if (is_array($wgDefaultExternalStore)) {
$stores = $wgDefaultExternalStore;
} elseif ($stores) {
$stores = array($wgDefaultExternalStore);
} else {
$stores = array();
}
if (count($stores)) {
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ($stores as $storeURL) {
$m = array();
if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) {
continue;
}
$cluster = $m[1];
$this->output("Initialising external storage {$cluster}...\n");
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb = $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$dbName}");
$extdb->selectDB($dbName);
# Hack x2
$blobsTable = $store->getTable($extdb);
$sedCmd = "sed s/blobs\\\\\\>/{$blobsTable}/ " . $this->getDir() . "/storage/blobs.sql";
$blobsFile = popen($sedCmd, 'r');
$extdb->sourceStream($blobsFile);
pclose($blobsFile);
$extdb->commit();
}
}
global $wgTitle, $wgArticle;
$wgTitle = Title::newFromText(wfMsgWeirdKey("mainpage/{$lang}"));
$this->output("Writing main page to " . $wgTitle->getPrefixedDBkey() . "\n");
$wgArticle = new Article($wgTitle);
$ucsite = ucfirst($site);
$wgArticle->insertNewArticle($this->getFirstArticle($ucsite, $name), '', false, false);
$this->output("Adding to dblists\n");
# Add to dblist
$file = fopen("{$common}/all.dblist", "a");
fwrite($file, "{$dbName}\n");
fclose($file);
# Update the sublists
shell_exec("cd {$common} && ./refresh-dblist");
#print "Constructing interwiki SQL\n";
# Rebuild interwiki tables
#passthru( '/home/wikipedia/conf/interwiki/update' );
$this->output("Script ended. You still have to:\n\t* Add any required settings in InitialiseSettings.php\n\t* Run sync-common-all\n\t* Run /home/wikipedia/conf/interwiki/update\n\t");
}
function addWiki($lang, $site, $dbName)
{
global $IP, $wgLanguageNames, $wgDefaultExternalStore;
if (!isset($wgLanguageNames[$lang])) {
print "Language {$lang} not found in \$wgLanguageNames\n";
return;
}
$name = $wgLanguageNames[$lang];
$dbw = wfGetDB(DB_MASTER);
$common = "/home/wikipedia/common";
$maintenance = "{$IP}/maintenance";
print "Creating database {$dbName} for {$lang}.{$site} ({$name})\n";
# Set up the database
$dbw->query("SET table_type=Innodb");
$dbw->query("CREATE DATABASE {$dbName}");
$dbw->selectDB($dbName);
print "Initialising tables\n";
dbsource("{$maintenance}/tables.sql", $dbw);
dbsource("{$IP}/extensions/OAI/update_table.sql", $dbw);
dbsource("{$IP}/extensions/AntiSpoof/sql/patch-antispoof.mysql.sql", $dbw);
dbsource("{$IP}/extensions/CheckUser/cu_changes.sql", $dbw);
dbsource("{$IP}/extensions/CheckUser/cu_log.sql", $dbw);
dbsource("{$IP}/extensions/TitleKey/titlekey.sql", $dbw);
dbsource("{$IP}/extensions/Oversight/hidden.sql", $dbw);
dbsource("{$IP}/extensions/GlobalBlocking/localdb_patches/setup-global_block_whitelist.sql", $dbw);
$dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)");
# Initialise external storage
if (is_array($wgDefaultExternalStore)) {
$stores = $wgDefaultExternalStore;
} elseif ($stores) {
$stores = array($wgDefaultExternalStore);
} else {
$stores = array();
}
if (count($stores)) {
require_once 'ExternalStoreDB.php';
print "Initialising external storage {$store}...\n";
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ($stores as $storeURL) {
$m = array();
if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) {
continue;
}
$cluster = $m[1];
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb =& $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$dbName}");
$extdb->selectDB($dbName);
dbsource("{$maintenance}/storage/blobs.sql", $extdb);
$extdb->immediateCommit();
}
}
global $wgTitle, $wgArticle;
$wgTitle = Title::newFromText(wfMsgWeirdKey("mainpage/{$lang}"));
print "Writing main page to " . $wgTitle->getPrefixedDBkey() . "\n";
$wgArticle = new Article($wgTitle);
$ucsite = ucfirst($site);
$wgArticle->insertNewArticle(<<<EOT
==This subdomain is reserved for the creation of a [[wikimedia:Our projects|{$ucsite}]] in '''[[w:en:{$name}|{$name}]]''' language==
* Please '''do not start editing''' this new site. This site has a test project on the [[incubator:|Wikimedia Incubator]] (or on the [[betawikiversity:|BetaWikiversity]] or on the [[oldwikisource:|Old Wikisource]]) and it will be imported to here.
* If you would like to help translating the interface to this language, please do not translate here, but go to [[betawiki:|Betawiki]], a special wiki for translating the interface. That way everyone can use it on every wiki using the [[mw:|same software]].
* For information about how to edit and for other general help, see [[m:Help:Contents|Help on Wikimedia's Meta-Wiki]] or [[mw:Help:Contents|Help on MediaWiki.org]].
== Sister projects ==
<span class="plainlinks">
[http://www.wikipedia.org Wikipedia] |
[http://www.wiktionary.org Wiktonary] |
[http://www.wikibooks.org Wikibooks] |
[http://www.wikinews.org Wikinews] |
[http://www.wikiquote.org Wikiquote] |
[http://www.wikisource.org Wikisource]
[http://www.wikiversity.org Wikiversity]
</span>
See Wikimedia's [[m:|Meta-Wiki]] for the coordination of these projects.
[[aa:]]
[[af:]]
[[als:]]
[[ar:]]
[[de:]]
[[en:]]
[[as:]]
[[ast:]]
[[ay:]]
[[az:]]
[[bcl:]]
[[be:]]
[[bg:]]
[[bn:]]
[[bo:]]
[[bs:]]
[[cs:]]
[[co:]]
[[cs:]]
[[cy:]]
[[da:]]
[[el:]]
[[eo:]]
[[es:]]
[[et:]]
[[eu:]]
[[fa:]]
[[fi:]]
[[fr:]]
[[fy:]]
[[ga:]]
[[gl:]]
[[gn:]]
[[gu:]]
[[he:]]
[[hi:]]
[[hr:]]
[[hsb:]]
[[hy:]]
[[ia:]]
[[id:]]
[[is:]]
[[it:]]
[[ja:]]
[[ka:]]
[[kk:]]
[[km:]]
[[kn:]]
[[ko:]]
[[ks:]]
[[ku:]]
[[ky:]]
[[la:]]
[[ln:]]
[[lo:]]
[[lt:]]
[[lv:]]
[[hu:]]
[[mi:]]
[[mk:]]
[[ml:]]
[[mn:]]
[[mr:]]
[[ms:]]
[[mt:]]
[[my:]]
[[na:]]
[[nah:]]
[[nds:]]
[[ne:]]
[[nl:]]
[[no:]]
[[oc:]]
[[om:]]
[[pa:]]
[[pl:]]
[[ps:]]
[[pt:]]
[[qu:]]
[[ro:]]
[[ru:]]
[[sa:]]
[[si:]]
[[sk:]]
[[sl:]]
[[sq:]]
[[sr:]]
[[sv:]]
[[sw:]]
[[ta:]]
[[te:]]
[[tg:]]
[[th:]]
[[tk:]]
[[tl:]]
[[tr:]]
[[tt:]]
[[ug:]]
[[uk:]]
[[ur:]]
[[uz:]]
[[vi:]]
[[vo:]]
[[xh:]]
[[yo:]]
[[za:]]
[[zh:]]
[[zu:]]
EOT
, '', false, false);
print "Adding to dblists\n";
# Add to dblist
$file = fopen("{$common}/all.dblist", "a");
fwrite($file, "{$dbName}\n");
fclose($file);
# Update the sublists
shell_exec("cd {$common} && ./refresh-dblist");
#print "Constructing interwiki SQL\n";
# Rebuild interwiki tables
#passthru( '/home/wikipedia/conf/interwiki/update' );
print "Script ended. You still have to:\n* Add any required settings in InitialiseSettings.php\n* Run sync-common-all\n* Run /home/wikipedia/conf/interwiki/update\n";
}
