/**
* hook for ExternalStoreDB::FetchBlob
*
* @param string $cluster storage identifier
* @param integer $id blob identifier
* @param integer $itemID item identifier when revision text is merged & archived
* @param string $ret returned blob text
*/
function ExternalStoreDBFetchBlobHook($cluster, $id, $itemID, &$ret)
{
global $wgTheSchwartzSecretToken;
wfProfileIn(__METHOD__);
// there's already blob text
if ($ret !== false) {
wfProfileOut(__METHOD__);
return true;
}
// wikia doesn't use $itemID
$url = sprintf("%s?action=fetchblob&store=%s&id=%d&token=%s&format=json", F::app()->wg->FetchBlobApiURL, $cluster, $id, $wgTheSchwartzSecretToken);
$response = json_decode(Http::get($url, "default", array('noProxy' => true)));
if (isset($response->fetchblob)) {
$blob = isset($response->fetchblob->blob) ? $response->fetchblob->blob : false;
$hash = isset($response->fetchblob->hash) ? $response->fetchblob->hash : null;
if ($blob) {
// pack to binary
$blob = pack("H*", $blob);
$hash = md5($blob);
// check md5 sum for binary
if ($hash == $response->fetchblob->hash) {
wfDebug(__METHOD__ . ": md5 sum match\n");
$ret = $blob;
// now store blob in local database but only when it's Poznan's devbox
$isPoznanDevbox = F::app()->wg->DevelEnvironment === true && F::app()->wg->WikiaDatacenter == "poz";
if ($isPoznanDevbox) {
wfDebug(__METHOD__ . ": this is poznań devbox\n");
$store = new ExternalStoreDB();
$dbw = $store->getMaster($cluster);
if ($dbw) {
wfDebug(__METHOD__ . ": storing blob {$id} on local storage {$cluster}\n");
$dbw->insert($store->getTable($dbw), array("blob_id" => $id, "blob_text" => $ret), __METHOD__, array("IGNORE"));
$dbw->commit();
}
}
} else {
wfDebug(__METHOD__ . ": md5 sum not match, {$hash} != {$response->fetchblob}->hash\n");
}
}
} else {
wfDebug(__METHOD__ . ": malformed response from API call\n");
}
wfProfileOut(__METHOD__);
return true;
}
public function execute()
{
global $IP, $wgDefaultExternalStore, $wmfVersionNumber;
if (!$wmfVersionNumber) {
// set in CommonSettings.php
$this->error('$wmfVersionNumber is not set, please use MWScript.php wrapper.', true);
}
$lang = $this->getArg(0);
$site = $this->getArg(1);
$dbName = $this->getArg(2);
$domain = $this->getArg(3);
$languageNames = Language::getLanguageNames();
if (!isset($languageNames[$lang])) {
$this->error("Language {$lang} not found in Names.php", true);
}
$name = $languageNames[$lang];
$dbw = wfGetDB(DB_MASTER);
$common = "/home/wikipedia/common";
$this->output("Creating database {$dbName} for {$lang}.{$site} ({$name})\n");
# Set up the database
$dbw->query("SET table_type=Innodb");
$dbw->query("CREATE DATABASE {$dbName}");
$dbw->selectDB($dbName);
$this->output("Initialising tables\n");
$dbw->sourceFile($this->getDir() . '/tables.sql');
$dbw->sourceFile("{$IP}/extensions/OAI/update_table.sql");
$dbw->sourceFile("{$IP}/extensions/AntiSpoof/sql/patch-antispoof.mysql.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_changes.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_log.sql");
$dbw->sourceFile("{$IP}/extensions/TitleKey/titlekey.sql");
$dbw->sourceFile("{$IP}/extensions/Oversight/hidden.sql");
$dbw->sourceFile("{$IP}/extensions/GlobalBlocking/localdb_patches/setup-global_block_whitelist.sql");
$dbw->sourceFile("{$IP}/extensions/AbuseFilter/abusefilter.tables.sql");
$dbw->sourceFile("{$IP}/extensions/PrefStats/patches/PrefStats.sql");
$dbw->sourceFile("{$IP}/extensions/ProofreadPage/ProofreadPage.sql");
$dbw->sourceFile("{$IP}/extensions/ClickTracking/patches/ClickTrackingEvents.sql");
$dbw->sourceFile("{$IP}/extensions/ClickTracking/patches/ClickTracking.sql");
$dbw->sourceFile("{$IP}/extensions/UserDailyContribs/patches/UserDailyContribs.sql");
$dbw->sourceFile("{$IP}/extensions/Math/db/math.sql");
$dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)");
# Initialise external storage
if (is_array($wgDefaultExternalStore)) {
$stores = $wgDefaultExternalStore;
} elseif ($wgDefaultExternalStore) {
$stores = array($wgDefaultExternalStore);
} else {
$stores = array();
}
if (count($stores)) {
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ($stores as $storeURL) {
$m = array();
if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) {
continue;
}
$cluster = $m[1];
$this->output("Initialising external storage {$cluster}...\n");
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb = $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$dbName}");
$extdb->selectDB($dbName);
# Hack x2
$blobsTable = $store->getTable($extdb);
$sedCmd = "sed s/blobs\\\\\\>/{$blobsTable}/ " . $this->getDir() . "/storage/blobs.sql";
$blobsFile = popen($sedCmd, 'r');
$extdb->sourceStream($blobsFile);
pclose($blobsFile);
$extdb->commit();
}
}
$title = Title::newFromText(wfMessage('mainpage')->inLanguage($lang)->useDatabase(false)->plain());
$this->output("Writing main page to " . $title->getPrefixedDBkey() . "\n");
$article = new Article($title);
$ucsite = ucfirst($site);
$article->doEdit($this->getFirstArticle($ucsite, $name), '', EDIT_NEW | EDIT_AUTOSUMMARY);
$this->output("Adding to dblists\n");
# Add to dblist
$file = fopen("{$common}/all.dblist", "a");
fwrite($file, "{$dbName}\n");
fclose($file);
# Update the sublists
shell_exec("cd {$common} && ./refresh-dblist");
# Add to wikiversions.dat
$file = fopen("{$common}/wikiversions.dat", "a");
fwrite($file, "{$dbName} php-{$wmfVersionNumber}\n");
fclose($file);
# Rebuild wikiversions.cdb
shell_exec("cd {$common}/multiversion && ./refreshWikiversionsCDB");
# print "Constructing interwiki SQL\n";
# Rebuild interwiki tables
# passthru( '/home/wikipedia/conf/interwiki/update' );
$time = wfTimestamp(TS_RFC2822);
// These arguments need to be escaped twice: once for echo and once for at
$escDbName = wfEscapeShellArg(wfEscapeShellArg($dbName));
$escTime = wfEscapeShellArg(wfEscapeShellArg($time));
$escUcsite = wfEscapeShellArg(wfEscapeShellArg($ucsite));
$escName = wfEscapeShellArg(wfEscapeShellArg($name));
$escLang = wfEscapeShellArg(wfEscapeShellArg($lang));
$escDomain = wfEscapeShellArg(wfEscapeShellArg($domain));
shell_exec("echo notifyNewProjects {$escDbName} {$escTime} {$escUcsite} {$escName} {$escLang} {$escDomain} | at now + 15 minutes");
$this->output("Script ended. You still have to:\n\t* Add any required settings in InitialiseSettings.php\n\t* Run sync-common-all\n");
}
public function execute()
{
global $IP, $wgLanguageNames, $wgDefaultExternalStore, $wgNoDBParam;
$wgNoDBParam = true;
$lang = $this->getArg(0);
$site = $this->getArg(1);
$dbName = $this->getArg(2);
if (!isset($wgLanguageNames[$lang])) {
$this->error("Language {$lang} not found in \$wgLanguageNames", true);
}
$name = $wgLanguageNames[$lang];
$dbw = wfGetDB(DB_MASTER);
$common = "/home/wikipedia/common";
$this->output("Creating database {$dbName} for {$lang}.{$site} ({$name})\n");
# Set up the database
$dbw->query("SET table_type=Innodb");
$dbw->query("CREATE DATABASE {$dbName}");
$dbw->selectDB($dbName);
$this->output("Initialising tables\n");
$dbw->sourceFile($this->getDir() . '/tables.sql');
$dbw->sourceFile("{$IP}/extensions/OAI/update_table.sql");
$dbw->sourceFile("{$IP}/extensions/AntiSpoof/sql/patch-antispoof.mysql.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_changes.sql");
$dbw->sourceFile("{$IP}/extensions/CheckUser/cu_log.sql");
$dbw->sourceFile("{$IP}/extensions/TitleKey/titlekey.sql");
$dbw->sourceFile("{$IP}/extensions/Oversight/hidden.sql");
$dbw->sourceFile("{$IP}/extensions/GlobalBlocking/localdb_patches/setup-global_block_whitelist.sql");
$dbw->sourceFile("{$IP}/extensions/AbuseFilter/abusefilter.tables.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/PrefStats/PrefStats.sql");
$dbw->sourceFile("{$IP}/extensions/ProofreadPage/ProofreadPage.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/ClickTracking/ClickTrackingEvents.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/ClickTracking/ClickTracking.sql");
$dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/UserDailyContribs/UserDailyContribs.sql");
$dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)");
# Initialise external storage
if (is_array($wgDefaultExternalStore)) {
$stores = $wgDefaultExternalStore;
} elseif ($stores) {
$stores = array($wgDefaultExternalStore);
} else {
$stores = array();
}
if (count($stores)) {
global $wgDBuser, $wgDBpassword, $wgExternalServers;
foreach ($stores as $storeURL) {
$m = array();
if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) {
continue;
}
$cluster = $m[1];
$this->output("Initialising external storage {$cluster}...\n");
# Hack
$wgExternalServers[$cluster][0]['user'] = $wgDBuser;
$wgExternalServers[$cluster][0]['password'] = $wgDBpassword;
$store = new ExternalStoreDB();
$extdb = $store->getMaster($cluster);
$extdb->query("SET table_type=InnoDB");
$extdb->query("CREATE DATABASE {$dbName}");
$extdb->selectDB($dbName);
# Hack x2
$blobsTable = $store->getTable($extdb);
$sedCmd = "sed s/blobs\\\\\\>/{$blobsTable}/ " . $this->getDir() . "/storage/blobs.sql";
$blobsFile = popen($sedCmd, 'r');
$extdb->sourceStream($blobsFile);
pclose($blobsFile);
$extdb->commit();
}
}
global $wgTitle, $wgArticle;
$wgTitle = Title::newFromText(wfMsgWeirdKey("mainpage/{$lang}"));
$this->output("Writing main page to " . $wgTitle->getPrefixedDBkey() . "\n");
$wgArticle = new Article($wgTitle);
$ucsite = ucfirst($site);
$wgArticle->insertNewArticle($this->getFirstArticle($ucsite, $name), '', false, false);
$this->output("Adding to dblists\n");
# Add to dblist
$file = fopen("{$common}/all.dblist", "a");
fwrite($file, "{$dbName}\n");
fclose($file);
# Update the sublists
shell_exec("cd {$common} && ./refresh-dblist");
#print "Constructing interwiki SQL\n";
# Rebuild interwiki tables
#passthru( '/home/wikipedia/conf/interwiki/update' );
$this->output("Script ended. You still have to:\n\t* Add any required settings in InitialiseSettings.php\n\t* Run sync-common-all\n\t* Run /home/wikipedia/conf/interwiki/update\n\t");
}
