/** * hook for ExternalStoreDB::FetchBlob * * @param string $cluster storage identifier * @param integer $id blob identifier * @param integer $itemID item identifier when revision text is merged & archived * @param string $ret returned blob text */ function ExternalStoreDBFetchBlobHook($cluster, $id, $itemID, &$ret) { global $wgTheSchwartzSecretToken; wfProfileIn(__METHOD__); // there's already blob text if ($ret !== false) { wfProfileOut(__METHOD__); return true; } // wikia doesn't use $itemID $url = sprintf("%s?action=fetchblob&store=%s&id=%d&token=%s&format=json", F::app()->wg->FetchBlobApiURL, $cluster, $id, $wgTheSchwartzSecretToken); $response = json_decode(Http::get($url, "default", array('noProxy' => true))); if (isset($response->fetchblob)) { $blob = isset($response->fetchblob->blob) ? $response->fetchblob->blob : false; $hash = isset($response->fetchblob->hash) ? $response->fetchblob->hash : null; if ($blob) { // pack to binary $blob = pack("H*", $blob); $hash = md5($blob); // check md5 sum for binary if ($hash == $response->fetchblob->hash) { wfDebug(__METHOD__ . ": md5 sum match\n"); $ret = $blob; // now store blob in local database but only when it's Poznan's devbox $isPoznanDevbox = F::app()->wg->DevelEnvironment === true && F::app()->wg->WikiaDatacenter == "poz"; if ($isPoznanDevbox) { wfDebug(__METHOD__ . ": this is poznań devbox\n"); $store = new ExternalStoreDB(); $dbw = $store->getMaster($cluster); if ($dbw) { wfDebug(__METHOD__ . ": storing blob {$id} on local storage {$cluster}\n"); $dbw->insert($store->getTable($dbw), array("blob_id" => $id, "blob_text" => $ret), __METHOD__, array("IGNORE")); $dbw->commit(); } } } else { wfDebug(__METHOD__ . ": md5 sum not match, {$hash} != {$response->fetchblob}->hash\n"); } } } else { wfDebug(__METHOD__ . ": malformed response from API call\n"); } wfProfileOut(__METHOD__); return true; }
public function execute() { global $IP, $wgDefaultExternalStore, $wmfVersionNumber; if (!$wmfVersionNumber) { // set in CommonSettings.php $this->error('$wmfVersionNumber is not set, please use MWScript.php wrapper.', true); } $lang = $this->getArg(0); $site = $this->getArg(1); $dbName = $this->getArg(2); $domain = $this->getArg(3); $languageNames = Language::getLanguageNames(); if (!isset($languageNames[$lang])) { $this->error("Language {$lang} not found in Names.php", true); } $name = $languageNames[$lang]; $dbw = wfGetDB(DB_MASTER); $common = "/home/wikipedia/common"; $this->output("Creating database {$dbName} for {$lang}.{$site} ({$name})\n"); # Set up the database $dbw->query("SET table_type=Innodb"); $dbw->query("CREATE DATABASE {$dbName}"); $dbw->selectDB($dbName); $this->output("Initialising tables\n"); $dbw->sourceFile($this->getDir() . '/tables.sql'); $dbw->sourceFile("{$IP}/extensions/OAI/update_table.sql"); $dbw->sourceFile("{$IP}/extensions/AntiSpoof/sql/patch-antispoof.mysql.sql"); $dbw->sourceFile("{$IP}/extensions/CheckUser/cu_changes.sql"); $dbw->sourceFile("{$IP}/extensions/CheckUser/cu_log.sql"); $dbw->sourceFile("{$IP}/extensions/TitleKey/titlekey.sql"); $dbw->sourceFile("{$IP}/extensions/Oversight/hidden.sql"); $dbw->sourceFile("{$IP}/extensions/GlobalBlocking/localdb_patches/setup-global_block_whitelist.sql"); $dbw->sourceFile("{$IP}/extensions/AbuseFilter/abusefilter.tables.sql"); $dbw->sourceFile("{$IP}/extensions/PrefStats/patches/PrefStats.sql"); $dbw->sourceFile("{$IP}/extensions/ProofreadPage/ProofreadPage.sql"); $dbw->sourceFile("{$IP}/extensions/ClickTracking/patches/ClickTrackingEvents.sql"); $dbw->sourceFile("{$IP}/extensions/ClickTracking/patches/ClickTracking.sql"); $dbw->sourceFile("{$IP}/extensions/UserDailyContribs/patches/UserDailyContribs.sql"); $dbw->sourceFile("{$IP}/extensions/Math/db/math.sql"); $dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)"); # Initialise external storage if (is_array($wgDefaultExternalStore)) { $stores = $wgDefaultExternalStore; } elseif ($wgDefaultExternalStore) { $stores = array($wgDefaultExternalStore); } else { $stores = array(); } if (count($stores)) { global $wgDBuser, $wgDBpassword, $wgExternalServers; foreach ($stores as $storeURL) { $m = array(); if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) { continue; } $cluster = $m[1]; $this->output("Initialising external storage {$cluster}...\n"); # Hack $wgExternalServers[$cluster][0]['user'] = $wgDBuser; $wgExternalServers[$cluster][0]['password'] = $wgDBpassword; $store = new ExternalStoreDB(); $extdb = $store->getMaster($cluster); $extdb->query("SET table_type=InnoDB"); $extdb->query("CREATE DATABASE {$dbName}"); $extdb->selectDB($dbName); # Hack x2 $blobsTable = $store->getTable($extdb); $sedCmd = "sed s/blobs\\\\\\>/{$blobsTable}/ " . $this->getDir() . "/storage/blobs.sql"; $blobsFile = popen($sedCmd, 'r'); $extdb->sourceStream($blobsFile); pclose($blobsFile); $extdb->commit(); } } $title = Title::newFromText(wfMessage('mainpage')->inLanguage($lang)->useDatabase(false)->plain()); $this->output("Writing main page to " . $title->getPrefixedDBkey() . "\n"); $article = new Article($title); $ucsite = ucfirst($site); $article->doEdit($this->getFirstArticle($ucsite, $name), '', EDIT_NEW | EDIT_AUTOSUMMARY); $this->output("Adding to dblists\n"); # Add to dblist $file = fopen("{$common}/all.dblist", "a"); fwrite($file, "{$dbName}\n"); fclose($file); # Update the sublists shell_exec("cd {$common} && ./refresh-dblist"); # Add to wikiversions.dat $file = fopen("{$common}/wikiversions.dat", "a"); fwrite($file, "{$dbName} php-{$wmfVersionNumber}\n"); fclose($file); # Rebuild wikiversions.cdb shell_exec("cd {$common}/multiversion && ./refreshWikiversionsCDB"); # print "Constructing interwiki SQL\n"; # Rebuild interwiki tables # passthru( '/home/wikipedia/conf/interwiki/update' ); $time = wfTimestamp(TS_RFC2822); // These arguments need to be escaped twice: once for echo and once for at $escDbName = wfEscapeShellArg(wfEscapeShellArg($dbName)); $escTime = wfEscapeShellArg(wfEscapeShellArg($time)); $escUcsite = wfEscapeShellArg(wfEscapeShellArg($ucsite)); $escName = wfEscapeShellArg(wfEscapeShellArg($name)); $escLang = wfEscapeShellArg(wfEscapeShellArg($lang)); $escDomain = wfEscapeShellArg(wfEscapeShellArg($domain)); shell_exec("echo notifyNewProjects {$escDbName} {$escTime} {$escUcsite} {$escName} {$escLang} {$escDomain} | at now + 15 minutes"); $this->output("Script ended. You still have to:\n\t* Add any required settings in InitialiseSettings.php\n\t* Run sync-common-all\n"); }
public function execute() { global $IP, $wgLanguageNames, $wgDefaultExternalStore, $wgNoDBParam; $wgNoDBParam = true; $lang = $this->getArg(0); $site = $this->getArg(1); $dbName = $this->getArg(2); if (!isset($wgLanguageNames[$lang])) { $this->error("Language {$lang} not found in \$wgLanguageNames", true); } $name = $wgLanguageNames[$lang]; $dbw = wfGetDB(DB_MASTER); $common = "/home/wikipedia/common"; $this->output("Creating database {$dbName} for {$lang}.{$site} ({$name})\n"); # Set up the database $dbw->query("SET table_type=Innodb"); $dbw->query("CREATE DATABASE {$dbName}"); $dbw->selectDB($dbName); $this->output("Initialising tables\n"); $dbw->sourceFile($this->getDir() . '/tables.sql'); $dbw->sourceFile("{$IP}/extensions/OAI/update_table.sql"); $dbw->sourceFile("{$IP}/extensions/AntiSpoof/sql/patch-antispoof.mysql.sql"); $dbw->sourceFile("{$IP}/extensions/CheckUser/cu_changes.sql"); $dbw->sourceFile("{$IP}/extensions/CheckUser/cu_log.sql"); $dbw->sourceFile("{$IP}/extensions/TitleKey/titlekey.sql"); $dbw->sourceFile("{$IP}/extensions/Oversight/hidden.sql"); $dbw->sourceFile("{$IP}/extensions/GlobalBlocking/localdb_patches/setup-global_block_whitelist.sql"); $dbw->sourceFile("{$IP}/extensions/AbuseFilter/abusefilter.tables.sql"); $dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/PrefStats/PrefStats.sql"); $dbw->sourceFile("{$IP}/extensions/ProofreadPage/ProofreadPage.sql"); $dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/ClickTracking/ClickTrackingEvents.sql"); $dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/ClickTracking/ClickTracking.sql"); $dbw->sourceFile("{$IP}/extensions/UsabilityInitiative/UserDailyContribs/UserDailyContribs.sql"); $dbw->query("INSERT INTO site_stats(ss_row_id) VALUES (1)"); # Initialise external storage if (is_array($wgDefaultExternalStore)) { $stores = $wgDefaultExternalStore; } elseif ($stores) { $stores = array($wgDefaultExternalStore); } else { $stores = array(); } if (count($stores)) { global $wgDBuser, $wgDBpassword, $wgExternalServers; foreach ($stores as $storeURL) { $m = array(); if (!preg_match('!^DB://(.*)$!', $storeURL, $m)) { continue; } $cluster = $m[1]; $this->output("Initialising external storage {$cluster}...\n"); # Hack $wgExternalServers[$cluster][0]['user'] = $wgDBuser; $wgExternalServers[$cluster][0]['password'] = $wgDBpassword; $store = new ExternalStoreDB(); $extdb = $store->getMaster($cluster); $extdb->query("SET table_type=InnoDB"); $extdb->query("CREATE DATABASE {$dbName}"); $extdb->selectDB($dbName); # Hack x2 $blobsTable = $store->getTable($extdb); $sedCmd = "sed s/blobs\\\\\\>/{$blobsTable}/ " . $this->getDir() . "/storage/blobs.sql"; $blobsFile = popen($sedCmd, 'r'); $extdb->sourceStream($blobsFile); pclose($blobsFile); $extdb->commit(); } } global $wgTitle, $wgArticle; $wgTitle = Title::newFromText(wfMsgWeirdKey("mainpage/{$lang}")); $this->output("Writing main page to " . $wgTitle->getPrefixedDBkey() . "\n"); $wgArticle = new Article($wgTitle); $ucsite = ucfirst($site); $wgArticle->insertNewArticle($this->getFirstArticle($ucsite, $name), '', false, false); $this->output("Adding to dblists\n"); # Add to dblist $file = fopen("{$common}/all.dblist", "a"); fwrite($file, "{$dbName}\n"); fclose($file); # Update the sublists shell_exec("cd {$common} && ./refresh-dblist"); #print "Constructing interwiki SQL\n"; # Rebuild interwiki tables #passthru( '/home/wikipedia/conf/interwiki/update' ); $this->output("Script ended. You still have to:\n\t* Add any required settings in InitialiseSettings.php\n\t* Run sync-common-all\n\t* Run /home/wikipedia/conf/interwiki/update\n\t"); }