static function executeAndAssertDebug($command, $debug) { $exit_code = 0; $output = null; log_scriptLog("Executing {$command}"); exec($command, $output, $exit_code); if ($exit_code || $debug) { log_scriptLog('Output: ' . implode("\n", $output)); } if ($exit_code) { self::errorAndExit("Failed command: {$command} (code {$exit_code})"); } }
<?php require_once "../phplib/util.php"; $PS_COMMAND = 'ps -eo user,pid,etime,args --no-headers --sort etime'; $APACHE_USER = '******'; $PHP_EXECUTABLE = '/usr/lib/cgi-bin/php5'; $TIME_LIMIT = 3600; /*seconds */ log_scriptLog('Running killOrphanPhpProcesses.php.'); $output = OS::executeAndReturnOutput($PS_COMMAND); foreach ($output as $line) { $parts = preg_split('/ +/', $line, 4); $runningTime = getRunningTime($parts[2]); if ($parts[0] == $APACHE_USER && $runningTime > $TIME_LIMIT && $parts[3] == $PHP_EXECUTABLE) { log_scriptLog("killing process {$parts[1]}"); OS::executeAndAssert("kill -9 {$parts[1]}"); } } log_scriptLog('killOrphanPhpProcesses.php done.'); /****************************************************************************/ // ps gives us the running time in [[DD-]hh:]mm:ss format. function getRunningTime($string) { $matches = array(); preg_match("/^(?:(?:(\\d+)-)?(\\d+):)?(\\d+):(\\d+)\$/", $string, $matches); return $matches[1] * 86400 + $matches[2] * 3600 + $matches[3] * 60 + $matches[4]; }
function postProcess($fileName) { $tmpFile = tempnam($this->tmpDir, 'loc_'); log_scriptLog('* removing diacritics'); $s = file_get_contents($fileName); $s = StringUtil::unicodeToLatin($s); file_put_contents($tmpFile, $s); log_scriptLog('* removing duplicates and sorting'); OS::executeAndAssert("sort -u {$tmpFile} -o {$fileName}"); unlink($tmpFile); }
echo "Am resetat tabela..."; } foreach ($xml->query->allcategories->c as $section) { log_scriptLog('Diving into ' . $section); if (DEBUG) { echo "Citim secțiunea " . $section . "\n"; } $url = sprintf(SECTION_PAGES_TMPL_URL, $section); $xmlpages = simplexml_load_file($url); if ($xmlpages === false) { log_scriptLog('Cannot get sections pages from ' . $url); if (DEBUG) { echo "Eroare la citirea paginilor din secțiuni\n"; } exit(1); } foreach ($xmlpages->query->categorymembers->cm as $cm) { $pageId = (string) $cm->attributes()->pageid; $pageTitle = (string) $cm->attributes()->title; $ws = Model::factory('WikiSection')->create(); $ws->pageId = $pageId; $ws->section = str_replace('Articole:', '', $section); $ws->save(); if (DEBUG) { echo "{$pageTitle} ({$pageId}) ---> {$section}\n"; } log_scriptLog("Page {$pageTitle} ({$pageId}) saved in section {$section}"); } } log_scriptLog('syncWikiSections.php finished');
$doDatabaseCopy = true; $doCodeUpdate = true; for ($i = 1; $i < count($argv); $i++) { $arg = $argv[$i]; if ($arg == "-nc") { $doCodeUpdate = false; } else { if ($arg == '-nd') { $doDatabaseCopy = false; } else { OS::errorAndExit("Unknown flag: {$arg}"); } } } log_scriptLog('Running updateMirror.php with databaseCopy:' . ($doDatabaseCopy ? 'yes' : 'no') . ' codeUpdate:' . ($doCodeUpdate ? 'yes' : 'no')); if ($doDatabaseCopy) { $wget = sprintf("wget -q -O %s %s", DATABASE_TMPFILE_GZIP, DATABASE_URL); OS::executeAndAssert($wget); $gzip = sprintf("gunzip %s", DATABASE_TMPFILE_GZIP); OS::executeAndAssert($gzip); $parts = db_splitDsn(); $mysql = sprintf("mysql -h %s -u %s --password='******' %s < %s", $parts['host'], $parts['user'], $parts['password'], $parts['database'], DATABASE_TMPFILE); OS::executeAndAssert($mysql); $rm = sprintf("rm -f %s", DATABASE_TMPFILE); OS::executeAndAssert($rm); } if ($doCodeUpdate) { OS::executeAndAssert('cd ' . util_getRootPath() . '; /usr/bin/git pull --quiet'); } log_scriptLog('updateMirror.php completed successfully (against all odds)');
function generateIndexFile() { global $g_numFiles; log_scriptLog("Writing sitemap index sitemap.xml"); $f = fopen('wwwbase/sitemap.xml', 'w'); fprintf($f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); fprintf($f, "<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n"); for ($i = 1; $i <= $g_numFiles; $i++) { fprintf($f, " <sitemap>\n"); fprintf($f, " <loc>http://dexonline.ro/sitemap{$i}.xml.gz</loc>\n"); fprintf($f, " </sitemap>\n"); } fprintf($f, "</sitemapindex>\n"); fclose($f); }
<?php require_once __DIR__ . '/../phplib/util.php'; log_scriptLog('Running updateSourceCounts.php.'); foreach (Model::factory('Source')->find_many() as $src) { $src->ourDefCount = Model::factory('Definition')->where('sourceId', $src->id)->where('status', Definition::ST_ACTIVE)->count(); $src->updatePercentComplete(); $src->save(); } log_scriptLog('updateSourceCounts.php completed.');
$command .= " {$table}"; } $command .= " >> {$SQL_FILE}"; OS::executeAndAssert($command); if (!$doFullDump) { // Anonymize the User table. Handle the case for id = 0 separately, since // "insert into _User_Copy set id = 0" doesn't work (it inserts an id of 1). log_scriptLog('Anonymizing the User table'); db_execute("create table _User_Copy like User"); db_execute("insert into _User_Copy select * from User where id = 0"); db_execute("update _User_Copy set id = 0 where id = 1"); db_execute("insert into _User_Copy select * from User where id > 0"); db_execute("update _User_Copy set password = md5('1234'), email = concat(id, '@anonymous.com'), identity = null"); OS::executeAndAssert("{$COMMON_COMMAND} _User_Copy | sed 's/_User_Copy/User/g' >> {$SQL_FILE}"); db_execute("drop table _User_Copy"); log_scriptLog('Anonymizing the Source table'); db_execute("create table _Source_Copy like Source"); db_execute("insert into _Source_Copy select * from Source"); db_execute("update _Source_Copy set link = null"); OS::executeAndAssert("{$COMMON_COMMAND} _Source_Copy | sed 's/_Source_Copy/Source/g' >> {$SQL_FILE}"); db_execute("drop table _Source_Copy"); // Dump only the Definitions for which we have redistribution rights log_scriptLog('Filtering the Definition table'); OS::executeAndAssert("{$COMMON_COMMAND} Definition --lock-all-tables --where='Definition.sourceId in (select id from Source where canDistribute)' " . ">> {$SQL_FILE}"); } OS::executeAndAssert("gzip -f {$SQL_FILE}"); $f = new FtpUtil(); $f->staticServerPut($GZ_FILE, $remoteFile); unlink($GZ_FILE); log_scriptLog('dumpDatabase.php completed successfully (against all odds)');
throw new Exception("Cuvântul zilei '{$word}' are deja o imagine ataşată. Nu puteți modifica imaginile cuvintelor din trecut."); } $wotdDisplayDate = new DateTime($wotd->displayDate); $wotd->image = sprintf("%s/%s.%s", $wotdDisplayDate->format('Y-m'), $word, $imageExtension); $wotd->save(); $wotdImagePath = '/img/wotd/' . $wotd->image; $f = new FtpUtil(); $f->staticServerPut($tmpFilePath, $wotdImagePath); unlink($tmpFilePath); ReplyToEmail($sender, $subject, "Am adăugat imaginea pentru '{$word}'."); } catch (Exception $e) { unlink($tmpFilePath); log_scriptLog($e->getMessage()); ReplyToEmail($sender, $subject, $e->getMessage()); } log_scriptLog("getWotdImageByEmail: done"); /***************************************************************************/ function ReplyToEmail($senderAddress, $subject, $message) { $sender = Config::get('WotD.sender'); $replyto = Config::get('WotD.reply-to'); $headers = array("From: {$sender}", "Reply-To: {$replyto}", 'Content-Type: text/plain; charset=UTF-8'); mail($senderAddress, "Re: {$subject}", $message, implode("\r\n", $headers)); } function GetWotdFromSubject($subject) { $parts = preg_split("/\\s+/", trim($subject)); if (count($parts) != 2) { OS::errorAndExit("Ignoring message '{$subject}' due to invalid subject", 0); } if ($parts[0] != Config::get('WotD.password')) {
} log_scriptLog("Saved page #{$pageId} \"{$title}\""); } } // Now delete all the pages on our side that aren't category members because // (a) they have been deleted or // (b) they have been removed from the category $ourIds = db_getArray('select pageId from WikiArticle'); foreach ($ourIds as $ourId) { if (!array_key_exists($ourId, $pageIdHash)) { $curPage = WikiArticle::get_by_pageId($ourId); log_scriptLog("Deleting page #{$curPage->pageId} \"{$curPage->title}\""); $curPage->delete(); } } log_scriptLog('syncWikiArticles.php finished'); /*************************************************************************/ function parse($text) { // Preprocessing $text = "__NOEDITSECTION__\n" . $text; // Otherwise the returned HTML will contain section edit links $text = str_replace(array('ş', 'Ş', 'ţ', 'Ţ'), array('ș', 'Ș', 'ț', 'Ț'), $text); // Actual parsing $xmlString = util_makePostRequest(PARSER_URL, array('action' => 'parse', 'text' => $text, 'format' => 'xml', 'editsection' => false)); $xml = simplexml_load_string($xmlString); $html = (string) $xml->parse->text; if (!$html) { return false; } // Postprocessing
$runTime = DebugInfo::getRunningTimeInMillis() / 1000; $speed = round($defsSeen / $runTime); log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed ({$speed} defs/sec). " . "Word map has " . count($ifMap) . " entries. " . "Memory used: " . round(memory_get_usage() / 1048576, 1) . " MB."); } } fclose($handle); log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed."); log_scriptLog("Index size: {$indexSize} entries."); OS::executeAndAssert("chmod 666 {$fileName}"); log_scriptLog("Importing file {$fileName} into table FullTextIndex"); db_executeFromOS("load data local infile '{$fileName}' into table FullTextIndex"); util_deleteFile($fileName); if (!Lock::release(LOCK_FULL_TEXT_INDEX)) { log_scriptLog('WARNING: could not release lock!'); } log_scriptLog('rebuildFullTextIndex.php completed successfully ' . '(against all odds)'); /***************************************************************************/ function extractWords($text) { $alphabet = 'abcdefghijklmnopqrstuvwxyzăâîșț'; $text = mb_strtolower($text); $text = AdminStringUtil::removeAccents($text); $result = array(); $currentWord = ''; $chars = AdminStringUtil::unicodeExplode($text); foreach ($chars as $c) { if (strpos($alphabet, $c) !== false) { $currentWord .= $c; } else { if ($currentWord) { $result[] = $currentWord;
<?php require_once __DIR__ . '/../phplib/util.php'; log_scriptLog('Running rebuildFirefoxSpellChecker.php.'); $tmpDir = tempnam(Config::get('global.tempDir'), 'xpi_'); log_scriptLog('Setting up directories'); chdir(util_getRootPath()); OS::executeAndAssert("rm {$tmpDir}"); OS::executeAndAssert("mkdir {$tmpDir}"); OS::executeAndAssert("mkdir {$tmpDir}/chrome"); OS::executeAndAssert("mkdir {$tmpDir}/dictionaries"); OS::executeAndAssert("echo 'SET UTF-8' > {$tmpDir}/dictionaries/ro-dex.aff"); OS::executeAndAssert("cp docs/install.rdf {$tmpDir}/"); $mysqlFile = tempnam(Config::get('global.tempDir'), 'mysql_'); unlink($mysqlFile); $query = "select distinct formNoAccent from InflectedForm where formNoAccent rlike '^[a-zăâîșț]+\$' into outfile '{$mysqlFile}'"; log_scriptLog("Running mysql query: [{$query}]"); db_execute($query); log_scriptLog("Prepending line count"); OS::executeAndAssert("wc -l {$mysqlFile} | cut -d ' ' -f 1 > {$tmpDir}/dictionaries/ro-dex.dic"); OS::executeAndAssert("cat {$mysqlFile} >> {$tmpDir}/dictionaries/ro-dex.dic"); log_scriptLog("Zipping"); OS::executeAndAssert("cd {$tmpDir} && zip -r dex-ff.xpi *"); $f = new FtpUtil(); $f->staticServerPut("{$tmpDir}/dex-ff.xpi", '/download/dex-ff.xpi'); OS::executeAndAssert("rm -rf {$tmpDir}"); log_scriptLog('rebuildFirefoxSpellChecker.php completed successfully (against all odds)'); // Note -- this leaves behind the temporary MySQL file created by "... into outfile...". // The file is owned by mysql so we cannot delete it.
function removeOldDumps($folder, $today, $lastDump) { log_scriptLog('removing old dumps'); foreach (scandir($folder) as $file) { if (preg_match('/^(\\d\\d\\d\\d-\\d\\d-\\d\\d)-(abbrevs|definitions|inflections|ldm|lexems|sources).xml.gz$/', $file, $matches)) { $date = $matches[1]; if ($date != $today && $date != $lastDump) { log_scriptLog(" deleting {$file}"); unlink("{$folder}/{$file}"); } } } }
$values = array(); foreach ($dbResult as $cnt => $row) { $lexem = Model::factory('Lexem')->create($row); $ngrams = NGram::split($lexem->formNoAccent); foreach ($ngrams as $i => $ngram) { $values[] = array($ngram, $i, $lexem->id); } if (count($values) >= INSERT_SIZE) { dumpValues($values); $values = array(); } if ($cnt % 1000 == 0) { log_scriptLog(sprintf("%d lexems processed, %0.3f lexems/second.", $cnt, $cnt / (microtime(true) - $start))); } } dumpValues($values); $end = microtime(true); log_scriptLog(sprintf("genNGram.php completed in %0.3f seconds\n", $end - $start)); /*********************************************************************/ function dumpValues($values) { // Assemble low-level MySQL query. Idiorm inserts records one by one, which is many times slower. $query = 'insert into NGram(ngram, pos, lexemId) values '; foreach ($values as $i => $set) { if ($i) { $query .= ','; } $query .= sprintf("('%s', %d, %d)", addslashes($set[0]), $set[1], $set[2]); } db_execute($query); }
function addMessage($type, $date, $text) { global $messages; $messages[] = array('type' => $type, 'date' => $date, 'text' => $text); log_scriptLog("checkWotd: adding message [{$type}] [{$date}] [{$text}]"); }
<?php require_once "../phplib/util.php"; log_scriptLog('Running purgeOldData.php'); $thirtyOneDaysAgo = time() - 31 * 24 * 3600; $cookies = Model::factory('Cookie')->where_lt('createDate', $thirtyOneDaysAgo)->find_many(); foreach ($cookies as $cookie) { $cookie->delete(); } $yesterday = time() - 24 * 3600; $pts = Model::factory('PasswordToken')->where_lt('createDate', $yesterday)->find_many(); foreach ($pts as $pt) { $pt->delete(); } log_scriptLog('purgeOldData.php completed successfully (against all odds)');
function removeOldDumps($folder, $today, $lastDump) { global $FTP, $STATIC_FILES; log_scriptLog('removing old dumps'); foreach ($STATIC_FILES as $file) { $matches = array(); $file = trim($file); if (preg_match(":^{$folder}/(\\d\\d\\d\\d-\\d\\d-\\d\\d)-(abbrevs|definitions|inflections|ldm|lexems|sources).xml.gz\$:", $file, $matches)) { $date = $matches[1]; if ($date != $today && $date != $lastDump) { log_scriptLog(" deleting {$file}"); $FTP->staticServerDelete($file); } } } }
// Gives each lexem a frequency between 0.00 and 1.00 // Stop words defined in stringUtil.php get 1.00 // Other lexems get frequencies distributed uniformly between 0.01 and 1.00 based on their percentile rankings in the full text index. require_once '../phplib/util.php'; ini_set('max_execution_time', '3600'); ini_set('memory_limit', '256M'); assert_options(ASSERT_BAIL, 1); log_scriptLog('Running rebuildLexemFrequencies.php.'); log_scriptLog('Setting frequency to 1.00 for manual stop words'); foreach (StringUtil::$STOPWORDS as $sw) { $lexems = Lexem::get_all_by_formNoAccent($sw); foreach ($lexems as $l) { $l->frequency = 1.0; $l->save(); } } log_scriptLog("Scanning full text index"); $dbResult = db_execute("select lexemId from FullTextIndex group by lexemId order by count(*)"); $numLexems = $dbResult->rowCount(); $i = 0; foreach ($dbResult as $row) { $lexem = Lexem::get_by_id($row[0]); $lexem->frequency = round($i / $numLexems + 0.005, 2); $lexem->save(); $i++; if ($i % 10000 == 0) { log_scriptLog("{$i} of {$numLexems} labeled"); } } log_scriptLog('rebuildLexemFrequencies.php completed successfully');
log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_5"); $user->medalMask |= Medal::MEDAL_VOLUNTEER_5; } else { if ($e->numChars >= 1000000 && $e->numChars < 10000000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_4)) { log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_4"); $user->medalMask |= Medal::MEDAL_VOLUNTEER_4; } else { if ($e->numChars >= 100000 && $e->numChars < 1000000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_3)) { log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_3"); $user->medalMask |= Medal::MEDAL_VOLUNTEER_3; } else { if ($e->numChars >= 10000 && $e->numChars < 100000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_2)) { log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_2"); $user->medalMask |= Medal::MEDAL_VOLUNTEER_2; } else { if ($e->numChars >= 1000 && $e->numChars < 10000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_1)) { log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_1"); $user->medalMask |= Medal::MEDAL_VOLUNTEER_1; } } } } } $user->medalMask = Medal::getCanonicalMask($user->medalMask); if (!$dryRun) { $user->save(); } } } log_scriptLog("updateMedals: ending"); /*********************************************************************/