static function executeAndAssertDebug($command, $debug)
{
$exit_code = 0;
$output = null;
log_scriptLog("Executing {$command}");
exec($command, $output, $exit_code);
if ($exit_code || $debug) {
log_scriptLog('Output: ' . implode("\n", $output));
}
if ($exit_code) {
self::errorAndExit("Failed command: {$command} (code {$exit_code})");
}
}
<?php
require_once "../phplib/util.php";
$PS_COMMAND = 'ps -eo user,pid,etime,args --no-headers --sort etime';
$APACHE_USER = '******';
$PHP_EXECUTABLE = '/usr/lib/cgi-bin/php5';
$TIME_LIMIT = 3600;
/*seconds */
log_scriptLog('Running killOrphanPhpProcesses.php.');
$output = OS::executeAndReturnOutput($PS_COMMAND);
foreach ($output as $line) {
$parts = preg_split('/ +/', $line, 4);
$runningTime = getRunningTime($parts[2]);
if ($parts[0] == $APACHE_USER && $runningTime > $TIME_LIMIT && $parts[3] == $PHP_EXECUTABLE) {
log_scriptLog("killing process {$parts[1]}");
OS::executeAndAssert("kill -9 {$parts[1]}");
}
}
log_scriptLog('killOrphanPhpProcesses.php done.');
/****************************************************************************/
// ps gives us the running time in [[DD-]hh:]mm:ss format.
function getRunningTime($string)
{
$matches = array();
preg_match("/^(?:(?:(\\d+)-)?(\\d+):)?(\\d+):(\\d+)\$/", $string, $matches);
return $matches[1] * 86400 + $matches[2] * 3600 + $matches[3] * 60 + $matches[4];
}
function postProcess($fileName)
{
$tmpFile = tempnam($this->tmpDir, 'loc_');
log_scriptLog('* removing diacritics');
$s = file_get_contents($fileName);
$s = StringUtil::unicodeToLatin($s);
file_put_contents($tmpFile, $s);
log_scriptLog('* removing duplicates and sorting');
OS::executeAndAssert("sort -u {$tmpFile} -o {$fileName}");
unlink($tmpFile);
}
echo "Am resetat tabela...";
}
foreach ($xml->query->allcategories->c as $section) {
log_scriptLog('Diving into ' . $section);
if (DEBUG) {
echo "Citim secțiunea " . $section . "\n";
}
$url = sprintf(SECTION_PAGES_TMPL_URL, $section);
$xmlpages = simplexml_load_file($url);
if ($xmlpages === false) {
log_scriptLog('Cannot get sections pages from ' . $url);
if (DEBUG) {
echo "Eroare la citirea paginilor din secțiuni\n";
}
exit(1);
}
foreach ($xmlpages->query->categorymembers->cm as $cm) {
$pageId = (string) $cm->attributes()->pageid;
$pageTitle = (string) $cm->attributes()->title;
$ws = Model::factory('WikiSection')->create();
$ws->pageId = $pageId;
$ws->section = str_replace('Articole:', '', $section);
$ws->save();
if (DEBUG) {
echo "{$pageTitle} ({$pageId}) ---> {$section}\n";
}
log_scriptLog("Page {$pageTitle} ({$pageId}) saved in section {$section}");
}
}
log_scriptLog('syncWikiSections.php finished');
$doDatabaseCopy = true;
$doCodeUpdate = true;
for ($i = 1; $i < count($argv); $i++) {
$arg = $argv[$i];
if ($arg == "-nc") {
$doCodeUpdate = false;
} else {
if ($arg == '-nd') {
$doDatabaseCopy = false;
} else {
OS::errorAndExit("Unknown flag: {$arg}");
}
}
}
log_scriptLog('Running updateMirror.php with databaseCopy:' . ($doDatabaseCopy ? 'yes' : 'no') . ' codeUpdate:' . ($doCodeUpdate ? 'yes' : 'no'));
if ($doDatabaseCopy) {
$wget = sprintf("wget -q -O %s %s", DATABASE_TMPFILE_GZIP, DATABASE_URL);
OS::executeAndAssert($wget);
$gzip = sprintf("gunzip %s", DATABASE_TMPFILE_GZIP);
OS::executeAndAssert($gzip);
$parts = db_splitDsn();
$mysql = sprintf("mysql -h %s -u %s --password='******' %s < %s", $parts['host'], $parts['user'], $parts['password'], $parts['database'], DATABASE_TMPFILE);
OS::executeAndAssert($mysql);
$rm = sprintf("rm -f %s", DATABASE_TMPFILE);
OS::executeAndAssert($rm);
}
if ($doCodeUpdate) {
OS::executeAndAssert('cd ' . util_getRootPath() . '; /usr/bin/git pull --quiet');
}
log_scriptLog('updateMirror.php completed successfully (against all odds)');
function generateIndexFile()
{
global $g_numFiles;
log_scriptLog("Writing sitemap index sitemap.xml");
$f = fopen('wwwbase/sitemap.xml', 'w');
fprintf($f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
fprintf($f, "<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
for ($i = 1; $i <= $g_numFiles; $i++) {
fprintf($f, " <sitemap>\n");
fprintf($f, " <loc>http://dexonline.ro/sitemap{$i}.xml.gz</loc>\n");
fprintf($f, " </sitemap>\n");
}
fprintf($f, "</sitemapindex>\n");
fclose($f);
}
<?php
require_once __DIR__ . '/../phplib/util.php';
log_scriptLog('Running updateSourceCounts.php.');
foreach (Model::factory('Source')->find_many() as $src) {
$src->ourDefCount = Model::factory('Definition')->where('sourceId', $src->id)->where('status', Definition::ST_ACTIVE)->count();
$src->updatePercentComplete();
$src->save();
}
log_scriptLog('updateSourceCounts.php completed.');
$command .= " {$table}";
}
$command .= " >> {$SQL_FILE}";
OS::executeAndAssert($command);
if (!$doFullDump) {
// Anonymize the User table. Handle the case for id = 0 separately, since
// "insert into _User_Copy set id = 0" doesn't work (it inserts an id of 1).
log_scriptLog('Anonymizing the User table');
db_execute("create table _User_Copy like User");
db_execute("insert into _User_Copy select * from User where id = 0");
db_execute("update _User_Copy set id = 0 where id = 1");
db_execute("insert into _User_Copy select * from User where id > 0");
db_execute("update _User_Copy set password = md5('1234'), email = concat(id, '@anonymous.com'), identity = null");
OS::executeAndAssert("{$COMMON_COMMAND} _User_Copy | sed 's/_User_Copy/User/g' >> {$SQL_FILE}");
db_execute("drop table _User_Copy");
log_scriptLog('Anonymizing the Source table');
db_execute("create table _Source_Copy like Source");
db_execute("insert into _Source_Copy select * from Source");
db_execute("update _Source_Copy set link = null");
OS::executeAndAssert("{$COMMON_COMMAND} _Source_Copy | sed 's/_Source_Copy/Source/g' >> {$SQL_FILE}");
db_execute("drop table _Source_Copy");
// Dump only the Definitions for which we have redistribution rights
log_scriptLog('Filtering the Definition table');
OS::executeAndAssert("{$COMMON_COMMAND} Definition --lock-all-tables --where='Definition.sourceId in (select id from Source where canDistribute)' " . ">> {$SQL_FILE}");
}
OS::executeAndAssert("gzip -f {$SQL_FILE}");
$f = new FtpUtil();
$f->staticServerPut($GZ_FILE, $remoteFile);
unlink($GZ_FILE);
log_scriptLog('dumpDatabase.php completed successfully (against all odds)');
throw new Exception("Cuvântul zilei '{$word}' are deja o imagine ataşată. Nu puteți modifica imaginile cuvintelor din trecut.");
}
$wotdDisplayDate = new DateTime($wotd->displayDate);
$wotd->image = sprintf("%s/%s.%s", $wotdDisplayDate->format('Y-m'), $word, $imageExtension);
$wotd->save();
$wotdImagePath = '/img/wotd/' . $wotd->image;
$f = new FtpUtil();
$f->staticServerPut($tmpFilePath, $wotdImagePath);
unlink($tmpFilePath);
ReplyToEmail($sender, $subject, "Am adăugat imaginea pentru '{$word}'.");
} catch (Exception $e) {
unlink($tmpFilePath);
log_scriptLog($e->getMessage());
ReplyToEmail($sender, $subject, $e->getMessage());
}
log_scriptLog("getWotdImageByEmail: done");
/***************************************************************************/
function ReplyToEmail($senderAddress, $subject, $message)
{
$sender = Config::get('WotD.sender');
$replyto = Config::get('WotD.reply-to');
$headers = array("From: {$sender}", "Reply-To: {$replyto}", 'Content-Type: text/plain; charset=UTF-8');
mail($senderAddress, "Re: {$subject}", $message, implode("\r\n", $headers));
}
function GetWotdFromSubject($subject)
{
$parts = preg_split("/\\s+/", trim($subject));
if (count($parts) != 2) {
OS::errorAndExit("Ignoring message '{$subject}' due to invalid subject", 0);
}
if ($parts[0] != Config::get('WotD.password')) {
}
log_scriptLog("Saved page #{$pageId} \"{$title}\"");
}
}
// Now delete all the pages on our side that aren't category members because
// (a) they have been deleted or
// (b) they have been removed from the category
$ourIds = db_getArray('select pageId from WikiArticle');
foreach ($ourIds as $ourId) {
if (!array_key_exists($ourId, $pageIdHash)) {
$curPage = WikiArticle::get_by_pageId($ourId);
log_scriptLog("Deleting page #{$curPage->pageId} \"{$curPage->title}\"");
$curPage->delete();
}
}
log_scriptLog('syncWikiArticles.php finished');
/*************************************************************************/
function parse($text)
{
// Preprocessing
$text = "__NOEDITSECTION__\n" . $text;
// Otherwise the returned HTML will contain section edit links
$text = str_replace(array('ş', 'Ş', 'ţ', 'Ţ'), array('ș', 'Ș', 'ț', 'Ț'), $text);
// Actual parsing
$xmlString = util_makePostRequest(PARSER_URL, array('action' => 'parse', 'text' => $text, 'format' => 'xml', 'editsection' => false));
$xml = simplexml_load_string($xmlString);
$html = (string) $xml->parse->text;
if (!$html) {
return false;
}
// Postprocessing
$runTime = DebugInfo::getRunningTimeInMillis() / 1000;
$speed = round($defsSeen / $runTime);
log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed ({$speed} defs/sec). " . "Word map has " . count($ifMap) . " entries. " . "Memory used: " . round(memory_get_usage() / 1048576, 1) . " MB.");
}
}
fclose($handle);
log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed.");
log_scriptLog("Index size: {$indexSize} entries.");
OS::executeAndAssert("chmod 666 {$fileName}");
log_scriptLog("Importing file {$fileName} into table FullTextIndex");
db_executeFromOS("load data local infile '{$fileName}' into table FullTextIndex");
util_deleteFile($fileName);
if (!Lock::release(LOCK_FULL_TEXT_INDEX)) {
log_scriptLog('WARNING: could not release lock!');
}
log_scriptLog('rebuildFullTextIndex.php completed successfully ' . '(against all odds)');
/***************************************************************************/
function extractWords($text)
{
$alphabet = 'abcdefghijklmnopqrstuvwxyzăâîșț';
$text = mb_strtolower($text);
$text = AdminStringUtil::removeAccents($text);
$result = array();
$currentWord = '';
$chars = AdminStringUtil::unicodeExplode($text);
foreach ($chars as $c) {
if (strpos($alphabet, $c) !== false) {
$currentWord .= $c;
} else {
if ($currentWord) {
$result[] = $currentWord;
<?php
require_once __DIR__ . '/../phplib/util.php';
log_scriptLog('Running rebuildFirefoxSpellChecker.php.');
$tmpDir = tempnam(Config::get('global.tempDir'), 'xpi_');
log_scriptLog('Setting up directories');
chdir(util_getRootPath());
OS::executeAndAssert("rm {$tmpDir}");
OS::executeAndAssert("mkdir {$tmpDir}");
OS::executeAndAssert("mkdir {$tmpDir}/chrome");
OS::executeAndAssert("mkdir {$tmpDir}/dictionaries");
OS::executeAndAssert("echo 'SET UTF-8' > {$tmpDir}/dictionaries/ro-dex.aff");
OS::executeAndAssert("cp docs/install.rdf {$tmpDir}/");
$mysqlFile = tempnam(Config::get('global.tempDir'), 'mysql_');
unlink($mysqlFile);
$query = "select distinct formNoAccent from InflectedForm where formNoAccent rlike '^[a-zăâîșț]+\$' into outfile '{$mysqlFile}'";
log_scriptLog("Running mysql query: [{$query}]");
db_execute($query);
log_scriptLog("Prepending line count");
OS::executeAndAssert("wc -l {$mysqlFile} | cut -d ' ' -f 1 > {$tmpDir}/dictionaries/ro-dex.dic");
OS::executeAndAssert("cat {$mysqlFile} >> {$tmpDir}/dictionaries/ro-dex.dic");
log_scriptLog("Zipping");
OS::executeAndAssert("cd {$tmpDir} && zip -r dex-ff.xpi *");
$f = new FtpUtil();
$f->staticServerPut("{$tmpDir}/dex-ff.xpi", '/download/dex-ff.xpi');
OS::executeAndAssert("rm -rf {$tmpDir}");
log_scriptLog('rebuildFirefoxSpellChecker.php completed successfully (against all odds)');
// Note -- this leaves behind the temporary MySQL file created by "... into outfile...".
// The file is owned by mysql so we cannot delete it.
function removeOldDumps($folder, $today, $lastDump)
{
log_scriptLog('removing old dumps');
foreach (scandir($folder) as $file) {
if (preg_match('/^(\\d\\d\\d\\d-\\d\\d-\\d\\d)-(abbrevs|definitions|inflections|ldm|lexems|sources).xml.gz$/', $file, $matches)) {
$date = $matches[1];
if ($date != $today && $date != $lastDump) {
log_scriptLog(" deleting {$file}");
unlink("{$folder}/{$file}");
}
}
}
}
$values = array();
foreach ($dbResult as $cnt => $row) {
$lexem = Model::factory('Lexem')->create($row);
$ngrams = NGram::split($lexem->formNoAccent);
foreach ($ngrams as $i => $ngram) {
$values[] = array($ngram, $i, $lexem->id);
}
if (count($values) >= INSERT_SIZE) {
dumpValues($values);
$values = array();
}
if ($cnt % 1000 == 0) {
log_scriptLog(sprintf("%d lexems processed, %0.3f lexems/second.", $cnt, $cnt / (microtime(true) - $start)));
}
}
dumpValues($values);
$end = microtime(true);
log_scriptLog(sprintf("genNGram.php completed in %0.3f seconds\n", $end - $start));
/*********************************************************************/
function dumpValues($values)
{
// Assemble low-level MySQL query. Idiorm inserts records one by one, which is many times slower.
$query = 'insert into NGram(ngram, pos, lexemId) values ';
foreach ($values as $i => $set) {
if ($i) {
$query .= ',';
}
$query .= sprintf("('%s', %d, %d)", addslashes($set[0]), $set[1], $set[2]);
}
db_execute($query);
}
function addMessage($type, $date, $text)
{
global $messages;
$messages[] = array('type' => $type, 'date' => $date, 'text' => $text);
log_scriptLog("checkWotd: adding message [{$type}] [{$date}] [{$text}]");
}
<?php
require_once "../phplib/util.php";
log_scriptLog('Running purgeOldData.php');
$thirtyOneDaysAgo = time() - 31 * 24 * 3600;
$cookies = Model::factory('Cookie')->where_lt('createDate', $thirtyOneDaysAgo)->find_many();
foreach ($cookies as $cookie) {
$cookie->delete();
}
$yesterday = time() - 24 * 3600;
$pts = Model::factory('PasswordToken')->where_lt('createDate', $yesterday)->find_many();
foreach ($pts as $pt) {
$pt->delete();
}
log_scriptLog('purgeOldData.php completed successfully (against all odds)');
function removeOldDumps($folder, $today, $lastDump)
{
global $FTP, $STATIC_FILES;
log_scriptLog('removing old dumps');
foreach ($STATIC_FILES as $file) {
$matches = array();
$file = trim($file);
if (preg_match(":^{$folder}/(\\d\\d\\d\\d-\\d\\d-\\d\\d)-(abbrevs|definitions|inflections|ldm|lexems|sources).xml.gz\$:", $file, $matches)) {
$date = $matches[1];
if ($date != $today && $date != $lastDump) {
log_scriptLog(" deleting {$file}");
$FTP->staticServerDelete($file);
}
}
}
}
// Gives each lexem a frequency between 0.00 and 1.00
// Stop words defined in stringUtil.php get 1.00
// Other lexems get frequencies distributed uniformly between 0.01 and 1.00 based on their percentile rankings in the full text index.
require_once '../phplib/util.php';
ini_set('max_execution_time', '3600');
ini_set('memory_limit', '256M');
assert_options(ASSERT_BAIL, 1);
log_scriptLog('Running rebuildLexemFrequencies.php.');
log_scriptLog('Setting frequency to 1.00 for manual stop words');
foreach (StringUtil::$STOPWORDS as $sw) {
$lexems = Lexem::get_all_by_formNoAccent($sw);
foreach ($lexems as $l) {
$l->frequency = 1.0;
$l->save();
}
}
log_scriptLog("Scanning full text index");
$dbResult = db_execute("select lexemId from FullTextIndex group by lexemId order by count(*)");
$numLexems = $dbResult->rowCount();
$i = 0;
foreach ($dbResult as $row) {
$lexem = Lexem::get_by_id($row[0]);
$lexem->frequency = round($i / $numLexems + 0.005, 2);
$lexem->save();
$i++;
if ($i % 10000 == 0) {
log_scriptLog("{$i} of {$numLexems} labeled");
}
}
log_scriptLog('rebuildLexemFrequencies.php completed successfully');
log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_5");
$user->medalMask |= Medal::MEDAL_VOLUNTEER_5;
} else {
if ($e->numChars >= 1000000 && $e->numChars < 10000000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_4)) {
log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_4");
$user->medalMask |= Medal::MEDAL_VOLUNTEER_4;
} else {
if ($e->numChars >= 100000 && $e->numChars < 1000000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_3)) {
log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_3");
$user->medalMask |= Medal::MEDAL_VOLUNTEER_3;
} else {
if ($e->numChars >= 10000 && $e->numChars < 100000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_2)) {
log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_2");
$user->medalMask |= Medal::MEDAL_VOLUNTEER_2;
} else {
if ($e->numChars >= 1000 && $e->numChars < 10000 && !($user->medalMask & Medal::MEDAL_VOLUNTEER_1)) {
log_scriptLog("Granting {$user->nick} a MEDAL_VOLUNTEER_1");
$user->medalMask |= Medal::MEDAL_VOLUNTEER_1;
}
}
}
}
}
$user->medalMask = Medal::getCanonicalMask($user->medalMask);
if (!$dryRun) {
$user->save();
}
}
}
log_scriptLog("updateMedals: ending");
/*********************************************************************/
