function db_executeSqlFile($filename) { $dsn = Config::get('global.database'); $parts = db_splitDsn($dsn); $command = sprintf("cat {$filename} | mysql -u %s %s %s", $parts['user'], $parts['database'], $parts['password'] ? "-p" . $parts['password'] : ''); OS::executeAndAssert($command); }
function closeCurrentFile() { global $g_numFiles; global $g_curFileName; global $g_curFile; fprintf($g_curFile, "</urlset>\n"); fclose($g_curFile); OS::executeAndAssert("gzip - < {$g_curFileName} > wwwbase/sitemap{$g_numFiles}.xml.gz"); util_deleteFile($g_curFileName); }
public function ensureThumbnail() { if (!$this->image) { return; } $fullImage = self::$IMAGE_DIR . "/{$this->image}"; $fullThumb = self::$THUMB_DIR . "/{$this->image}"; if (!file_exists($fullThumb) && file_exists($fullImage)) { $oldumask = umask(0); @mkdir(dirname($fullThumb), 0777, true); umask($oldumask); OS::executeAndAssert(sprintf("convert -strip -geometry %dx%d -sharpen 1x1 '%s' '%s'", self::$THUMB_SIZE, self::$THUMB_SIZE, $fullImage, $fullThumb)); } }
function recursiveScan($path, $logFile) { global $IGNORED_DIRS, $EXTENSIONS, $beforeBytes, $afterBytes; $files = scandir($path); foreach ($files as $file) { if (in_array($file, $IGNORED_DIRS)) { continue; } $full = "{$path}/{$file}"; if (is_dir($full)) { recursiveScan($full, $logFile); } else { $extension = pathinfo(strtolower($full), PATHINFO_EXTENSION); $fullNoExt = substr($full, 0, strlen($full) - strlen($extension) - 1); // Strip the dot as well if (in_array($extension, $EXTENSIONS)) { OS::executeAndAssert("convert -strip '{$full}' '" . Config::get('global.tempDir') . "/fileNoExif.{$extension}'"); OS::executeAndAssert("convert '" . Config::get('global.tempDir') . "/fileNoExif.{$extension}' '/fileNoExifPng.png'"); OS::executeAndAssert("optipng '" . Config::get('global.tempDir') . "/fileNoExifPng.png'"); $fs1 = filesize($full); $fs2 = filesize(Config::get('global.tempDir') . "/fileNoExif.{$extension}"); $fs3 = filesize(Config::get('global.tempDir') . '/fileNoExifPng.png'); $beforeBytes += $fs1; if ($fs3 < $fs1 && $fs3 < $fs2) { $compression = 100.0 * (1 - $fs3 / $fs1); $afterBytes += $fs3; fprintf($logFile, "%s -- Strip EXIF, convert to PNG and optimize: %d/%d bytes, %.2f%% saved\n", $full, $fs3, $fs1, $compression); unlink($full); unlink(Config::get('global.tempDir') . "/fileNoExif.{$extension}"); rename(Config::get('global.tempDir') . '/fileNoExifPng.png', "{$fullNoExt}.png"); } else { if ($fs2 < $fs1) { $compression = 100.0 * (1 - $fs2 / $fs1); $afterBytes += $fs2; fprintf($logFile, "%s -- Strip EXIF: %d/%d bytes, %.2f%% saved\n", $full, $fs2, $fs1, $compression); unlink($full); rename(Config::get('global.tempDir') . "/fileNoExif.{$extension}", $full); unlink(Config::get('global.tempDir') . '/fileNoExifPng.png'); } else { $afterBytes += $fs1; fprintf($logFile, "{$full} -- leave unchanged\n"); unlink(Config::get('global.tempDir') . "/fileNoExif.{$extension}"); unlink(Config::get('global.tempDir') . '/fileNoExifPng.png'); } } } } } }
function createThumb() { $url = Config::get('static.url') . self::STATIC_DIR . $this->path; $ext = pathinfo($url, PATHINFO_EXTENSION); $localFile = "/tmp/a.{$ext}"; $localThumbFile = "/tmp/thumb.{$ext}"; $contents = file_get_contents($url); file_put_contents($localFile, $contents); $command = sprintf("convert -strip -geometry %sx%s -sharpen 1x1 '%s' '%s'", self::THUMB_SIZE, self::THUMB_SIZE, $localFile, $localThumbFile); OS::executeAndAssert($command); $f = new FtpUtil(); $f->staticServerPut($localThumbFile, self::STATIC_THUMB_DIR . $this->path); unlink($localFile); unlink($localThumbFile); }
<?php require_once "../phplib/util.php"; $PS_COMMAND = 'ps -eo user,pid,etime,args --no-headers --sort etime'; $APACHE_USER = '******'; $PHP_EXECUTABLE = '/usr/lib/cgi-bin/php5'; $TIME_LIMIT = 3600; /*seconds */ log_scriptLog('Running killOrphanPhpProcesses.php.'); $output = OS::executeAndReturnOutput($PS_COMMAND); foreach ($output as $line) { $parts = preg_split('/ +/', $line, 4); $runningTime = getRunningTime($parts[2]); if ($parts[0] == $APACHE_USER && $runningTime > $TIME_LIMIT && $parts[3] == $PHP_EXECUTABLE) { log_scriptLog("killing process {$parts[1]}"); OS::executeAndAssert("kill -9 {$parts[1]}"); } } log_scriptLog('killOrphanPhpProcesses.php done.'); /****************************************************************************/ // ps gives us the running time in [[DD-]hh:]mm:ss format. function getRunningTime($string) { $matches = array(); preg_match("/^(?:(?:(\\d+)-)?(\\d+):)?(\\d+):(\\d+)\$/", $string, $matches); return $matches[1] * 86400 + $matches[2] * 3600 + $matches[3] * 60 + $matches[4]; }
function postProcess($fileName) { $tmpFile = tempnam($this->tmpDir, 'loc_'); log_scriptLog('* removing diacritics'); $s = file_get_contents($fileName); $s = StringUtil::unicodeToLatin($s); file_put_contents($tmpFile, $s); log_scriptLog('* removing duplicates and sorting'); OS::executeAndAssert("sort -u {$tmpFile} -o {$fileName}"); unlink($tmpFile); }
function wgetAndGunzip($url) { $tmpFile = tempnam(Config::get('global.tempDir'), 'xmldump_'); OS::executeAndAssert("wget -q -O {$tmpFile}.gz {$url}"); OS::executeAndAssert("gunzip -f {$tmpFile}.gz"); return $tmpFile; }
} // Assert that the last version is the new current version. $currentLv = $lvs[count($lvs) - 1]; print "Asserting that version {$currentLv->name} does not exist.\n"; $dbName = $locDbPrefix . $currentLv->getDbName(); if (databaseExists($dbName)) { die("ERROR: Database {$dbName} for version {$currentLv->name} " . "should not exist.\n"); } if ($currentLv->freezeTimestamp) { die("ERROR: Version {$currentLv->name} should not have a freeze date\n"); } // Now create a database for $lvToFreeze and copy the relevant tables there. $dbName = $locDbPrefix . $lvToFreeze->getDbName(); print "Creating database {$dbName}\n"; db_execute("create database {$dbName}"); $fileName = tempnam(Config::get('global.tempDir'), 'freeze_'); print "Dumping tables to {$fileName}\n"; $tablesToDump = "ConstraintMap Inflection Lexem ModelDescription ModelType Model ParticipleModel Transform InflectedForm"; $parts = db_splitDsn(); $mysql = sprintf("mysqldump -h %s -u %s --password='******' %s %s > %s", $parts['host'], $parts['user'], $parts['password'], $parts['database'], $tablesToDump, $fileName); OS::executeAndAssert($mysql); print "Importing {$fileName} to {$dbName}\n"; $import = sprintf("mysql -h %s -u %s --password='******' %s < %s", $parts['host'], $parts['user'], $parts['password'], $dbName, $fileName); OS::executeAndAssert($import); print "Success!\n"; /****************************************************************************/ function databaseExists($dbName) { $r = ORM::for_table('Definition')->raw_query("show databases like '{$dbName}'")->find_one(); return $r !== false; }
<?php require_once "../phplib/util.php"; ini_set('max_execution_time', '3600'); ini_set("memory_limit", "128000000"); $query = "select I.formNoAccent from InflectedForm I, Lexem L, Model M, ModelDescription MD, ModelType MT " . "where I.lexemId = L.id and L.modelType = MT.code and MT.canonical = M.modelType and L.modelNumber = M.number and M.id = MD.modelId " . "and MD.inflectionId = I.inflectionId and MD.variant = I.variant and MD.applOrder = 0 and L.isLoc and MD.isLoc " . "and char_length(I.formNoAccent) between 2 and 15"; $dbResult = db_execute($query); $fileName = tempnam('/tmp', 'unique_'); $file = fopen($fileName, 'w'); foreach ($dbResult as $dbRow) { fwrite($file, "{$dbRow[0]}\r\n"); } fclose($file); $s = file_get_contents($fileName); $s = StringUtil::unicodeToLatin($s); $s = strtoupper($s); $file = fopen($fileName, 'w'); fwrite($file, $s); fclose($file); $fileName2 = tempnam('/tmp', 'unique_'); OS::executeAndAssert("sort -u {$fileName} -o {$fileName2}"); header('Content-type: text/plain'); print file_get_contents($fileName2); OS::executeAndAssert("rm -f {$fileName} {$fileName2}");
} } else { // print "Not found: $word\n"; } } } if (++$defsSeen % 10000 == 0) { $runTime = DebugInfo::getRunningTimeInMillis() / 1000; $speed = round($defsSeen / $runTime); log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed ({$speed} defs/sec). " . "Word map has " . count($ifMap) . " entries. " . "Memory used: " . round(memory_get_usage() / 1048576, 1) . " MB."); } } fclose($handle); log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed."); log_scriptLog("Index size: {$indexSize} entries."); OS::executeAndAssert("chmod 666 {$fileName}"); log_scriptLog("Importing file {$fileName} into table FullTextIndex"); db_executeFromOS("load data local infile '{$fileName}' into table FullTextIndex"); util_deleteFile($fileName); if (!Lock::release(LOCK_FULL_TEXT_INDEX)) { log_scriptLog('WARNING: could not release lock!'); } log_scriptLog('rebuildFullTextIndex.php completed successfully ' . '(against all odds)'); /***************************************************************************/ function extractWords($text) { $alphabet = 'abcdefghijklmnopqrstuvwxyzăâîșț'; $text = mb_strtolower($text); $text = AdminStringUtil::removeAccents($text); $result = array(); $currentWord = '';
<?php require_once __DIR__ . '/../phplib/util.php'; log_scriptLog('Running rebuildFirefoxSpellChecker.php.'); $tmpDir = tempnam(Config::get('global.tempDir'), 'xpi_'); log_scriptLog('Setting up directories'); chdir(util_getRootPath()); OS::executeAndAssert("rm {$tmpDir}"); OS::executeAndAssert("mkdir {$tmpDir}"); OS::executeAndAssert("mkdir {$tmpDir}/chrome"); OS::executeAndAssert("mkdir {$tmpDir}/dictionaries"); OS::executeAndAssert("echo 'SET UTF-8' > {$tmpDir}/dictionaries/ro-dex.aff"); OS::executeAndAssert("cp docs/install.rdf {$tmpDir}/"); $mysqlFile = tempnam(Config::get('global.tempDir'), 'mysql_'); unlink($mysqlFile); $query = "select distinct formNoAccent from InflectedForm where formNoAccent rlike '^[a-zăâîșț]+\$' into outfile '{$mysqlFile}'"; log_scriptLog("Running mysql query: [{$query}]"); db_execute($query); log_scriptLog("Prepending line count"); OS::executeAndAssert("wc -l {$mysqlFile} | cut -d ' ' -f 1 > {$tmpDir}/dictionaries/ro-dex.dic"); OS::executeAndAssert("cat {$mysqlFile} >> {$tmpDir}/dictionaries/ro-dex.dic"); log_scriptLog("Zipping"); OS::executeAndAssert("cd {$tmpDir} && zip -r dex-ff.xpi *"); $f = new FtpUtil(); $f->staticServerPut("{$tmpDir}/dex-ff.xpi", '/download/dex-ff.xpi'); OS::executeAndAssert("rm -rf {$tmpDir}"); log_scriptLog('rebuildFirefoxSpellChecker.php completed successfully (against all odds)'); // Note -- this leaves behind the temporary MySQL file created by "... into outfile...". // The file is owned by mysql so we cannot delete it.
function dumpLdmDiff($oldFileName, $newFileName, $diffFileName) { log_scriptLog('dumping lexem-definition map diff'); $tmpFile1 = tempnam('/tmp', 'ldm'); $tmpFile2 = tempnam('/tmp', 'ldm'); OS::executeAndAssert("gunzip -c {$oldFileName} > {$tmpFile1}"); OS::executeAndAssert("gunzip -c {$newFileName} > {$tmpFile2}"); $output = null; exec("diff {$tmpFile1} {$tmpFile2}", $output, $exit_code); $file = gzopen($diffFileName, 'wb9'); gzwrite($file, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); gzwrite($file, "<LexemDefinitionMap>\n"); foreach ($output as $line) { if (substr($line, 0, 2) == '< ') { gzwrite($file, preg_replace('/<Map /', '<Unmap ', substr($line, 2)) . "\n"); // Deleted line } else { if (substr($line, 0, 2) == '> ') { gzwrite($file, substr($line, 2) . "\n"); // Added line } } } gzwrite($file, "</LexemDefinitionMap>\n"); gzclose($file); unlink($tmpFile1); unlink($tmpFile2); }
case 'reduced': $keyword = 'reduse'; $listType = 'forme reduse'; break; default: FlashMessage::add('Ați introdus o listă incorectă.'); util_redirect('scrabble'); } $zipUrl = sprintf('%sdownload/scrabble/loc-dif-%s-%s-%s.zip', Config::get('static.url'), $keyword, $locVersions[0], $locVersions[1]); $zipFile = tempnam(Config::get('global.tempDir'), 'loc_') . '.zip'; $txtFile = tempnam(Config::get('global.tempDir'), 'loc_') . '.txt'; if (!@copy($zipUrl, $zipFile)) { FlashMessage::add('Ați introdus o listă incorectă.'); util_redirect('scrabble'); } OS::executeAndAssert("unzip -p {$zipFile} > {$txtFile}"); $diff = array(); foreach (file($txtFile) as $line) { $line = trim($line); if ($line[0] == '<') { $diff[] = array('del', substr($line, 2)); } else { $diff[] = array('ins', substr($line, 2)); } } @unlink($zipFile); @unlink($txtFile); SmartyWrap::assign('page_title', 'Lista Oficială de Cuvinte'); SmartyWrap::assign('suggestHiddenSearchForm', 1); SmartyWrap::assign('keyword', $keyword); SmartyWrap::assign('listType', $listType);
$merge = false; $split = false; foreach ($argv as $i => $arg) { if ($arg == "--merge") { $merge = true; } else { if ($arg == '--split') { $split = true; } else { if ($i) { OS::errorAndExit("Unknown flag: {$arg}"); } } } } if ($merge === $split) { OS::errorAndExit("Please specify exactly one of --merge or --split"); } if ($merge) { OS::executeAndAssert("convert -size {$GEOMETRY} xc:white {$SPRITE}"); foreach ($IMAGES as $i) { if ($i['bg']) { OS::executeAndAssert(sprintf("convert {$SPRITE} -fill '%s' -draw 'rectangle %s,%s %s,%s' {$SPRITE}", $i['bg'], $i['x'], $i['y'], $i['x'] + $i['width'] - 1, $i['y'] + $i['height'] - 1)); } OS::executeAndAssert("convert {$SPRITE} {$i['file']} -geometry +{$i['x']}+{$i['y']} -compose over -composite {$SPRITE}"); } OS::executeAndAssert("optipng {$SPRITE}"); print "Composed and optimized sprite in {$SPRITE}\n"; } else { print "Splitting is not implemented yet.\n"; }
$img = util_fetchUrl($book->imageUrl); if ($img !== false) { // Dump the image to a file $file = fopen($fileName, "w"); fwrite($file, $img); fclose($file); $haveFile = true; } } $alreadyResized = file_exists($thumbName); if ($haveFile && !$alreadyResized) { $imgType = isImage($fileName); if ($imgType == IMG_NORMAL) { list($width, $height, $bytes) = preg_split('/\\|/', getImageInfo($fileName)); print " {$width}x{$height}, {$bytes} bytes "; OS::executeAndAssert("convert -trim -fuzz \"3%\" -geometry 200x84 \"{$fileName}\" \"{$thumbName}\""); if ($width <= 90 && $height <= 90) { print "*small* "; } list($thumbWidth, $thumbHeight, $ignored) = preg_split('/\\|/', getImageInfo($thumbName)); $book->thumbWidth = $thumbWidth; $book->thumbHeight = $thumbHeight; $book->save(); } else { if ($imgType == IMG_NOT_JPEG) { print " Not an image "; } else { print " Corrupted image "; } } } elseif ($alreadyResized) {
OS::executeAndAssert("echo \"-- Copyright (C) 2004-{$currentYear} DEX online (http://dexonline.ro)\" > {$TMP_DIR}/{$FILENAME}"); OS::executeAndAssert("cat {$LICENSE} >> {$TMP_DIR}/{$FILENAME}"); $mysql = "{$COMMON_COMMAND} {$tablesToIgnore} >> {$TMP_DIR}/{$FILENAME}"; OS::executeAndAssert($mysql); // Dump only the schema for some tables $command = "{$COMMON_COMMAND} --no-data"; foreach ($schemaOnly as $table) { $command .= " {$table}"; } $command .= " >> {$TMP_DIR}/{$FILENAME}"; OS::executeAndAssert($command); if (!$doFullDump) { // Anonymize the User table. Handle the case for id = 0 separately, since // "insert into _User_Copy set id = 0" doesn't work (it inserts an id of 1). log_scriptLog('Anonymizing the User table'); db_execute("create table _User_Copy like User"); db_execute("insert into _User_Copy select * from User where id = 0"); db_execute("update _User_Copy set id = 0 where id = 1"); db_execute("insert into _User_Copy select * from User where id > 0"); db_execute("update _User_Copy set password = md5('1234'), email = concat(id, '@anonymous.com'), identity = null"); OS::executeAndAssert("{$COMMON_COMMAND} _User_Copy | sed 's/_User_Copy/User/g' >> {$TMP_DIR}/{$FILENAME}"); db_execute("drop table _User_Copy"); // Dump only the Definitions for which we have redistribution rights log_scriptLog('Filtering the Definition table'); OS::executeAndAssert("{$COMMON_COMMAND} Definition --lock-all-tables --where='Definition.sourceId in (select id from Source where canDistribute)' " . ">> {$TMP_DIR}/{$FILENAME}"); } OS::executeAndAssert("gzip -f {$TMP_DIR}/{$FILENAME}"); OS::executeAndAssert("rm -f {$targetDir}/{$GZ_FILENAME}"); OS::executeAndAssert("mv {$TMP_DIR}/{$GZ_FILENAME} {$targetDir}"); OS::executeAndAssert("chmod 644 {$targetDir}/{$GZ_FILENAME}"); log_scriptLog('dumpDatabase.php completed successfully (against all odds)');
foreach ($schemaOnly as $table) { $command .= " {$table}"; } $command .= " >> {$SQL_FILE}"; OS::executeAndAssert($command); if (!$doFullDump) { // Anonymize the User table. Handle the case for id = 0 separately, since // "insert into _User_Copy set id = 0" doesn't work (it inserts an id of 1). log_scriptLog('Anonymizing the User table'); db_execute("create table _User_Copy like User"); db_execute("insert into _User_Copy select * from User where id = 0"); db_execute("update _User_Copy set id = 0 where id = 1"); db_execute("insert into _User_Copy select * from User where id > 0"); db_execute("update _User_Copy set password = md5('1234'), email = concat(id, '@anonymous.com'), identity = null"); OS::executeAndAssert("{$COMMON_COMMAND} _User_Copy | sed 's/_User_Copy/User/g' >> {$SQL_FILE}"); db_execute("drop table _User_Copy"); log_scriptLog('Anonymizing the Source table'); db_execute("create table _Source_Copy like Source"); db_execute("insert into _Source_Copy select * from Source"); db_execute("update _Source_Copy set link = null"); OS::executeAndAssert("{$COMMON_COMMAND} _Source_Copy | sed 's/_Source_Copy/Source/g' >> {$SQL_FILE}"); db_execute("drop table _Source_Copy"); // Dump only the Definitions for which we have redistribution rights log_scriptLog('Filtering the Definition table'); OS::executeAndAssert("{$COMMON_COMMAND} Definition --lock-all-tables --where='Definition.sourceId in (select id from Source where canDistribute)' " . ">> {$SQL_FILE}"); } OS::executeAndAssert("gzip -f {$SQL_FILE}"); $f = new FtpUtil(); $f->staticServerPut($GZ_FILE, $remoteFile); unlink($GZ_FILE); log_scriptLog('dumpDatabase.php completed successfully (against all odds)');
define('DATABASE_TMPFILE_GZIP', DATABASE_TMPFILE . '.gz'); $doDatabaseCopy = true; $doCodeUpdate = true; for ($i = 1; $i < count($argv); $i++) { $arg = $argv[$i]; if ($arg == "-nc") { $doCodeUpdate = false; } else { if ($arg == '-nd') { $doDatabaseCopy = false; } else { OS::errorAndExit("Unknown flag: {$arg}"); } } } log_scriptLog('Running updateMirror.php with databaseCopy:' . ($doDatabaseCopy ? 'yes' : 'no') . ' codeUpdate:' . ($doCodeUpdate ? 'yes' : 'no')); if ($doDatabaseCopy) { $wget = sprintf("wget -q -O %s %s", DATABASE_TMPFILE_GZIP, DATABASE_URL); OS::executeAndAssert($wget); $gzip = sprintf("gunzip %s", DATABASE_TMPFILE_GZIP); OS::executeAndAssert($gzip); $parts = db_splitDsn(); $mysql = sprintf("mysql -h %s -u %s --password='******' %s < %s", $parts['host'], $parts['user'], $parts['password'], $parts['database'], DATABASE_TMPFILE); OS::executeAndAssert($mysql); $rm = sprintf("rm -f %s", DATABASE_TMPFILE); OS::executeAndAssert($rm); } if ($doCodeUpdate) { OS::executeAndAssert('cd ' . util_getRootPath() . '; /usr/bin/git pull --quiet'); } log_scriptLog('updateMirror.php completed successfully (against all odds)');