function db_executeSqlFile($filename)
{
$dsn = Config::get('global.database');
$parts = db_splitDsn($dsn);
$command = sprintf("cat {$filename} | mysql -u %s %s %s", $parts['user'], $parts['database'], $parts['password'] ? "-p" . $parts['password'] : '');
OS::executeAndAssert($command);
}
function closeCurrentFile()
{
global $g_numFiles;
global $g_curFileName;
global $g_curFile;
fprintf($g_curFile, "</urlset>\n");
fclose($g_curFile);
OS::executeAndAssert("gzip - < {$g_curFileName} > wwwbase/sitemap{$g_numFiles}.xml.gz");
util_deleteFile($g_curFileName);
}
public function ensureThumbnail()
{
if (!$this->image) {
return;
}
$fullImage = self::$IMAGE_DIR . "/{$this->image}";
$fullThumb = self::$THUMB_DIR . "/{$this->image}";
if (!file_exists($fullThumb) && file_exists($fullImage)) {
$oldumask = umask(0);
@mkdir(dirname($fullThumb), 0777, true);
umask($oldumask);
OS::executeAndAssert(sprintf("convert -strip -geometry %dx%d -sharpen 1x1 '%s' '%s'", self::$THUMB_SIZE, self::$THUMB_SIZE, $fullImage, $fullThumb));
}
}
function recursiveScan($path, $logFile)
{
global $IGNORED_DIRS, $EXTENSIONS, $beforeBytes, $afterBytes;
$files = scandir($path);
foreach ($files as $file) {
if (in_array($file, $IGNORED_DIRS)) {
continue;
}
$full = "{$path}/{$file}";
if (is_dir($full)) {
recursiveScan($full, $logFile);
} else {
$extension = pathinfo(strtolower($full), PATHINFO_EXTENSION);
$fullNoExt = substr($full, 0, strlen($full) - strlen($extension) - 1);
// Strip the dot as well
if (in_array($extension, $EXTENSIONS)) {
OS::executeAndAssert("convert -strip '{$full}' '" . Config::get('global.tempDir') . "/fileNoExif.{$extension}'");
OS::executeAndAssert("convert '" . Config::get('global.tempDir') . "/fileNoExif.{$extension}' '/fileNoExifPng.png'");
OS::executeAndAssert("optipng '" . Config::get('global.tempDir') . "/fileNoExifPng.png'");
$fs1 = filesize($full);
$fs2 = filesize(Config::get('global.tempDir') . "/fileNoExif.{$extension}");
$fs3 = filesize(Config::get('global.tempDir') . '/fileNoExifPng.png');
$beforeBytes += $fs1;
if ($fs3 < $fs1 && $fs3 < $fs2) {
$compression = 100.0 * (1 - $fs3 / $fs1);
$afterBytes += $fs3;
fprintf($logFile, "%s -- Strip EXIF, convert to PNG and optimize: %d/%d bytes, %.2f%% saved\n", $full, $fs3, $fs1, $compression);
unlink($full);
unlink(Config::get('global.tempDir') . "/fileNoExif.{$extension}");
rename(Config::get('global.tempDir') . '/fileNoExifPng.png', "{$fullNoExt}.png");
} else {
if ($fs2 < $fs1) {
$compression = 100.0 * (1 - $fs2 / $fs1);
$afterBytes += $fs2;
fprintf($logFile, "%s -- Strip EXIF: %d/%d bytes, %.2f%% saved\n", $full, $fs2, $fs1, $compression);
unlink($full);
rename(Config::get('global.tempDir') . "/fileNoExif.{$extension}", $full);
unlink(Config::get('global.tempDir') . '/fileNoExifPng.png');
} else {
$afterBytes += $fs1;
fprintf($logFile, "{$full} -- leave unchanged\n");
unlink(Config::get('global.tempDir') . "/fileNoExif.{$extension}");
unlink(Config::get('global.tempDir') . '/fileNoExifPng.png');
}
}
}
}
}
}
function createThumb()
{
$url = Config::get('static.url') . self::STATIC_DIR . $this->path;
$ext = pathinfo($url, PATHINFO_EXTENSION);
$localFile = "/tmp/a.{$ext}";
$localThumbFile = "/tmp/thumb.{$ext}";
$contents = file_get_contents($url);
file_put_contents($localFile, $contents);
$command = sprintf("convert -strip -geometry %sx%s -sharpen 1x1 '%s' '%s'", self::THUMB_SIZE, self::THUMB_SIZE, $localFile, $localThumbFile);
OS::executeAndAssert($command);
$f = new FtpUtil();
$f->staticServerPut($localThumbFile, self::STATIC_THUMB_DIR . $this->path);
unlink($localFile);
unlink($localThumbFile);
}
<?php
require_once "../phplib/util.php";
$PS_COMMAND = 'ps -eo user,pid,etime,args --no-headers --sort etime';
$APACHE_USER = '******';
$PHP_EXECUTABLE = '/usr/lib/cgi-bin/php5';
$TIME_LIMIT = 3600;
/*seconds */
log_scriptLog('Running killOrphanPhpProcesses.php.');
$output = OS::executeAndReturnOutput($PS_COMMAND);
foreach ($output as $line) {
$parts = preg_split('/ +/', $line, 4);
$runningTime = getRunningTime($parts[2]);
if ($parts[0] == $APACHE_USER && $runningTime > $TIME_LIMIT && $parts[3] == $PHP_EXECUTABLE) {
log_scriptLog("killing process {$parts[1]}");
OS::executeAndAssert("kill -9 {$parts[1]}");
}
}
log_scriptLog('killOrphanPhpProcesses.php done.');
/****************************************************************************/
// ps gives us the running time in [[DD-]hh:]mm:ss format.
function getRunningTime($string)
{
$matches = array();
preg_match("/^(?:(?:(\\d+)-)?(\\d+):)?(\\d+):(\\d+)\$/", $string, $matches);
return $matches[1] * 86400 + $matches[2] * 3600 + $matches[3] * 60 + $matches[4];
}
function postProcess($fileName)
{
$tmpFile = tempnam($this->tmpDir, 'loc_');
log_scriptLog('* removing diacritics');
$s = file_get_contents($fileName);
$s = StringUtil::unicodeToLatin($s);
file_put_contents($tmpFile, $s);
log_scriptLog('* removing duplicates and sorting');
OS::executeAndAssert("sort -u {$tmpFile} -o {$fileName}");
unlink($tmpFile);
}
function wgetAndGunzip($url)
{
$tmpFile = tempnam(Config::get('global.tempDir'), 'xmldump_');
OS::executeAndAssert("wget -q -O {$tmpFile}.gz {$url}");
OS::executeAndAssert("gunzip -f {$tmpFile}.gz");
return $tmpFile;
}
}
// Assert that the last version is the new current version.
$currentLv = $lvs[count($lvs) - 1];
print "Asserting that version {$currentLv->name} does not exist.\n";
$dbName = $locDbPrefix . $currentLv->getDbName();
if (databaseExists($dbName)) {
die("ERROR: Database {$dbName} for version {$currentLv->name} " . "should not exist.\n");
}
if ($currentLv->freezeTimestamp) {
die("ERROR: Version {$currentLv->name} should not have a freeze date\n");
}
// Now create a database for $lvToFreeze and copy the relevant tables there.
$dbName = $locDbPrefix . $lvToFreeze->getDbName();
print "Creating database {$dbName}\n";
db_execute("create database {$dbName}");
$fileName = tempnam(Config::get('global.tempDir'), 'freeze_');
print "Dumping tables to {$fileName}\n";
$tablesToDump = "ConstraintMap Inflection Lexem ModelDescription ModelType Model ParticipleModel Transform InflectedForm";
$parts = db_splitDsn();
$mysql = sprintf("mysqldump -h %s -u %s --password='******' %s %s > %s", $parts['host'], $parts['user'], $parts['password'], $parts['database'], $tablesToDump, $fileName);
OS::executeAndAssert($mysql);
print "Importing {$fileName} to {$dbName}\n";
$import = sprintf("mysql -h %s -u %s --password='******' %s < %s", $parts['host'], $parts['user'], $parts['password'], $dbName, $fileName);
OS::executeAndAssert($import);
print "Success!\n";
/****************************************************************************/
function databaseExists($dbName)
{
$r = ORM::for_table('Definition')->raw_query("show databases like '{$dbName}'")->find_one();
return $r !== false;
}
<?php
require_once "../phplib/util.php";
ini_set('max_execution_time', '3600');
ini_set("memory_limit", "128000000");
$query = "select I.formNoAccent from InflectedForm I, Lexem L, Model M, ModelDescription MD, ModelType MT " . "where I.lexemId = L.id and L.modelType = MT.code and MT.canonical = M.modelType and L.modelNumber = M.number and M.id = MD.modelId " . "and MD.inflectionId = I.inflectionId and MD.variant = I.variant and MD.applOrder = 0 and L.isLoc and MD.isLoc " . "and char_length(I.formNoAccent) between 2 and 15";
$dbResult = db_execute($query);
$fileName = tempnam('/tmp', 'unique_');
$file = fopen($fileName, 'w');
foreach ($dbResult as $dbRow) {
fwrite($file, "{$dbRow[0]}\r\n");
}
fclose($file);
$s = file_get_contents($fileName);
$s = StringUtil::unicodeToLatin($s);
$s = strtoupper($s);
$file = fopen($fileName, 'w');
fwrite($file, $s);
fclose($file);
$fileName2 = tempnam('/tmp', 'unique_');
OS::executeAndAssert("sort -u {$fileName} -o {$fileName2}");
header('Content-type: text/plain');
print file_get_contents($fileName2);
OS::executeAndAssert("rm -f {$fileName} {$fileName2}");
}
} else {
// print "Not found: $word\n";
}
}
}
if (++$defsSeen % 10000 == 0) {
$runTime = DebugInfo::getRunningTimeInMillis() / 1000;
$speed = round($defsSeen / $runTime);
log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed ({$speed} defs/sec). " . "Word map has " . count($ifMap) . " entries. " . "Memory used: " . round(memory_get_usage() / 1048576, 1) . " MB.");
}
}
fclose($handle);
log_scriptLog("{$defsSeen} of {$numDefs} definitions indexed.");
log_scriptLog("Index size: {$indexSize} entries.");
OS::executeAndAssert("chmod 666 {$fileName}");
log_scriptLog("Importing file {$fileName} into table FullTextIndex");
db_executeFromOS("load data local infile '{$fileName}' into table FullTextIndex");
util_deleteFile($fileName);
if (!Lock::release(LOCK_FULL_TEXT_INDEX)) {
log_scriptLog('WARNING: could not release lock!');
}
log_scriptLog('rebuildFullTextIndex.php completed successfully ' . '(against all odds)');
/***************************************************************************/
function extractWords($text)
{
$alphabet = 'abcdefghijklmnopqrstuvwxyzăâîșț';
$text = mb_strtolower($text);
$text = AdminStringUtil::removeAccents($text);
$result = array();
$currentWord = '';
<?php
require_once __DIR__ . '/../phplib/util.php';
log_scriptLog('Running rebuildFirefoxSpellChecker.php.');
$tmpDir = tempnam(Config::get('global.tempDir'), 'xpi_');
log_scriptLog('Setting up directories');
chdir(util_getRootPath());
OS::executeAndAssert("rm {$tmpDir}");
OS::executeAndAssert("mkdir {$tmpDir}");
OS::executeAndAssert("mkdir {$tmpDir}/chrome");
OS::executeAndAssert("mkdir {$tmpDir}/dictionaries");
OS::executeAndAssert("echo 'SET UTF-8' > {$tmpDir}/dictionaries/ro-dex.aff");
OS::executeAndAssert("cp docs/install.rdf {$tmpDir}/");
$mysqlFile = tempnam(Config::get('global.tempDir'), 'mysql_');
unlink($mysqlFile);
$query = "select distinct formNoAccent from InflectedForm where formNoAccent rlike '^[a-zăâîșț]+\$' into outfile '{$mysqlFile}'";
log_scriptLog("Running mysql query: [{$query}]");
db_execute($query);
log_scriptLog("Prepending line count");
OS::executeAndAssert("wc -l {$mysqlFile} | cut -d ' ' -f 1 > {$tmpDir}/dictionaries/ro-dex.dic");
OS::executeAndAssert("cat {$mysqlFile} >> {$tmpDir}/dictionaries/ro-dex.dic");
log_scriptLog("Zipping");
OS::executeAndAssert("cd {$tmpDir} && zip -r dex-ff.xpi *");
$f = new FtpUtil();
$f->staticServerPut("{$tmpDir}/dex-ff.xpi", '/download/dex-ff.xpi');
OS::executeAndAssert("rm -rf {$tmpDir}");
log_scriptLog('rebuildFirefoxSpellChecker.php completed successfully (against all odds)');
// Note -- this leaves behind the temporary MySQL file created by "... into outfile...".
// The file is owned by mysql so we cannot delete it.
function dumpLdmDiff($oldFileName, $newFileName, $diffFileName)
{
log_scriptLog('dumping lexem-definition map diff');
$tmpFile1 = tempnam('/tmp', 'ldm');
$tmpFile2 = tempnam('/tmp', 'ldm');
OS::executeAndAssert("gunzip -c {$oldFileName} > {$tmpFile1}");
OS::executeAndAssert("gunzip -c {$newFileName} > {$tmpFile2}");
$output = null;
exec("diff {$tmpFile1} {$tmpFile2}", $output, $exit_code);
$file = gzopen($diffFileName, 'wb9');
gzwrite($file, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
gzwrite($file, "<LexemDefinitionMap>\n");
foreach ($output as $line) {
if (substr($line, 0, 2) == '< ') {
gzwrite($file, preg_replace('/<Map /', '<Unmap ', substr($line, 2)) . "\n");
// Deleted line
} else {
if (substr($line, 0, 2) == '> ') {
gzwrite($file, substr($line, 2) . "\n");
// Added line
}
}
}
gzwrite($file, "</LexemDefinitionMap>\n");
gzclose($file);
unlink($tmpFile1);
unlink($tmpFile2);
}
case 'reduced':
$keyword = 'reduse';
$listType = 'forme reduse';
break;
default:
FlashMessage::add('Ați introdus o listă incorectă.');
util_redirect('scrabble');
}
$zipUrl = sprintf('%sdownload/scrabble/loc-dif-%s-%s-%s.zip', Config::get('static.url'), $keyword, $locVersions[0], $locVersions[1]);
$zipFile = tempnam(Config::get('global.tempDir'), 'loc_') . '.zip';
$txtFile = tempnam(Config::get('global.tempDir'), 'loc_') . '.txt';
if (!@copy($zipUrl, $zipFile)) {
FlashMessage::add('Ați introdus o listă incorectă.');
util_redirect('scrabble');
}
OS::executeAndAssert("unzip -p {$zipFile} > {$txtFile}");
$diff = array();
foreach (file($txtFile) as $line) {
$line = trim($line);
if ($line[0] == '<') {
$diff[] = array('del', substr($line, 2));
} else {
$diff[] = array('ins', substr($line, 2));
}
}
@unlink($zipFile);
@unlink($txtFile);
SmartyWrap::assign('page_title', 'Lista Oficială de Cuvinte');
SmartyWrap::assign('suggestHiddenSearchForm', 1);
SmartyWrap::assign('keyword', $keyword);
SmartyWrap::assign('listType', $listType);
$merge = false;
$split = false;
foreach ($argv as $i => $arg) {
if ($arg == "--merge") {
$merge = true;
} else {
if ($arg == '--split') {
$split = true;
} else {
if ($i) {
OS::errorAndExit("Unknown flag: {$arg}");
}
}
}
}
if ($merge === $split) {
OS::errorAndExit("Please specify exactly one of --merge or --split");
}
if ($merge) {
OS::executeAndAssert("convert -size {$GEOMETRY} xc:white {$SPRITE}");
foreach ($IMAGES as $i) {
if ($i['bg']) {
OS::executeAndAssert(sprintf("convert {$SPRITE} -fill '%s' -draw 'rectangle %s,%s %s,%s' {$SPRITE}", $i['bg'], $i['x'], $i['y'], $i['x'] + $i['width'] - 1, $i['y'] + $i['height'] - 1));
}
OS::executeAndAssert("convert {$SPRITE} {$i['file']} -geometry +{$i['x']}+{$i['y']} -compose over -composite {$SPRITE}");
}
OS::executeAndAssert("optipng {$SPRITE}");
print "Composed and optimized sprite in {$SPRITE}\n";
} else {
print "Splitting is not implemented yet.\n";
}
$img = util_fetchUrl($book->imageUrl);
if ($img !== false) {
// Dump the image to a file
$file = fopen($fileName, "w");
fwrite($file, $img);
fclose($file);
$haveFile = true;
}
}
$alreadyResized = file_exists($thumbName);
if ($haveFile && !$alreadyResized) {
$imgType = isImage($fileName);
if ($imgType == IMG_NORMAL) {
list($width, $height, $bytes) = preg_split('/\\|/', getImageInfo($fileName));
print " {$width}x{$height}, {$bytes} bytes ";
OS::executeAndAssert("convert -trim -fuzz \"3%\" -geometry 200x84 \"{$fileName}\" \"{$thumbName}\"");
if ($width <= 90 && $height <= 90) {
print "*small* ";
}
list($thumbWidth, $thumbHeight, $ignored) = preg_split('/\\|/', getImageInfo($thumbName));
$book->thumbWidth = $thumbWidth;
$book->thumbHeight = $thumbHeight;
$book->save();
} else {
if ($imgType == IMG_NOT_JPEG) {
print " Not an image ";
} else {
print " Corrupted image ";
}
}
} elseif ($alreadyResized) {
OS::executeAndAssert("echo \"-- Copyright (C) 2004-{$currentYear} DEX online (http://dexonline.ro)\" > {$TMP_DIR}/{$FILENAME}");
OS::executeAndAssert("cat {$LICENSE} >> {$TMP_DIR}/{$FILENAME}");
$mysql = "{$COMMON_COMMAND} {$tablesToIgnore} >> {$TMP_DIR}/{$FILENAME}";
OS::executeAndAssert($mysql);
// Dump only the schema for some tables
$command = "{$COMMON_COMMAND} --no-data";
foreach ($schemaOnly as $table) {
$command .= " {$table}";
}
$command .= " >> {$TMP_DIR}/{$FILENAME}";
OS::executeAndAssert($command);
if (!$doFullDump) {
// Anonymize the User table. Handle the case for id = 0 separately, since
// "insert into _User_Copy set id = 0" doesn't work (it inserts an id of 1).
log_scriptLog('Anonymizing the User table');
db_execute("create table _User_Copy like User");
db_execute("insert into _User_Copy select * from User where id = 0");
db_execute("update _User_Copy set id = 0 where id = 1");
db_execute("insert into _User_Copy select * from User where id > 0");
db_execute("update _User_Copy set password = md5('1234'), email = concat(id, '@anonymous.com'), identity = null");
OS::executeAndAssert("{$COMMON_COMMAND} _User_Copy | sed 's/_User_Copy/User/g' >> {$TMP_DIR}/{$FILENAME}");
db_execute("drop table _User_Copy");
// Dump only the Definitions for which we have redistribution rights
log_scriptLog('Filtering the Definition table');
OS::executeAndAssert("{$COMMON_COMMAND} Definition --lock-all-tables --where='Definition.sourceId in (select id from Source where canDistribute)' " . ">> {$TMP_DIR}/{$FILENAME}");
}
OS::executeAndAssert("gzip -f {$TMP_DIR}/{$FILENAME}");
OS::executeAndAssert("rm -f {$targetDir}/{$GZ_FILENAME}");
OS::executeAndAssert("mv {$TMP_DIR}/{$GZ_FILENAME} {$targetDir}");
OS::executeAndAssert("chmod 644 {$targetDir}/{$GZ_FILENAME}");
log_scriptLog('dumpDatabase.php completed successfully (against all odds)');
foreach ($schemaOnly as $table) {
$command .= " {$table}";
}
$command .= " >> {$SQL_FILE}";
OS::executeAndAssert($command);
if (!$doFullDump) {
// Anonymize the User table. Handle the case for id = 0 separately, since
// "insert into _User_Copy set id = 0" doesn't work (it inserts an id of 1).
log_scriptLog('Anonymizing the User table');
db_execute("create table _User_Copy like User");
db_execute("insert into _User_Copy select * from User where id = 0");
db_execute("update _User_Copy set id = 0 where id = 1");
db_execute("insert into _User_Copy select * from User where id > 0");
db_execute("update _User_Copy set password = md5('1234'), email = concat(id, '@anonymous.com'), identity = null");
OS::executeAndAssert("{$COMMON_COMMAND} _User_Copy | sed 's/_User_Copy/User/g' >> {$SQL_FILE}");
db_execute("drop table _User_Copy");
log_scriptLog('Anonymizing the Source table');
db_execute("create table _Source_Copy like Source");
db_execute("insert into _Source_Copy select * from Source");
db_execute("update _Source_Copy set link = null");
OS::executeAndAssert("{$COMMON_COMMAND} _Source_Copy | sed 's/_Source_Copy/Source/g' >> {$SQL_FILE}");
db_execute("drop table _Source_Copy");
// Dump only the Definitions for which we have redistribution rights
log_scriptLog('Filtering the Definition table');
OS::executeAndAssert("{$COMMON_COMMAND} Definition --lock-all-tables --where='Definition.sourceId in (select id from Source where canDistribute)' " . ">> {$SQL_FILE}");
}
OS::executeAndAssert("gzip -f {$SQL_FILE}");
$f = new FtpUtil();
$f->staticServerPut($GZ_FILE, $remoteFile);
unlink($GZ_FILE);
log_scriptLog('dumpDatabase.php completed successfully (against all odds)');
define('DATABASE_TMPFILE_GZIP', DATABASE_TMPFILE . '.gz');
$doDatabaseCopy = true;
$doCodeUpdate = true;
for ($i = 1; $i < count($argv); $i++) {
$arg = $argv[$i];
if ($arg == "-nc") {
$doCodeUpdate = false;
} else {
if ($arg == '-nd') {
$doDatabaseCopy = false;
} else {
OS::errorAndExit("Unknown flag: {$arg}");
}
}
}
log_scriptLog('Running updateMirror.php with databaseCopy:' . ($doDatabaseCopy ? 'yes' : 'no') . ' codeUpdate:' . ($doCodeUpdate ? 'yes' : 'no'));
if ($doDatabaseCopy) {
$wget = sprintf("wget -q -O %s %s", DATABASE_TMPFILE_GZIP, DATABASE_URL);
OS::executeAndAssert($wget);
$gzip = sprintf("gunzip %s", DATABASE_TMPFILE_GZIP);
OS::executeAndAssert($gzip);
$parts = db_splitDsn();
$mysql = sprintf("mysql -h %s -u %s --password='******' %s < %s", $parts['host'], $parts['user'], $parts['password'], $parts['database'], DATABASE_TMPFILE);
OS::executeAndAssert($mysql);
$rm = sprintf("rm -f %s", DATABASE_TMPFILE);
OS::executeAndAssert($rm);
}
if ($doCodeUpdate) {
OS::executeAndAssert('cd ' . util_getRootPath() . '; /usr/bin/git pull --quiet');
}
log_scriptLog('updateMirror.php completed successfully (against all odds)');
