/**
* Do a job from the job queue
*/
private function doJobs()
{
global $wgJobRunRate, $wgPhpCli, $IP;
if ($wgJobRunRate <= 0 || wfReadOnly()) {
return;
}
if ($wgJobRunRate < 1) {
$max = mt_getrandmax();
if (mt_rand(0, $max) > $max * $wgJobRunRate) {
return;
// the higher $wgJobRunRate, the less likely we return here
}
$n = 1;
} else {
$n = intval($wgJobRunRate);
}
if (!wfShellExecDisabled() && is_executable($wgPhpCli)) {
// Start a background process to run some of the jobs.
// This will be asynchronous on *nix though not on Windows.
wfProfileIn(__METHOD__ . '-exec');
$retVal = 1;
$cmd = wfShellWikiCmd("{$IP}/maintenance/runJobs.php", array('--maxjobs', $n));
wfShellExec("{$cmd} &", $retVal);
wfProfileOut(__METHOD__ . '-exec');
} else {
// Fallback to running the jobs here while the user waits
$group = JobQueueGroup::singleton();
do {
$job = $group->pop(JobQueueGroup::USE_CACHE);
// job from any queue
if ($job) {
$output = $job->toString() . "\n";
$t = -microtime(true);
wfProfileIn(__METHOD__ . '-' . get_class($job));
$success = $job->run();
wfProfileOut(__METHOD__ . '-' . get_class($job));
$group->ack($job);
// done
$t += microtime(true);
$t = round($t * 1000);
if ($success === false) {
$output .= "Error: " . $job->getLastError() . ", Time: {$t} ms\n";
} else {
$output .= "Success, Time: {$t} ms\n";
}
wfDebugLog('jobqueue', $output);
}
} while (--$n && $job);
}
}
/**
* Execute a shell command, with time and memory limits mirrored from the PHP
* configuration if supported.
*
* @param string|string[] $cmd If string, a properly shell-escaped command line,
* or an array of unescaped arguments, in which case each value will be escaped
* Example: [ 'convert', '-font', 'font name' ] would produce "'convert' '-font' 'font name'"
* @param null|mixed &$retval Optional, will receive the program's exit code.
* (non-zero is usually failure). If there is an error from
* read, select, or proc_open(), this will be set to -1.
* @param array $environ Optional environment variables which should be
* added to the executed command environment.
* @param array $limits Optional array with limits(filesize, memory, time, walltime)
* this overwrites the global wgMaxShell* limits.
* @param array $options Array of options:
* - duplicateStderr: Set this to true to duplicate stderr to stdout,
* including errors from limit.sh
* - profileMethod: By default this function will profile based on the calling
* method. Set this to a string for an alternative method to profile from
*
* @return string Collected stdout as a string
*/
function wfShellExec($cmd, &$retval = null, $environ = array(), $limits = array(), $options = array())
{
global $IP, $wgMaxShellMemory, $wgMaxShellFileSize, $wgMaxShellTime, $wgMaxShellWallClockTime, $wgShellCgroup;
$disabled = wfShellExecDisabled();
if ($disabled) {
$retval = 1;
return $disabled == 'safemode' ? 'Unable to run external programs in safe mode.' : 'Unable to run external programs, proc_open() is disabled.';
}
$includeStderr = isset($options['duplicateStderr']) && $options['duplicateStderr'];
$profileMethod = isset($options['profileMethod']) ? $options['profileMethod'] : wfGetCaller();
wfInitShellLocale();
$envcmd = '';
foreach ($environ as $k => $v) {
if (wfIsWindows()) {
/* Surrounding a set in quotes (method used by wfEscapeShellArg) makes the quotes themselves
* appear in the environment variable, so we must use carat escaping as documented in
* http://technet.microsoft.com/en-us/library/cc723564.aspx
* Note however that the quote isn't listed there, but is needed, and the parentheses
* are listed there but doesn't appear to need it.
*/
$envcmd .= "set {$k}=" . preg_replace('/([&|()<>^"])/', '^\\1', $v) . '&& ';
} else {
/* Assume this is a POSIX shell, thus required to accept variable assignments before the command
* http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_09_01
*/
$envcmd .= "{$k}=" . escapeshellarg($v) . ' ';
}
}
if (is_array($cmd)) {
$cmd = wfEscapeShellArg($cmd);
}
$cmd = $envcmd . $cmd;
$useLogPipe = false;
if (is_executable('/bin/bash')) {
$time = intval(isset($limits['time']) ? $limits['time'] : $wgMaxShellTime);
if (isset($limits['walltime'])) {
$wallTime = intval($limits['walltime']);
} elseif (isset($limits['time'])) {
$wallTime = $time;
} else {
$wallTime = intval($wgMaxShellWallClockTime);
}
$mem = intval(isset($limits['memory']) ? $limits['memory'] : $wgMaxShellMemory);
$filesize = intval(isset($limits['filesize']) ? $limits['filesize'] : $wgMaxShellFileSize);
if ($time > 0 || $mem > 0 || $filesize > 0 || $wallTime > 0) {
$cmd = '/bin/bash ' . escapeshellarg("{$IP}/includes/limit.sh") . ' ' . escapeshellarg($cmd) . ' ' . escapeshellarg("MW_INCLUDE_STDERR=" . ($includeStderr ? '1' : '') . ';' . "MW_CPU_LIMIT={$time}; " . 'MW_CGROUP=' . escapeshellarg($wgShellCgroup) . '; ' . "MW_MEM_LIMIT={$mem}; " . "MW_FILE_SIZE_LIMIT={$filesize}; " . "MW_WALL_CLOCK_LIMIT={$wallTime}; " . "MW_USE_LOG_PIPE=yes");
$useLogPipe = true;
} elseif ($includeStderr) {
$cmd .= ' 2>&1';
}
} elseif ($includeStderr) {
$cmd .= ' 2>&1';
}
wfDebug("wfShellExec: {$cmd}\n");
$desc = array(0 => array('file', 'php://stdin', 'r'), 1 => array('pipe', 'w'), 2 => array('file', 'php://stderr', 'w'));
if ($useLogPipe) {
$desc[3] = array('pipe', 'w');
}
$pipes = null;
$scoped = Profiler::instance()->scopedProfileIn(__FUNCTION__ . '-' . $profileMethod);
$proc = proc_open($cmd, $desc, $pipes);
if (!$proc) {
wfDebugLog('exec', "proc_open() failed: {$cmd}");
$retval = -1;
return '';
}
$outBuffer = $logBuffer = '';
$emptyArray = array();
$status = false;
$logMsg = false;
// According to the documentation, it is possible for stream_select()
// to fail due to EINTR. I haven't managed to induce this in testing
// despite sending various signals. If it did happen, the error
// message would take the form:
//
// stream_select(): unable to select [4]: Interrupted system call (max_fd=5)
//
// where [4] is the value of the macro EINTR and "Interrupted system
// call" is string which according to the Linux manual is "possibly"
// localised according to LC_MESSAGES.
$eintr = defined('SOCKET_EINTR') ? SOCKET_EINTR : 4;
$eintrMessage = "stream_select(): unable to select [{$eintr}]";
// Build a table mapping resource IDs to pipe FDs to work around a
// PHP 5.3 issue in which stream_select() does not preserve array keys
// <https://bugs.php.net/bug.php?id=53427>.
$fds = array();
foreach ($pipes as $fd => $pipe) {
$fds[(int) $pipe] = $fd;
}
$running = true;
$timeout = null;
$numReadyPipes = 0;
while ($running === true || $numReadyPipes !== 0) {
if ($running) {
$status = proc_get_status($proc);
// If the process has terminated, switch to nonblocking selects
// for getting any data still waiting to be read.
if (!$status['running']) {
$running = false;
$timeout = 0;
}
}
$readyPipes = $pipes;
// Clear last error
// @codingStandardsIgnoreStart Generic.PHP.NoSilencedErrors.Discouraged
@trigger_error('');
$numReadyPipes = @stream_select($readyPipes, $emptyArray, $emptyArray, $timeout);
if ($numReadyPipes === false) {
// @codingStandardsIgnoreEnd
$error = error_get_last();
if (strncmp($error['message'], $eintrMessage, strlen($eintrMessage)) == 0) {
continue;
} else {
trigger_error($error['message'], E_USER_WARNING);
$logMsg = $error['message'];
break;
}
}
foreach ($readyPipes as $pipe) {
$block = fread($pipe, 65536);
$fd = $fds[(int) $pipe];
if ($block === '') {
// End of file
fclose($pipes[$fd]);
unset($pipes[$fd]);
if (!$pipes) {
break 2;
}
} elseif ($block === false) {
// Read error
$logMsg = "Error reading from pipe";
break 2;
} elseif ($fd == 1) {
// From stdout
$outBuffer .= $block;
} elseif ($fd == 3) {
// From log FD
$logBuffer .= $block;
if (strpos($block, "\n") !== false) {
$lines = explode("\n", $logBuffer);
$logBuffer = array_pop($lines);
foreach ($lines as $line) {
wfDebugLog('exec', $line);
}
}
}
}
}
foreach ($pipes as $pipe) {
fclose($pipe);
}
// Use the status previously collected if possible, since proc_get_status()
// just calls waitpid() which will not return anything useful the second time.
if ($running) {
$status = proc_get_status($proc);
}
if ($logMsg !== false) {
// Read/select error
$retval = -1;
proc_close($proc);
} elseif ($status['signaled']) {
$logMsg = "Exited with signal {$status['termsig']}";
$retval = 128 + $status['termsig'];
proc_close($proc);
} else {
if ($status['running']) {
$retval = proc_close($proc);
} else {
$retval = $status['exitcode'];
proc_close($proc);
}
if ($retval == 127) {
$logMsg = "Possibly missing executable file";
} elseif ($retval >= 129 && $retval <= 192) {
$logMsg = "Probably exited with signal " . ($retval - 128);
}
}
if ($logMsg !== false) {
wfDebugLog('exec', "{$logMsg}: {$cmd}");
}
return $outBuffer;
}
/**
* Execute a shell command, with time and memory limits mirrored from the PHP
* configuration if supported.
* @param string $cmd Command line, properly escaped for shell.
* @param &$retval null|Mixed optional, will receive the program's exit code.
* (non-zero is usually failure)
* @param array $environ optional environment variables which should be
* added to the executed command environment.
* @param array $limits optional array with limits(filesize, memory, time, walltime)
* this overwrites the global wgShellMax* limits.
* @return string collected stdout as a string (trailing newlines stripped)
*/
function wfShellExec($cmd, &$retval = null, $environ = array(), $limits = array())
{
global $IP, $wgMaxShellMemory, $wgMaxShellFileSize, $wgMaxShellTime, $wgMaxShellWallClockTime, $wgShellCgroup;
$disabled = wfShellExecDisabled();
if ($disabled) {
$retval = 1;
return $disabled == 'safemode' ? 'Unable to run external programs in safe mode.' : 'Unable to run external programs, passthru() is disabled.';
}
wfInitShellLocale();
$envcmd = '';
foreach ($environ as $k => $v) {
if (wfIsWindows()) {
/* Surrounding a set in quotes (method used by wfEscapeShellArg) makes the quotes themselves
* appear in the environment variable, so we must use carat escaping as documented in
* http://technet.microsoft.com/en-us/library/cc723564.aspx
* Note however that the quote isn't listed there, but is needed, and the parentheses
* are listed there but doesn't appear to need it.
*/
$envcmd .= "set {$k}=" . preg_replace('/([&|()<>^"])/', '^\\1', $v) . '&& ';
} else {
/* Assume this is a POSIX shell, thus required to accept variable assignments before the command
* http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_09_01
*/
$envcmd .= "{$k}=" . escapeshellarg($v) . ' ';
}
}
$cmd = $envcmd . $cmd;
if (php_uname('s') == 'Linux') {
$time = intval(isset($limits['time']) ? $limits['time'] : $wgMaxShellTime);
if (isset($limits['walltime'])) {
$wallTime = intval($limits['walltime']);
} elseif (isset($limits['time'])) {
$wallTime = $time;
} else {
$wallTime = intval($wgMaxShellWallClockTime);
}
$mem = intval(isset($limits['memory']) ? $limits['memory'] : $wgMaxShellMemory);
$filesize = intval(isset($limits['filesize']) ? $limits['filesize'] : $wgMaxShellFileSize);
if ($time > 0 || $mem > 0 || $filesize > 0 || $wallTime > 0) {
$cmd = '/bin/bash ' . escapeshellarg("{$IP}/includes/limit.sh") . ' ' . escapeshellarg($cmd) . ' ' . escapeshellarg("MW_CPU_LIMIT={$time}; " . 'MW_CGROUP=' . escapeshellarg($wgShellCgroup) . '; ' . "MW_MEM_LIMIT={$mem}; " . "MW_FILE_SIZE_LIMIT={$filesize}; " . "MW_WALL_CLOCK_LIMIT={$wallTime}");
}
}
wfDebug("wfShellExec: {$cmd}\n");
$retval = 1;
// error by default?
ob_start();
passthru($cmd, $retval);
$output = ob_get_contents();
ob_end_clean();
if ($retval == 127) {
wfDebugLog('exec', "Possibly missing executable file: {$cmd}\n");
}
return $output;
}
/**
* Highlight a code-block using a particular lexer.
*
* @param string $code Code to highlight.
* @param string|null $lang Language name, or null to use plain markup.
* @param array $args Associative array of additional arguments.
* If it contains a 'line' key, the output will include line numbers.
* If it includes a 'highlight' key, the value will be parsed as a
* comma-separated list of lines and line-ranges to highlight.
* If it contains a 'start' key, the value will be used as the line at which to
* start highlighting.
* If it contains a 'inline' key, the output will not be wrapped in `<div><pre/></div>`.
* @return Status Status object, with HTML representing the highlighted
* code as its value.
*/
protected static function highlight($code, $lang = null, $args = array())
{
global $wgPygmentizePath;
$status = new Status();
$lexer = self::getLexer($lang);
if ($lexer === null && $lang !== null) {
$status->warning('syntaxhighlight-error-unknown-language', $lang);
}
$length = strlen($code);
if (strlen($code) > self::HIGHLIGHT_MAX_BYTES) {
$status->warning('syntaxhighlight-error-exceeds-size-limit', $length, self::HIGHLIGHT_MAX_BYTES);
$lexer = null;
}
if (wfShellExecDisabled() !== false) {
$status->warning('syntaxhighlight-error-pygments-invocation-failure');
wfWarn('MediaWiki determined that it cannot invoke Pygments. ' . 'As a result, SyntaxHighlight_GeSHi will not perform any syntax highlighting. ' . 'See the debug log for details: ' . 'https://www.mediawiki.org/wiki/Manual:$wgDebugLogFile');
$lexer = null;
}
$inline = isset($args['inline']);
if ($lexer === null) {
if ($inline) {
$status->value = htmlspecialchars(trim($code), ENT_NOQUOTES);
} else {
$pre = Html::element('pre', array(), $code);
$status->value = Html::rawElement('div', array('class' => self::HIGHLIGHT_CSS_CLASS), $pre);
}
return $status;
}
$options = array('cssclass' => self::HIGHLIGHT_CSS_CLASS, 'encoding' => 'utf-8');
// Line numbers
if (isset($args['line'])) {
$options['linenos'] = 'inline';
}
if ($lexer === 'php' && strpos($code, '<?php') === false) {
$options['startinline'] = 1;
}
// Highlight specific lines
if (isset($args['highlight'])) {
$lines = self::parseHighlightLines($args['highlight']);
if (count($lines)) {
$options['hl_lines'] = implode(' ', $lines);
}
}
// Starting line number
if (isset($args['start'])) {
$options['linenostart'] = $args['start'];
}
if ($inline) {
$options['nowrap'] = 1;
}
$cache = wfGetMainCache();
$cacheKey = self::makeCacheKey($code, $lexer, $options);
$output = $cache->get($cacheKey);
if ($output === false) {
$optionPairs = array();
foreach ($options as $k => $v) {
$optionPairs[] = "{$k}={$v}";
}
$builder = new ProcessBuilder();
$builder->setPrefix($wgPygmentizePath);
$process = $builder->add('-l')->add($lexer)->add('-f')->add('html')->add('-O')->add(implode(',', $optionPairs))->getProcess();
$process->setInput($code);
$process->run();
if (!$process->isSuccessful()) {
$status->warning('syntaxhighlight-error-pygments-invocation-failure');
wfWarn('Failed to invoke Pygments: ' . $process->getErrorOutput());
$status->value = self::highlight($code, null, $args)->getValue();
return $status;
}
$output = $process->getOutput();
$cache->set($cacheKey, $output);
}
if ($inline) {
$output = trim($output);
}
$status->value = $output;
return $status;
}
/**
* Do a job from the job queue
*/
private function doJobs() {
global $wgJobRunRate, $wgPhpCli, $IP;
if ( $wgJobRunRate <= 0 || wfReadOnly() ) {
return;
}
if ( $wgJobRunRate < 1 ) {
$max = mt_getrandmax();
if ( mt_rand( 0, $max ) > $max * $wgJobRunRate ) {
return; // the higher $wgJobRunRate, the less likely we return here
}
$n = 1;
} else {
$n = intval( $wgJobRunRate );
}
if ( !wfShellExecDisabled() && is_executable( $wgPhpCli ) ) {
// Start a background process to run some of the jobs
wfProfileIn( __METHOD__ . '-exec' );
$retVal = 1;
$cmd = wfShellWikiCmd( "$IP/maintenance/runJobs.php", array( '--maxjobs', $n ) );
$cmd .= " >" . wfGetNull() . " 2>&1"; // don't hang PHP on pipes
if ( wfIsWindows() ) {
// Using START makes this async and also works around a bug where using
// wfShellExec() with a quoted script name causes a filename syntax error.
$cmd = "START /B \"bg\" $cmd";
} else {
$cmd = "$cmd &";
}
wfShellExec( $cmd, $retVal );
wfProfileOut( __METHOD__ . '-exec' );
} else {
try {
// Fallback to running the jobs here while the user waits
$group = JobQueueGroup::singleton();
do {
$job = $group->pop( JobQueueGroup::USE_CACHE ); // job from any queue
if ( $job ) {
$output = $job->toString() . "\n";
$t = - microtime( true );
wfProfileIn( __METHOD__ . '-' . get_class( $job ) );
$success = $job->run();
wfProfileOut( __METHOD__ . '-' . get_class( $job ) );
$group->ack( $job ); // done
$t += microtime( true );
$t = round( $t * 1000 );
if ( $success === false ) {
$output .= "Error: " . $job->getLastError() . ", Time: $t ms\n";
} else {
$output .= "Success, Time: $t ms\n";
}
wfDebugLog( 'jobqueue', $output );
}
} while ( --$n && $job );
} catch ( MWException $e ) {
// We don't want exceptions thrown during job execution to
// be reported to the user since the output is already sent.
// Instead we just log them.
MWExceptionHandler::logException( $e );
}
}
}