Ejemplo n.º 1
0
 /**
  * Do a job from the job queue
  */
 private function doJobs()
 {
     global $wgJobRunRate, $wgPhpCli, $IP;
     if ($wgJobRunRate <= 0 || wfReadOnly()) {
         return;
     }
     if ($wgJobRunRate < 1) {
         $max = mt_getrandmax();
         if (mt_rand(0, $max) > $max * $wgJobRunRate) {
             return;
             // the higher $wgJobRunRate, the less likely we return here
         }
         $n = 1;
     } else {
         $n = intval($wgJobRunRate);
     }
     if (!wfShellExecDisabled() && is_executable($wgPhpCli)) {
         // Start a background process to run some of the jobs.
         // This will be asynchronous on *nix though not on Windows.
         wfProfileIn(__METHOD__ . '-exec');
         $retVal = 1;
         $cmd = wfShellWikiCmd("{$IP}/maintenance/runJobs.php", array('--maxjobs', $n));
         wfShellExec("{$cmd} &", $retVal);
         wfProfileOut(__METHOD__ . '-exec');
     } else {
         // Fallback to running the jobs here while the user waits
         $group = JobQueueGroup::singleton();
         do {
             $job = $group->pop(JobQueueGroup::USE_CACHE);
             // job from any queue
             if ($job) {
                 $output = $job->toString() . "\n";
                 $t = -microtime(true);
                 wfProfileIn(__METHOD__ . '-' . get_class($job));
                 $success = $job->run();
                 wfProfileOut(__METHOD__ . '-' . get_class($job));
                 $group->ack($job);
                 // done
                 $t += microtime(true);
                 $t = round($t * 1000);
                 if ($success === false) {
                     $output .= "Error: " . $job->getLastError() . ", Time: {$t} ms\n";
                 } else {
                     $output .= "Success, Time: {$t} ms\n";
                 }
                 wfDebugLog('jobqueue', $output);
             }
         } while (--$n && $job);
     }
 }
Ejemplo n.º 2
0
/**
 * Execute a shell command, with time and memory limits mirrored from the PHP
 * configuration if supported.
 *
 * @param string|string[] $cmd If string, a properly shell-escaped command line,
 *   or an array of unescaped arguments, in which case each value will be escaped
 *   Example:   [ 'convert', '-font', 'font name' ] would produce "'convert' '-font' 'font name'"
 * @param null|mixed &$retval Optional, will receive the program's exit code.
 *   (non-zero is usually failure). If there is an error from
 *   read, select, or proc_open(), this will be set to -1.
 * @param array $environ Optional environment variables which should be
 *   added to the executed command environment.
 * @param array $limits Optional array with limits(filesize, memory, time, walltime)
 *   this overwrites the global wgMaxShell* limits.
 * @param array $options Array of options:
 *   - duplicateStderr: Set this to true to duplicate stderr to stdout,
 *     including errors from limit.sh
 *   - profileMethod: By default this function will profile based on the calling
 *     method. Set this to a string for an alternative method to profile from
 *
 * @return string Collected stdout as a string
 */
function wfShellExec($cmd, &$retval = null, $environ = array(), $limits = array(), $options = array())
{
    global $IP, $wgMaxShellMemory, $wgMaxShellFileSize, $wgMaxShellTime, $wgMaxShellWallClockTime, $wgShellCgroup;
    $disabled = wfShellExecDisabled();
    if ($disabled) {
        $retval = 1;
        return $disabled == 'safemode' ? 'Unable to run external programs in safe mode.' : 'Unable to run external programs, proc_open() is disabled.';
    }
    $includeStderr = isset($options['duplicateStderr']) && $options['duplicateStderr'];
    $profileMethod = isset($options['profileMethod']) ? $options['profileMethod'] : wfGetCaller();
    wfInitShellLocale();
    $envcmd = '';
    foreach ($environ as $k => $v) {
        if (wfIsWindows()) {
            /* Surrounding a set in quotes (method used by wfEscapeShellArg) makes the quotes themselves
             * appear in the environment variable, so we must use carat escaping as documented in
             * http://technet.microsoft.com/en-us/library/cc723564.aspx
             * Note however that the quote isn't listed there, but is needed, and the parentheses
             * are listed there but doesn't appear to need it.
             */
            $envcmd .= "set {$k}=" . preg_replace('/([&|()<>^"])/', '^\\1', $v) . '&& ';
        } else {
            /* Assume this is a POSIX shell, thus required to accept variable assignments before the command
             * http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_09_01
             */
            $envcmd .= "{$k}=" . escapeshellarg($v) . ' ';
        }
    }
    if (is_array($cmd)) {
        $cmd = wfEscapeShellArg($cmd);
    }
    $cmd = $envcmd . $cmd;
    $useLogPipe = false;
    if (is_executable('/bin/bash')) {
        $time = intval(isset($limits['time']) ? $limits['time'] : $wgMaxShellTime);
        if (isset($limits['walltime'])) {
            $wallTime = intval($limits['walltime']);
        } elseif (isset($limits['time'])) {
            $wallTime = $time;
        } else {
            $wallTime = intval($wgMaxShellWallClockTime);
        }
        $mem = intval(isset($limits['memory']) ? $limits['memory'] : $wgMaxShellMemory);
        $filesize = intval(isset($limits['filesize']) ? $limits['filesize'] : $wgMaxShellFileSize);
        if ($time > 0 || $mem > 0 || $filesize > 0 || $wallTime > 0) {
            $cmd = '/bin/bash ' . escapeshellarg("{$IP}/includes/limit.sh") . ' ' . escapeshellarg($cmd) . ' ' . escapeshellarg("MW_INCLUDE_STDERR=" . ($includeStderr ? '1' : '') . ';' . "MW_CPU_LIMIT={$time}; " . 'MW_CGROUP=' . escapeshellarg($wgShellCgroup) . '; ' . "MW_MEM_LIMIT={$mem}; " . "MW_FILE_SIZE_LIMIT={$filesize}; " . "MW_WALL_CLOCK_LIMIT={$wallTime}; " . "MW_USE_LOG_PIPE=yes");
            $useLogPipe = true;
        } elseif ($includeStderr) {
            $cmd .= ' 2>&1';
        }
    } elseif ($includeStderr) {
        $cmd .= ' 2>&1';
    }
    wfDebug("wfShellExec: {$cmd}\n");
    $desc = array(0 => array('file', 'php://stdin', 'r'), 1 => array('pipe', 'w'), 2 => array('file', 'php://stderr', 'w'));
    if ($useLogPipe) {
        $desc[3] = array('pipe', 'w');
    }
    $pipes = null;
    $scoped = Profiler::instance()->scopedProfileIn(__FUNCTION__ . '-' . $profileMethod);
    $proc = proc_open($cmd, $desc, $pipes);
    if (!$proc) {
        wfDebugLog('exec', "proc_open() failed: {$cmd}");
        $retval = -1;
        return '';
    }
    $outBuffer = $logBuffer = '';
    $emptyArray = array();
    $status = false;
    $logMsg = false;
    // According to the documentation, it is possible for stream_select()
    // to fail due to EINTR. I haven't managed to induce this in testing
    // despite sending various signals. If it did happen, the error
    // message would take the form:
    //
    // stream_select(): unable to select [4]: Interrupted system call (max_fd=5)
    //
    // where [4] is the value of the macro EINTR and "Interrupted system
    // call" is string which according to the Linux manual is "possibly"
    // localised according to LC_MESSAGES.
    $eintr = defined('SOCKET_EINTR') ? SOCKET_EINTR : 4;
    $eintrMessage = "stream_select(): unable to select [{$eintr}]";
    // Build a table mapping resource IDs to pipe FDs to work around a
    // PHP 5.3 issue in which stream_select() does not preserve array keys
    // <https://bugs.php.net/bug.php?id=53427>.
    $fds = array();
    foreach ($pipes as $fd => $pipe) {
        $fds[(int) $pipe] = $fd;
    }
    $running = true;
    $timeout = null;
    $numReadyPipes = 0;
    while ($running === true || $numReadyPipes !== 0) {
        if ($running) {
            $status = proc_get_status($proc);
            // If the process has terminated, switch to nonblocking selects
            // for getting any data still waiting to be read.
            if (!$status['running']) {
                $running = false;
                $timeout = 0;
            }
        }
        $readyPipes = $pipes;
        // Clear last error
        // @codingStandardsIgnoreStart Generic.PHP.NoSilencedErrors.Discouraged
        @trigger_error('');
        $numReadyPipes = @stream_select($readyPipes, $emptyArray, $emptyArray, $timeout);
        if ($numReadyPipes === false) {
            // @codingStandardsIgnoreEnd
            $error = error_get_last();
            if (strncmp($error['message'], $eintrMessage, strlen($eintrMessage)) == 0) {
                continue;
            } else {
                trigger_error($error['message'], E_USER_WARNING);
                $logMsg = $error['message'];
                break;
            }
        }
        foreach ($readyPipes as $pipe) {
            $block = fread($pipe, 65536);
            $fd = $fds[(int) $pipe];
            if ($block === '') {
                // End of file
                fclose($pipes[$fd]);
                unset($pipes[$fd]);
                if (!$pipes) {
                    break 2;
                }
            } elseif ($block === false) {
                // Read error
                $logMsg = "Error reading from pipe";
                break 2;
            } elseif ($fd == 1) {
                // From stdout
                $outBuffer .= $block;
            } elseif ($fd == 3) {
                // From log FD
                $logBuffer .= $block;
                if (strpos($block, "\n") !== false) {
                    $lines = explode("\n", $logBuffer);
                    $logBuffer = array_pop($lines);
                    foreach ($lines as $line) {
                        wfDebugLog('exec', $line);
                    }
                }
            }
        }
    }
    foreach ($pipes as $pipe) {
        fclose($pipe);
    }
    // Use the status previously collected if possible, since proc_get_status()
    // just calls waitpid() which will not return anything useful the second time.
    if ($running) {
        $status = proc_get_status($proc);
    }
    if ($logMsg !== false) {
        // Read/select error
        $retval = -1;
        proc_close($proc);
    } elseif ($status['signaled']) {
        $logMsg = "Exited with signal {$status['termsig']}";
        $retval = 128 + $status['termsig'];
        proc_close($proc);
    } else {
        if ($status['running']) {
            $retval = proc_close($proc);
        } else {
            $retval = $status['exitcode'];
            proc_close($proc);
        }
        if ($retval == 127) {
            $logMsg = "Possibly missing executable file";
        } elseif ($retval >= 129 && $retval <= 192) {
            $logMsg = "Probably exited with signal " . ($retval - 128);
        }
    }
    if ($logMsg !== false) {
        wfDebugLog('exec', "{$logMsg}: {$cmd}");
    }
    return $outBuffer;
}
Ejemplo n.º 3
0
/**
 * Execute a shell command, with time and memory limits mirrored from the PHP
 * configuration if supported.
 * @param string $cmd Command line, properly escaped for shell.
 * @param &$retval null|Mixed optional, will receive the program's exit code.
 *                 (non-zero is usually failure)
 * @param array $environ optional environment variables which should be
 *                 added to the executed command environment.
 * @param array $limits optional array with limits(filesize, memory, time, walltime)
 *                 this overwrites the global wgShellMax* limits.
 * @return string collected stdout as a string (trailing newlines stripped)
 */
function wfShellExec($cmd, &$retval = null, $environ = array(), $limits = array())
{
    global $IP, $wgMaxShellMemory, $wgMaxShellFileSize, $wgMaxShellTime, $wgMaxShellWallClockTime, $wgShellCgroup;
    $disabled = wfShellExecDisabled();
    if ($disabled) {
        $retval = 1;
        return $disabled == 'safemode' ? 'Unable to run external programs in safe mode.' : 'Unable to run external programs, passthru() is disabled.';
    }
    wfInitShellLocale();
    $envcmd = '';
    foreach ($environ as $k => $v) {
        if (wfIsWindows()) {
            /* Surrounding a set in quotes (method used by wfEscapeShellArg) makes the quotes themselves
             * appear in the environment variable, so we must use carat escaping as documented in
             * http://technet.microsoft.com/en-us/library/cc723564.aspx
             * Note however that the quote isn't listed there, but is needed, and the parentheses
             * are listed there but doesn't appear to need it.
             */
            $envcmd .= "set {$k}=" . preg_replace('/([&|()<>^"])/', '^\\1', $v) . '&& ';
        } else {
            /* Assume this is a POSIX shell, thus required to accept variable assignments before the command
             * http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_09_01
             */
            $envcmd .= "{$k}=" . escapeshellarg($v) . ' ';
        }
    }
    $cmd = $envcmd . $cmd;
    if (php_uname('s') == 'Linux') {
        $time = intval(isset($limits['time']) ? $limits['time'] : $wgMaxShellTime);
        if (isset($limits['walltime'])) {
            $wallTime = intval($limits['walltime']);
        } elseif (isset($limits['time'])) {
            $wallTime = $time;
        } else {
            $wallTime = intval($wgMaxShellWallClockTime);
        }
        $mem = intval(isset($limits['memory']) ? $limits['memory'] : $wgMaxShellMemory);
        $filesize = intval(isset($limits['filesize']) ? $limits['filesize'] : $wgMaxShellFileSize);
        if ($time > 0 || $mem > 0 || $filesize > 0 || $wallTime > 0) {
            $cmd = '/bin/bash ' . escapeshellarg("{$IP}/includes/limit.sh") . ' ' . escapeshellarg($cmd) . ' ' . escapeshellarg("MW_CPU_LIMIT={$time}; " . 'MW_CGROUP=' . escapeshellarg($wgShellCgroup) . '; ' . "MW_MEM_LIMIT={$mem}; " . "MW_FILE_SIZE_LIMIT={$filesize}; " . "MW_WALL_CLOCK_LIMIT={$wallTime}");
        }
    }
    wfDebug("wfShellExec: {$cmd}\n");
    $retval = 1;
    // error by default?
    ob_start();
    passthru($cmd, $retval);
    $output = ob_get_contents();
    ob_end_clean();
    if ($retval == 127) {
        wfDebugLog('exec', "Possibly missing executable file: {$cmd}\n");
    }
    return $output;
}
 /**
  * Highlight a code-block using a particular lexer.
  *
  * @param string $code Code to highlight.
  * @param string|null $lang Language name, or null to use plain markup.
  * @param array $args Associative array of additional arguments.
  *  If it contains a 'line' key, the output will include line numbers.
  *  If it includes a 'highlight' key, the value will be parsed as a
  *  comma-separated list of lines and line-ranges to highlight.
  *  If it contains a 'start' key, the value will be used as the line at which to
  *  start highlighting.
  *  If it contains a 'inline' key, the output will not be wrapped in `<div><pre/></div>`.
  * @return Status Status object, with HTML representing the highlighted
  *  code as its value.
  */
 protected static function highlight($code, $lang = null, $args = array())
 {
     global $wgPygmentizePath;
     $status = new Status();
     $lexer = self::getLexer($lang);
     if ($lexer === null && $lang !== null) {
         $status->warning('syntaxhighlight-error-unknown-language', $lang);
     }
     $length = strlen($code);
     if (strlen($code) > self::HIGHLIGHT_MAX_BYTES) {
         $status->warning('syntaxhighlight-error-exceeds-size-limit', $length, self::HIGHLIGHT_MAX_BYTES);
         $lexer = null;
     }
     if (wfShellExecDisabled() !== false) {
         $status->warning('syntaxhighlight-error-pygments-invocation-failure');
         wfWarn('MediaWiki determined that it cannot invoke Pygments. ' . 'As a result, SyntaxHighlight_GeSHi will not perform any syntax highlighting. ' . 'See the debug log for details: ' . 'https://www.mediawiki.org/wiki/Manual:$wgDebugLogFile');
         $lexer = null;
     }
     $inline = isset($args['inline']);
     if ($lexer === null) {
         if ($inline) {
             $status->value = htmlspecialchars(trim($code), ENT_NOQUOTES);
         } else {
             $pre = Html::element('pre', array(), $code);
             $status->value = Html::rawElement('div', array('class' => self::HIGHLIGHT_CSS_CLASS), $pre);
         }
         return $status;
     }
     $options = array('cssclass' => self::HIGHLIGHT_CSS_CLASS, 'encoding' => 'utf-8');
     // Line numbers
     if (isset($args['line'])) {
         $options['linenos'] = 'inline';
     }
     if ($lexer === 'php' && strpos($code, '<?php') === false) {
         $options['startinline'] = 1;
     }
     // Highlight specific lines
     if (isset($args['highlight'])) {
         $lines = self::parseHighlightLines($args['highlight']);
         if (count($lines)) {
             $options['hl_lines'] = implode(' ', $lines);
         }
     }
     // Starting line number
     if (isset($args['start'])) {
         $options['linenostart'] = $args['start'];
     }
     if ($inline) {
         $options['nowrap'] = 1;
     }
     $cache = wfGetMainCache();
     $cacheKey = self::makeCacheKey($code, $lexer, $options);
     $output = $cache->get($cacheKey);
     if ($output === false) {
         $optionPairs = array();
         foreach ($options as $k => $v) {
             $optionPairs[] = "{$k}={$v}";
         }
         $builder = new ProcessBuilder();
         $builder->setPrefix($wgPygmentizePath);
         $process = $builder->add('-l')->add($lexer)->add('-f')->add('html')->add('-O')->add(implode(',', $optionPairs))->getProcess();
         $process->setInput($code);
         $process->run();
         if (!$process->isSuccessful()) {
             $status->warning('syntaxhighlight-error-pygments-invocation-failure');
             wfWarn('Failed to invoke Pygments: ' . $process->getErrorOutput());
             $status->value = self::highlight($code, null, $args)->getValue();
             return $status;
         }
         $output = $process->getOutput();
         $cache->set($cacheKey, $output);
     }
     if ($inline) {
         $output = trim($output);
     }
     $status->value = $output;
     return $status;
 }
Ejemplo n.º 5
0
	/**
	 * Do a job from the job queue
	 */
	private function doJobs() {
		global $wgJobRunRate, $wgPhpCli, $IP;

		if ( $wgJobRunRate <= 0 || wfReadOnly() ) {
			return;
		}

		if ( $wgJobRunRate < 1 ) {
			$max = mt_getrandmax();
			if ( mt_rand( 0, $max ) > $max * $wgJobRunRate ) {
				return; // the higher $wgJobRunRate, the less likely we return here
			}
			$n = 1;
		} else {
			$n = intval( $wgJobRunRate );
		}

		if ( !wfShellExecDisabled() && is_executable( $wgPhpCli ) ) {
			// Start a background process to run some of the jobs
			wfProfileIn( __METHOD__ . '-exec' );
			$retVal = 1;
			$cmd = wfShellWikiCmd( "$IP/maintenance/runJobs.php", array( '--maxjobs', $n ) );
			$cmd .= " >" . wfGetNull() . " 2>&1"; // don't hang PHP on pipes
			if ( wfIsWindows() ) {
				// Using START makes this async and also works around a bug where using
				// wfShellExec() with a quoted script name causes a filename syntax error.
				$cmd = "START /B \"bg\" $cmd";
			} else {
				$cmd = "$cmd &";
			}
			wfShellExec( $cmd, $retVal );
			wfProfileOut( __METHOD__ . '-exec' );
		} else {
			try {
				// Fallback to running the jobs here while the user waits
				$group = JobQueueGroup::singleton();
				do {
					$job = $group->pop( JobQueueGroup::USE_CACHE ); // job from any queue
					if ( $job ) {
						$output = $job->toString() . "\n";
						$t = - microtime( true );
						wfProfileIn( __METHOD__ . '-' . get_class( $job ) );
						$success = $job->run();
						wfProfileOut( __METHOD__ . '-' . get_class( $job ) );
						$group->ack( $job ); // done
						$t += microtime( true );
						$t = round( $t * 1000 );
						if ( $success === false ) {
							$output .= "Error: " . $job->getLastError() . ", Time: $t ms\n";
						} else {
							$output .= "Success, Time: $t ms\n";
						}
						wfDebugLog( 'jobqueue', $output );
					}
				} while ( --$n && $job );
			} catch ( MWException $e ) {
				// We don't want exceptions thrown during job execution to
				// be reported to the user since the output is already sent.
				// Instead we just log them.
				MWExceptionHandler::logException( $e );
			}
		}
	}