/**
* Get the resulting notice ID from the reponse header of the API
* request.
*
* @param MWHttpRequest $response The response from the API request.
* @return int|boolean The ID of the resulting notice or
* false on failure.
*/
public function getNoticeIdFromResponse(\MWHttpRequest $response)
{
$location = $response->getResponseHeader('Location');
if (empty($location)) {
return false;
}
$path = parse_url($location, PHP_URL_PATH);
$matches = [];
if (preg_match('/^\\/notices\\/(\\d+)$/', $path, $matches)) {
return (int) $matches[1];
}
return false;
}
/**
* The general method for handling the communication with the service.
*/
public function request($resourceName, $getParams = [], $postData = [], $extraRequestOptions = [])
{
// Crash if we cannot make HTTP requests.
\Wikia\Util\Assert::true(\MWHttpRequest::canMakeRequests());
// Add client_id and client_secret to the GET data.
$getParams['client_id'] = $this->clientId;
$getParams['client_secret'] = $this->clientSecret;
// Request URI pre-processing.
$uri = "{$this->baseUri}{$resourceName}?" . http_build_query($getParams);
// Request options pre-processing.
$options = ['method' => 'GET', 'timeout' => 5, 'postData' => $postData, 'noProxy' => true, 'followRedirects' => false, 'returnInstance' => true, 'internalRequest' => true];
$options = array_merge($options, $extraRequestOptions);
/*
* MediaWiki's MWHttpRequest class heavily relies on Messaging API
* (wfMessage()) which happens to rely on the value of $wgLang.
* $wgLang is set after $wgUser. On per-request authentication with
* an access token we use MWHttpRequest before wgUser is created so
* we need $wgLang to be present. With GlobalStateWrapper we can set
* the global variable in the local, function's scope, so it is the
* same as the already existing $wgContLang.
*/
global $wgContLang;
$wrapper = new GlobalStateWrapper(['wgLang' => $wgContLang]);
// Request execution.
/** @var \MWHttpRequest $request */
$request = $wrapper->wrap(function () use($options, $uri) {
return \Http::request($options['method'], $uri, $options);
});
$this->status = $request->status;
$output = json_decode($request->getContent());
if (!$output) {
throw new ClientException('Invalid response.');
}
return $output;
}
public function getData () {
// Crazy workaround for HttpRequest not accepting user options
$req = MWHttpRequest::factory( $this->summaryDataURL, array ('method' => "GET", 'timeout' => 'default') );
$req->setHeader("http-x-license-key", $this->licenceKey);
$status = $req->execute();
$response = $req->getContent();
$data = array();
if ($response) {
// chop up xml
$xml = simplexml_load_string($response);
$data = array();
foreach ($xml->threshold_value as $node) {
$label = (string)$node['name'];
// just grab the time from the first field since we are rounding to the nearest hour anyway
if (empty($data['Time'])) {
$time = (string)$node['end_time'];
$date = strtotime($time); // raw mysql date format should parse ok
$data['Time'] = $date;
}
// we only want to use some of the fields returned from new relic
if (in_array($label, array('Errors', 'Response Time', 'Throughput')))
$data[$label] = (string)$node['metric_value'];
}
} else {
// print_pre("null response from newrelic");
}
return $data;
}
/**
* Check, if the user solved the captcha.
*
* Based on reference implementation:
* https://github.com/google/recaptcha#php
*
* @return boolean
*/
function passCaptcha()
{
global $wgRequest, $wgReCaptchaSecretKey, $wgReCaptchaSendRemoteIP;
$url = 'https://www.google.com/recaptcha/api/siteverify';
// Build data to append to request
$data = array('secret' => $wgReCaptchaSecretKey, 'response' => $wgRequest->getVal('g-recaptcha-response'));
if ($wgReCaptchaSendRemoteIP) {
$data['remoteip'] = $wgRequest->getIP();
}
$url = wfAppendQuery($url, $data);
$request = MWHttpRequest::factory($url, array('method' => 'GET'));
$status = $request->execute();
if (!$status->isOK()) {
$this->error = 'http';
$this->logStatusError($status);
return false;
}
$response = FormatJson::decode($request->getContent(), true);
if (!$response) {
$this->error = 'json';
$this->logStatusError($this->error);
return false;
}
if (isset($response['error-codes'])) {
$this->error = 'recaptcha-api';
$this->logCheckError($response['error-codes']);
return false;
}
return $response['success'];
}
function streamAppleTouch()
{
global $wgAppleTouchIcon;
wfResetOutputBuffers();
if ($wgAppleTouchIcon === false) {
# That's not very helpful, that's where we are already
header('HTTP/1.1 404 Not Found');
faviconShowError('$wgAppleTouchIcon is configured incorrectly, ' . 'it must be set to something other than false \\n');
return;
}
$req = RequestContext::getMain()->getRequest();
if ($req->getHeader('X-Favicon-Loop') !== false) {
header('HTTP/1.1 500 Internal Server Error');
faviconShowError('Proxy forwarding loop detected');
return;
}
$url = wfExpandUrl($wgAppleTouchIcon, PROTO_CANONICAL);
$client = MWHttpRequest::factory($url);
$client->setHeader('X-Favicon-Loop', '1');
$status = $client->execute();
if (!$status->isOK()) {
header('HTTP/1.1 500 Internal Server Error');
faviconShowError("Failed to fetch URL \"{$url}\"");
return;
}
$content = $client->getContent();
header('Content-Length: ' . strlen($content));
header('Content-Type: ' . $client->getResponseHeader('Content-Type'));
header('Cache-Control: public');
header('Expires: ' . gmdate('r', time() + 86400));
echo $content;
}
protected function checkContactLink($name, $url, &$countOk)
{
global $wgVersion;
$ok = false;
if (Sanitizer::validateEmail($url)) {
$ok = true;
// assume OK
} else {
$bits = wfParseUrl($url);
if ($bits && isset($bits['scheme'])) {
if ($bits['scheme'] == 'mailto') {
$ok = true;
// assume OK
} elseif (in_array($bits['scheme'], array('http', 'https'))) {
$req = MWHttpRequest::factory($url, array('method' => 'GET', 'timeout' => 8, 'sslVerifyHost' => false, 'sslVerifyCert' => false));
$req->setUserAgent("MediaWiki {$wgVersion}, CheckCongressLinks Checker");
$ok = $req->execute()->isOK();
}
}
}
if ($ok) {
++$countOk;
} else {
$this->output("Broken: [{$name}] [{$url}]\n");
}
}
protected function getInterfaceObjectFromType($type)
{
wfProfileIn(__METHOD__);
$apiUrl = $this->getApiUrl();
if (empty($this->videoId)) {
throw new EmptyResponseException($apiUrl);
}
$memcKey = wfMemcKey(static::$CACHE_KEY, $apiUrl, static::$CACHE_KEY_VERSION);
$processedResponse = F::app()->wg->memc->get($memcKey);
if (empty($processedResponse)) {
$req = MWHttpRequest::factory($apiUrl, array('noProxy' => true));
$req->setHeader('User-Agent', self::$REQUEST_USER_AGENT);
$status = $req->execute();
if ($status->isOK()) {
$response = $req->getContent();
$this->response = $response;
// Only for migration purposes
if (empty($response)) {
throw new EmptyResponseException($apiUrl);
} else {
if ($req->getStatus() == 301) {
throw new VideoNotFoundException($req->getStatus(), $this->videoId . ' Moved Permanently.', $apiUrl);
}
}
} else {
$this->checkForResponseErrors($req->status, $req->getContent(), $apiUrl);
}
$processedResponse = $this->processResponse($response, $type);
F::app()->wg->memc->set($memcKey, $processedResponse, static::$CACHE_EXPIRY);
}
wfProfileOut(__METHOD__);
return $processedResponse;
}
/**
* @group Broken
*/
public function testApiLoginGotCookie()
{
$this->markTestIncomplete("The server can't do external HTTP requests, " . "and the internal one won't give cookies");
global $wgServer, $wgScriptPath;
if (!isset($wgServer)) {
$this->markTestIncomplete('This test needs $wgServer to be set in LocalSettings.php');
}
$user = self::$users['sysop'];
$req = MWHttpRequest::factory(self::$apiUrl . "?action=login&format=xml", array("method" => "POST", "postData" => array("lgname" => $user->username, "lgpassword" => $user->password)), __METHOD__);
$req->execute();
libxml_use_internal_errors(true);
$sxe = simplexml_load_string($req->getContent());
$this->assertNotInternalType("bool", $sxe);
$this->assertThat($sxe, $this->isInstanceOf("SimpleXMLElement"));
$this->assertNotInternalType("null", $sxe->login[0]);
$a = $sxe->login[0]->attributes()->result[0];
$this->assertEquals(' result="NeedToken"', $a->asXML());
$token = (string) $sxe->login[0]->attributes()->token;
$req->setData(array("lgtoken" => $token, "lgname" => $user->username, "lgpassword" => $user->password));
$req->execute();
$cj = $req->getCookieJar();
$serverName = parse_url($wgServer, PHP_URL_HOST);
$this->assertNotEquals(false, $serverName);
$serializedCookie = $cj->serializeToHttpRequest($wgScriptPath, $serverName);
$this->assertNotEquals('', $serializedCookie);
$this->assertRegexp('/_session=[^;]*; .*UserID=[0-9]*; .*UserName='******'; .*Token=/', $serializedCookie);
}
/**
* getImage method
*
*/
public function getImage()
{
$this->wf->profileIn(__METHOD__);
if ($this->wg->User->isLoggedIn()) {
# make proper thumb path: c/central/images/thumb/....
$path = sprintf("%s/%s/images", substr($this->wg->DBname, 0, 1), $this->wg->DBname);
# take thumb request from request
$img = $this->getVal('image');
if (preg_match('/^(\\/?)thumb\\//', $img)) {
# build proper thumb url for thumbnailer
$thumb_url = sprintf("%s/%s/%s", $this->wg->ThumbnailerService, $path, $img);
# call thumbnailer
$options = array('method' => 'GET', 'timeout' => 'default', 'noProxy' => 1);
$thumb_request = MWHttpRequest::factory($thumb_url, $options);
$status = $thumb_request->execute();
$headers = $thumb_request->getResponseHeaders();
if ($status->isOK()) {
if (!empty($headers)) {
foreach ($headers as $header_name => $header_value) {
if (is_array($header_value)) {
list($value) = $header_value;
} else {
$value = $header_value;
}
header(sprintf("%s: %s", $header_name, $value));
}
}
echo $thumb_request->getContent();
} else {
$this->wf->debug("Cannot generate auth thumb");
$this->_access_forbidden('img-auth-accessdenied', 'img-auth-nofile', $img);
}
} else {
# serve original image
$filename = realpath(sprintf("%s/%s", $this->wg->UploadDirectory, $img));
$stat = @stat($filename);
if ($stat) {
$this->wf->ResetOutputBuffers();
$fileinfo = finfo_open(FILEINFO_MIME_TYPE);
$imageType = finfo_file($fileinfo, $filename);
header(sprintf("Content-Disposition: inline;filename*=utf-8'%s'%s", $this->wg->ContLanguageCode, urlencode(basename($filename))));
header(sprintf("Content-Type: %s", $imageType));
header(sprintf("Content-Length: %d" . $stat['size']));
readfile($filename);
} else {
$this->_access_forbidden('img-auth-accessdenied', 'img-auth-nopathinfo', $img);
}
}
} else {
$this->_access_forbidden('img-auth-accessdenied', 'img-auth-public', '');
}
$this->wf->profileOut(__METHOD__);
exit;
}
public function makeRequest($url, $method = 'POST')
{
$options = array('method' => $method);
if ($this->followRedirects) {
$options['followRedirects'] = true;
$this->followRedirects = false;
# Reset the flag
}
$req = MWHttpRequest::factory($url, $options);
$req->setUserAgent($this->userAgent);
$req->setCookieJar($this->cookie_jar);
return $req;
}
protected function importVideosForKeyphrase($keyword, $params = array())
{
wfProfileIn(__METHOD__);
$addlCategories = !empty($params['addlCategories']) ? $params['addlCategories'] : array();
$debug = !empty($params['debug']);
$startDate = !empty($params['startDate']) ? $params['startDate'] : '';
$endDate = !empty($params['endDate']) ? $params['endDate'] : '';
$articlesCreated = 0;
$page = 1;
do {
$numVideos = 0;
// connect to provider API
$url = $this->initFeedUrl($keyword, $startDate, $endDate, $page++);
print "Connecting to {$url}...\n";
$req = MWHttpRequest::factory($url);
$status = $req->execute();
if ($status->isOK()) {
$response = $req->getContent();
} else {
print "ERROR: problem downloading content!\n";
wfProfileOut(__METHOD__);
return 0;
}
// parse response
$videos = json_decode($response, true);
$numVideos = sizeof($videos['videos']);
print "Found {$numVideos} videos...\n";
for ($i = 0; $i < $numVideos; $i++) {
$clipData = array();
$video = $videos['videos'][$i];
$clipData['clipTitle'] = trim($video['title']);
$clipData['videoId'] = $video['guid'];
$clipData['thumbnail'] = $video['image'];
$clipData['duration'] = $video['duration'];
$clipData['published'] = $video['date'];
$clipData['category'] = $video['category_name'];
$clipData['keywords'] = trim($video['tags']);
$clipData['description'] = trim($video['description']);
$clipData['aspectRatio'] = $video['aspect_ratio'];
$msg = '';
$createParams = array('addlCategories' => $addlCategories, 'debug' => $debug);
$articlesCreated += $this->createVideo($clipData, $msg, $createParams);
if ($msg) {
print "ERROR: {$msg}\n";
}
}
} while ($numVideos == self::API_PAGE_SIZE);
wfProfileOut(__METHOD__);
return $articlesCreated;
}
protected function requestParsoid($method, $title, $params)
{
global $wgVisualEditorParsoidURL, $wgVisualEditorParsoidTimeout, $wgVisualEditorParsoidForwardCookies;
$url = $wgVisualEditorParsoidURL . '/' . urlencode($this->getApiSource()) . '/' . urlencode($title->getPrefixedDBkey());
$data = array_merge($this->getProxyConf(), array('method' => $method, 'timeout' => $wgVisualEditorParsoidTimeout));
if ($method === 'POST') {
$data['postData'] = $params;
} else {
$url = wfAppendQuery($url, $params);
}
$req = MWHttpRequest::factory($url, $data);
// Forward cookies, but only if configured to do so and if there are read restrictions
if ($wgVisualEditorParsoidForwardCookies && !User::isEveryoneAllowed('read')) {
$req->setHeader('Cookie', $this->getRequest()->getHeader('Cookie'));
}
$status = $req->execute();
if ($status->isOK()) {
// Pass thru performance data from Parsoid to the client, unless the response was
// served directly from Varnish, in which case discard the value of the XPP header
// and use it to declare the cache hit instead.
$xCache = $req->getResponseHeader('X-Cache');
if (is_string($xCache) && strpos(strtolower($xCache), 'hit') !== false) {
$xpp = 'cached-response=true';
$hit = true;
} else {
$xpp = $req->getResponseHeader('X-Parsoid-Performance');
$hit = false;
}
WikiaLogger::instance()->debug('ApiVisualEditor', array('hit' => $hit, 'method' => $method, 'url' => $url));
if ($xpp !== null) {
$resp = $this->getRequest()->response();
$resp->header('X-Parsoid-Performance: ' . $xpp);
}
} elseif ($status->isGood()) {
$this->dieUsage($req->getContent(), 'parsoidserver-http-' . $req->getStatus());
} elseif ($errors = $status->getErrorsByType('error')) {
$error = $errors[0];
$code = $error['message'];
if (count($error['params'])) {
$message = $error['params'][0];
} else {
$message = 'MWHttpRequest error';
}
$this->dieUsage($message, 'parsoidserver-' . $code);
}
// TODO pass through X-Parsoid-Performance header, merge with getHTML above
return $req->getContent();
}
public function hitUrl($zip, $attempt = 0)
{
$url = $this->makeUrl($zip);
//$this->output( "*Trying to hit $url\n" );
$req = MWHttpRequest::factory($url, array('method' => 'GET', 'timeout' => 2, 'sslVerifyHost' => false, 'sslVerifyCert' => false));
if ($req->execute()->isOK()) {
$this->isOK++;
} else {
sleep(2);
$attempt++;
if ($attempt < 3) {
$this->hitUrl($zip, $attempt);
} else {
$this->isBad++;
}
}
}
public function get($url, $postData = false)
{
$this->log("Connect: {$url} ", 1);
$options = array('followRedirects' => true, 'noProxy' => true, 'timeout' => 220);
if ($postData !== false) {
$options['postData'] = $postData;
print_r($postData);
}
$req = MWHttpRequest::factory($url, $options);
$status = $req->execute();
$decodedResponse = null;
if ($status->isOK()) {
$response = $req->getContent();
$response = iconv('UTF-8', 'UTF-8//IGNORE', utf8_encode($response));
$responseCode = $req->getStatus();
$decodedResponse = json_decode($response);
}
return array('code' => $responseCode, 'response' => $decodedResponse);
}
protected function hitThumbUrl($file, $transformParams)
{
global $wgUploadThumbnailRenderHttpCustomHost, $wgUploadThumbnailRenderHttpCustomDomain;
$thumbName = $file->thumbName($transformParams);
$thumbUrl = $file->getThumbUrl($thumbName);
if ($wgUploadThumbnailRenderHttpCustomDomain) {
$parsedUrl = wfParseUrl($thumbUrl);
if (!$parsedUrl || !isset($parsedUrl['path']) || !strlen($parsedUrl['path'])) {
return false;
}
$thumbUrl = '//' . $wgUploadThumbnailRenderHttpCustomDomain . $parsedUrl['path'];
}
wfDebug(__METHOD__ . ": hitting url {$thumbUrl}\n");
$request = MWHttpRequest::factory($thumbUrl, array('method' => 'HEAD', 'followRedirects' => true), __METHOD__);
if ($wgUploadThumbnailRenderHttpCustomHost) {
$request->setHeader('Host', $wgUploadThumbnailRenderHttpCustomHost);
}
$status = $request->execute();
return $request->getStatus();
}
/**
* Perform an HTTP request
*
* @param string $method HTTP method. Usually GET/POST
* @param string $url Full URL to act on. If protocol-relative, will be expanded to an http:// URL
* @param array $options Options to pass to MWHttpRequest object.
* Possible keys for the array:
* - timeout Timeout length in seconds
* - connectTimeout Timeout for connection, in seconds (curl only)
* - postData An array of key-value pairs or a url-encoded form data
* - proxy The proxy to use.
* Otherwise it will use $wgHTTPProxy (if set)
* Otherwise it will use the environment variable "http_proxy" (if set)
* - noProxy Don't use any proxy at all. Takes precedence over proxy value(s).
* - sslVerifyHost Verify hostname against certificate
* - sslVerifyCert Verify SSL certificate
* - caInfo Provide CA information
* - maxRedirects Maximum number of redirects to follow (defaults to 5)
* - followRedirects Whether to follow redirects (defaults to false).
* Note: this should only be used when the target URL is trusted,
* to avoid attacks on intranet services accessible by HTTP.
* - userAgent A user agent, if you want to override the default
* MediaWiki/$wgVersion
* - logger A \Psr\Logger\LoggerInterface instance for debug logging
* @param string $caller The method making this request, for profiling
* @return string|bool (bool)false on failure or a string on success
*/
public static function request($method, $url, $options = [], $caller = __METHOD__)
{
wfDebug("HTTP: {$method}: {$url}\n");
$options['method'] = strtoupper($method);
if (!isset($options['timeout'])) {
$options['timeout'] = 'default';
}
if (!isset($options['connectTimeout'])) {
$options['connectTimeout'] = 'default';
}
$req = MWHttpRequest::factory($url, $options, $caller);
$status = $req->execute();
if ($status->isOK()) {
return $req->getContent();
} else {
$errors = $status->getErrorsByType('error');
$logger = LoggerFactory::getInstance('http');
$logger->warning($status->getWikiText(false, false, 'en'), ['error' => $errors, 'caller' => $caller, 'content' => $req->getContent()]);
return false;
}
}
public function execute() {
wfProfileIn( __METHOD__ );
parent::execute();
if ( is_array( $this->postData ) ) {
$this->postData = wfArrayToCgi( $this->postData );
}
if ( $this->parsedUrl['scheme'] != 'http' &&
$this->parsedUrl['scheme'] != 'https' ) {
$this->status->fatal( 'http-invalid-scheme', $this->parsedUrl['scheme'] );
}
$this->reqHeaders['Accept'] = "*/*";
if ( $this->method == 'POST' ) {
// Required for HTTP 1.0 POSTs
$this->reqHeaders['Content-Length'] = strlen( $this->postData );
if ( !isset( $this->reqHeaders['Content-Type'] ) ) {
$this->reqHeaders['Content-Type'] = "application/x-www-form-urlencoded";
}
}
$options = array();
if ( $this->proxy ) {
$options['proxy'] = $this->urlToTCP( $this->proxy );
$options['request_fulluri'] = true;
}
if ( !$this->followRedirects ) {
$options['max_redirects'] = 0;
} else {
$options['max_redirects'] = $this->maxRedirects;
}
$options['method'] = $this->method;
$options['header'] = implode( "\r\n", $this->getHeaderList() );
// Note that at some future point we may want to support
// HTTP/1.1, but we'd have to write support for chunking
// in version of PHP < 5.3.1
$options['protocol_version'] = "1.0";
// This is how we tell PHP we want to deal with 404s (for example) ourselves.
// Only works on 5.2.10+
$options['ignore_errors'] = true;
if ( $this->postData ) {
$options['content'] = $this->postData;
}
$options['timeout'] = $this->timeout;
if ( $this->sslVerifyHost ) {
$options['CN_match'] = $this->parsedUrl['host'];
}
if ( $this->sslVerifyCert ) {
$options['verify_peer'] = true;
}
if ( is_dir( $this->caInfo ) ) {
$options['capath'] = $this->caInfo;
} elseif ( is_file( $this->caInfo ) ) {
$options['cafile'] = $this->caInfo;
} elseif ( $this->caInfo ) {
throw new MWException( "Invalid CA info passed: {$this->caInfo}" );
}
$scheme = $this->parsedUrl['scheme'];
$context = stream_context_create( array( "$scheme" => $options ) );
$this->headerList = array();
$reqCount = 0;
$url = $this->url;
$result = array();
do {
$reqCount++;
wfSuppressWarnings();
$fh = fopen( $url, "r", false, $context );
wfRestoreWarnings();
if ( !$fh ) {
break;
}
$result = stream_get_meta_data( $fh );
$this->headerList = $result['wrapper_data'];
$this->parseHeader();
if ( !$this->followRedirects ) {
break;
}
# Handle manual redirection
if ( !$this->isRedirect() || $reqCount > $this->maxRedirects ) {
break;
}
# Check security of URL
$url = $this->getResponseHeader( "Location" );
if ( !Http::isValidURI( $url ) ) {
wfDebug( __METHOD__ . ": insecure redirection\n" );
break;
}
} while ( true );
$this->setStatus();
if ( $fh === false ) {
$this->status->fatal( 'http-request-error' );
wfProfileOut( __METHOD__ );
return $this->status;
}
if ( $result['timed_out'] ) {
$this->status->fatal( 'http-timed-out', $this->url );
wfProfileOut( __METHOD__ );
return $this->status;
}
// If everything went OK, or we received some error code
// get the response body content.
if ( $this->status->isOK() || (int)$this->respStatus >= 300 ) {
while ( !feof( $fh ) ) {
$buf = fread( $fh, 8192 );
if ( $buf === false ) {
$this->status->fatal( 'http-read-error' );
break;
}
if ( strlen( $buf ) ) {
call_user_func( $this->callback, $fh, $buf );
}
}
}
fclose( $fh );
wfProfileOut( __METHOD__ );
return $this->status;
}
/**
* @param $xml
* @throws GWTException
*/
private function put_sitemap($xml)
{
$request = MWHttpRequest::factory($this->make_sitemaps_uri(), array('postData' => $xml, 'method' => 'POST'));
$request->setHeader('Content-type', 'application/atom+xml');
$request->setHeader('Content-length', strval(strlen($xml)));
$request->setHeader('Authorization', 'GoogleLogin auth=' . $this->mAuth);
$status = $request->execute();
if ($status->isOK()) {
$text = $request->getContent();
GWTLogHelper::debug($text);
} else {
throw new GWTException("Non 200 response.\n" . "\n" . "message:" . $status->getMessage() . "\n" . $request->getContent());
}
}
/**
* Like a Http:get request, but with custom User-Agent.
* @see Http::get
* @param string $url
* @param string $timeout
* @param array $options
* @param integer|bool &$mtime Resulting Last-Modified UNIX timestamp if received
* @return bool|string
*/
public static function httpGet($url, $timeout = 'default', $options = [], &$mtime = false)
{
$options['timeout'] = $timeout;
/* Http::get */
$url = wfExpandUrl($url, PROTO_HTTP);
wfDebug("ForeignAPIRepo: HTTP GET: {$url}\n");
$options['method'] = "GET";
if (!isset($options['timeout'])) {
$options['timeout'] = 'default';
}
$req = MWHttpRequest::factory($url, $options, __METHOD__);
$req->setUserAgent(ForeignAPIRepo::getUserAgent());
$status = $req->execute();
if ($status->isOK()) {
$lmod = $req->getResponseHeader('Last-Modified');
$mtime = $lmod ? wfTimestamp(TS_UNIX, $lmod) : false;
return $req->getContent();
} else {
$logger = LoggerFactory::getInstance('http');
$logger->warning($status->getWikiText(false, false, 'en'), ['caller' => 'ForeignAPIRepo::httpGet']);
return false;
}
}
public function getThumbnail()
{
wfProfileIn(__METHOD__);
$thumbnail = '';
$url = 'http://www.gamestar.de/emb/getVideoData5.cfm?vid=' . $this->videoId;
$req = MWHttpRequest::factory($url);
$req->setHeader('User-Agent', self::$REQUEST_USER_AGENT);
$status = $req->execute();
if ($status->isOK()) {
$response = trim($req->getContent());
if (!empty($response)) {
$xml = @simplexml_load_string($response);
if (isset($xml->image)) {
$thumbnail = (string) $xml->image;
}
}
}
wfProfileOut(__METHOD__);
return $thumbnail;
}
/**
* Download the file, save it to the temporary file and update the file
* size and set $mRemoveTempFile to true.
* @return Status
*/
protected function reallyFetchFile()
{
if ($this->mTempPath === false) {
return Status::newFatal('tmp-create-error');
}
// Note the temporary file should already be created by makeTemporaryFile()
$this->mTmpHandle = fopen($this->mTempPath, 'wb');
if (!$this->mTmpHandle) {
return Status::newFatal('tmp-create-error');
}
$this->mRemoveTempFile = true;
$this->mFileSize = 0;
$req = MWHttpRequest::factory($this->mUrl, array('followRedirects' => true));
$req->setCallback(array($this, 'saveTempFileChunk'));
$status = $req->execute();
if ($this->mTmpHandle) {
// File got written ok...
fclose($this->mTmpHandle);
$this->mTmpHandle = null;
} else {
// We encountered a write error during the download...
return Status::newFatal('tmp-write-error');
}
if (!$status->isOk()) {
return $status;
}
return $status;
}
/**
* @param $s Status
*/
private function subscribeToMediaWikiAnnounce(Status $s)
{
$params = array('email' => $this->getVar('_AdminEmail'), 'language' => 'en', 'digest' => 0);
// Mailman doesn't support as many languages as we do, so check to make
// sure their selected language is available
$myLang = $this->getVar('_UserLang');
if (in_array($myLang, $this->mediaWikiAnnounceLanguages)) {
$myLang = $myLang == 'pt-br' ? 'pt_BR' : $myLang;
// rewrite to Mailman's pt_BR
$params['language'] = $myLang;
}
if (MWHttpRequest::canMakeRequests()) {
$res = MWHttpRequest::factory($this->mediaWikiAnnounceUrl, array('method' => 'POST', 'postData' => $params))->execute();
if (!$res->isOK()) {
$s->warning('config-install-subscribe-fail', $res->getMessage());
}
} else {
$s->warning('config-install-subscribe-notpossible');
}
}
/**
* Set read/write permissions for a Swift container.
*
* $readGrps is a list of the possible criteria for a request to have
* access to read a container. Each item is one of the following formats:
* - account:user : Grants access if the request is by the given user
* - ".r:<regex>" : Grants access if the request is from a referrer host that
* matches the expression and the request is not for a listing.
* Setting this to '*' effectively makes a container public.
* -".rlistings:<regex>" : Grants access if the request is from a referrer host that
* matches the expression and the request is for a listing.
*
* $writeGrps is a list of the possible criteria for a request to have
* access to write to a container. Each item is of the following format:
* - account:user : Grants access if the request is by the given user
*
* @see http://swift.openstack.org/misc.html#acls
*
* In general, we don't allow listings to end-users. It's not useful, isn't well-defined
* (lists are truncated to 10000 item with no way to page), and is just a performance risk.
*
* @param $contObj CF_Container Swift container
* @param array $readGrps List of read access routes
* @param array $writeGrps List of write access routes
* @return Status
*/
protected function setContainerAccess(CF_Container $contObj, array $readGrps, array $writeGrps)
{
$creds = $contObj->cfs_auth->export_credentials();
$url = $creds['storage_url'] . '/' . rawurlencode($contObj->name);
// Note: 10 second timeout consistent with php-cloudfiles
$req = MWHttpRequest::factory($url, array('method' => 'POST', 'timeout' => 10));
$req->setHeader('X-Auth-Token', $creds['auth_token']);
$req->setHeader('X-Container-Read', implode(',', $readGrps));
$req->setHeader('X-Container-Write', implode(',', $writeGrps));
return $req->execute();
// should return 204
}
/**
* @param $url string
* @return Mixed|String
*/
function fetchScaryTemplateMaybeFromCache($url)
{
global $wgTranscludeCacheExpiry;
$dbr = wfGetDB(DB_SLAVE);
$tsCond = $dbr->timestamp(time() - $wgTranscludeCacheExpiry);
$obj = $dbr->selectRow('transcache', array('tc_time', 'tc_contents'), array('tc_url' => $url, "tc_time >= " . $dbr->addQuotes($tsCond)));
if ($obj) {
return $obj->tc_contents;
}
$req = MWHttpRequest::factory($url);
$status = $req->execute();
// Status object
if ($status->isOK()) {
$text = $req->getContent();
} elseif ($req->getStatus() != 200) {
// Though we failed to fetch the content, this status is useless.
return wfMessage('scarytranscludefailed-httpstatus', $url, $req->getStatus())->inContentLanguage()->text();
} else {
return wfMessage('scarytranscludefailed', $url)->inContentLanguage()->text();
}
$dbw = wfGetDB(DB_MASTER);
$dbw->replace('transcache', array('tc_url'), array('tc_url' => $url, 'tc_time' => $dbw->timestamp(time()), 'tc_contents' => $text));
return $text;
}
/**
* Set read/write permissions for a Swift container
*
* @param $contObj CF_Container Swift container
* @param $readGrps Array Swift users who can read (account:user)
* @param $writeGrps Array Swift users who can write (account:user)
* @return Status
*/
protected function setContainerAccess(CF_Container $contObj, array $readGrps, array $writeGrps)
{
// Wikia change - begin
// don't send multiple ACL requests for the same container over and over again (BAC-872)
static $reqSent = [];
$key = $contObj->name;
$entry = implode(',', $readGrps) . '::' . implode(',', $writeGrps);
if (isset($reqSent[$key]) && $reqSent[$key] === $entry) {
wfDebug(__METHOD__ . ": ACL already set\n");
return Status::newGood();
}
$reqSent[$key] = $entry;
// Wikia change - end
$creds = $contObj->cfs_auth->export_credentials();
$url = $creds['storage_url'] . '/' . rawurlencode($contObj->name);
wfDebug(sprintf("%s: %s (ACL - read: '%s', write: '%s')\n", __METHOD__, $url, implode(',', $readGrps), implode(',', $writeGrps)));
// Wikia change
// Note: 10 second timeout consistent with php-cloudfiles
/* @var CurlHttpRequest $req */
$req = MWHttpRequest::factory($url, array('method' => 'POST', 'timeout' => $this->swiftTimeout, 'noProxy' => true));
$req->setHeader('X-Auth-Token', $creds['auth_token']);
$req->setHeader('X-Container-Read', implode(',', $readGrps));
$req->setHeader('X-Container-Write', implode(',', $writeGrps));
return $req->execute();
// should return 204
}
protected function getMicrosoftSuggestion($serviceName, $config)
{
global $wgMemc;
$this->mustHaveDefinition();
self::checkTranslationServiceFailure($serviceName);
$code = $this->handle->getCode();
$definition = trim(strval($this->getDefinition()));
$definition = self::wrapUntranslatable($definition);
$memckey = wfMemckey('translate-tmsug-badcodes-' . $serviceName);
$unsupported = $wgMemc->get($memckey);
if (isset($unsupported[$code])) {
return null;
}
$options = array();
$options['timeout'] = $config['timeout'];
$params = array('text' => $definition, 'to' => $code);
if (isset($config['key'])) {
$params['appId'] = $config['key'];
} else {
return null;
}
$url = $config['url'] . '?' . wfArrayToCgi($params);
$url = wfExpandUrl($url);
$options['method'] = 'GET';
if (class_exists('MWHttpRequest')) {
$req = MWHttpRequest::factory($url, $options);
} else {
$req = HttpRequest::factory($url, $options);
}
$status = $req->execute();
if (!$status->isOK()) {
$error = $req->getContent();
if (strpos($error, 'must be a valid language') !== false) {
$unsupported[$code] = true;
$wgMemc->set($memckey, $unsupported, 60 * 60 * 8);
return null;
}
if ($error) {
error_log(__METHOD__ . ': Http::get failed:' . $error);
} else {
error_log(__METHOD__ . ': Unknown error, grr');
}
// Most likely a timeout or other general error
self::reportTranslationServiceFailure($serviceName);
}
$ret = $req->getContent();
$text = preg_replace('~<string.*>(.*)</string>~', '\\1', $ret);
$text = Sanitizer::decodeCharReferences($text);
$text = self::unwrapUntranslatable($text);
$text = $this->suggestionField($text);
return Html::rawElement('div', null, self::legend($serviceName) . $text . self::clear());
}
/**
* Scale a file with a remote "scaler", as exists on the Wikimedia Foundation cluster, and output it to STDOUT.
* Note: unlike the usual thumbnail process, the web client never sees the cluster URL; we do the whole HTTP transaction to the scaler ourselves
* and cat the results out.
* Note: We rely on NFS to have propagated the file contents to the scaler. However, we do not rely on the thumbnail being created in NFS and then
* propagated back to our filesystem. Instead we take the results of the HTTP request instead.
* Note: no caching is being done here, although we are instructing the client to cache it forever.
* @param $file: File object
* @param $params: scaling parameters ( e.g. array( width => '50' ) );
* @param $flags: scaling flags ( see File:: constants )
* @throws MWException
* @return boolean success
*/
private function outputRemoteScaledThumb($file, $params, $flags)
{
// this global probably looks something like 'http://upload.wikimedia.org/wikipedia/test/thumb/temp'
// do not use trailing slash
global $wgUploadStashScalerBaseUrl;
// We need to use generateThumbName() instead of thumbName(), because
// the suffix needs to match the file name for the remote thumbnailer
// to work
$scalerThumbName = $file->generateThumbName($file->getName(), $params);
$scalerThumbUrl = $wgUploadStashScalerBaseUrl . '/' . $file->getUrlRel() . '/' . rawurlencode($scalerThumbName);
// make a curl call to the scaler to create a thumbnail
$httpOptions = array('method' => 'GET', 'timeout' => 'default');
$req = MWHttpRequest::factory($scalerThumbUrl, $httpOptions);
$status = $req->execute();
if (!$status->isOK()) {
$errors = $status->getErrorsArray();
$errorStr = "Fetching thumbnail failed: " . print_r($errors, 1);
$errorStr .= "\nurl = {$scalerThumbUrl}\n";
throw new MWException($errorStr);
}
$contentType = $req->getResponseHeader("content-type");
if (!$contentType) {
throw new MWException("Missing content-type header");
}
return $this->outputContents($req->getContent(), $contentType);
}
public function execute()
{
parent::execute();
if (is_array($this->postData)) {
$this->postData = wfArrayToCgi($this->postData);
}
if ($this->parsedUrl['scheme'] != 'http' && $this->parsedUrl['scheme'] != 'https') {
$this->status->fatal('http-invalid-scheme', $this->parsedUrl['scheme']);
}
$this->reqHeaders['Accept'] = "*/*";
$this->reqHeaders['Connection'] = 'Close';
if ($this->method == 'POST') {
// Required for HTTP 1.0 POSTs
$this->reqHeaders['Content-Length'] = strlen($this->postData);
if (!isset($this->reqHeaders['Content-Type'])) {
$this->reqHeaders['Content-Type'] = "application/x-www-form-urlencoded";
}
}
// Set up PHP stream context
$options = ['http' => ['method' => $this->method, 'header' => implode("\r\n", $this->getHeaderList()), 'protocol_version' => '1.1', 'max_redirects' => $this->followRedirects ? $this->maxRedirects : 0, 'ignore_errors' => true, 'timeout' => $this->timeout, 'curl_verify_ssl_host' => $this->sslVerifyHost ? 2 : 0, 'curl_verify_ssl_peer' => $this->sslVerifyCert], 'ssl' => ['verify_peer' => $this->sslVerifyCert, 'SNI_enabled' => true, 'ciphers' => 'HIGH:!SSLv2:!SSLv3:-ADH:-kDH:-kECDH:-DSS', 'disable_compression' => true]];
if ($this->proxy) {
$options['http']['proxy'] = $this->urlToTcp($this->proxy);
$options['http']['request_fulluri'] = true;
}
if ($this->postData) {
$options['http']['content'] = $this->postData;
}
if ($this->sslVerifyHost) {
// PHP 5.6.0 deprecates CN_match, in favour of peer_name which
// actually checks SubjectAltName properly.
if (version_compare(PHP_VERSION, '5.6.0', '>=')) {
$options['ssl']['peer_name'] = $this->parsedUrl['host'];
} else {
$options['ssl']['CN_match'] = $this->parsedUrl['host'];
}
}
$options['ssl'] += $this->getCertOptions();
$context = stream_context_create($options);
$this->headerList = [];
$reqCount = 0;
$url = $this->url;
$result = [];
if ($this->profiler) {
$profileSection = $this->profiler->scopedProfileIn(__METHOD__ . '-' . $this->profileName);
}
do {
$reqCount++;
$this->fopenErrors = [];
set_error_handler([$this, 'errorHandler']);
$fh = fopen($url, "r", false, $context);
restore_error_handler();
if (!$fh) {
// HACK for instant commons.
// If we are contacting (commons|upload).wikimedia.org
// try again with CN_match for en.wikipedia.org
// as php does not handle SubjectAltName properly
// prior to "peer_name" option in php 5.6
if (isset($options['ssl']['CN_match']) && ($options['ssl']['CN_match'] === 'commons.wikimedia.org' || $options['ssl']['CN_match'] === 'upload.wikimedia.org')) {
$options['ssl']['CN_match'] = 'en.wikipedia.org';
$context = stream_context_create($options);
continue;
}
break;
}
$result = stream_get_meta_data($fh);
$this->headerList = $result['wrapper_data'];
$this->parseHeader();
if (!$this->followRedirects) {
break;
}
# Handle manual redirection
if (!$this->isRedirect() || $reqCount > $this->maxRedirects) {
break;
}
# Check security of URL
$url = $this->getResponseHeader("Location");
if (!Http::isValidURI($url)) {
$this->logger->debug(__METHOD__ . ": insecure redirection\n");
break;
}
} while (true);
if ($this->profiler) {
$this->profiler->scopedProfileOut($profileSection);
}
$this->setStatus();
if ($fh === false) {
if ($this->fopenErrors) {
$this->logger->warning(__CLASS__ . ': error opening connection: {errstr1}', $this->fopenErrors);
}
$this->status->fatal('http-request-error');
return $this->status;
}
if ($result['timed_out']) {
$this->status->fatal('http-timed-out', $this->url);
return $this->status;
}
// If everything went OK, or we received some error code
// get the response body content.
if ($this->status->isOK() || (int) $this->respStatus >= 300) {
while (!feof($fh)) {
$buf = fread($fh, 8192);
if ($buf === false) {
$this->status->fatal('http-read-error');
break;
}
if (strlen($buf)) {
call_user_func($this->callback, $fh, $buf);
}
}
}
fclose($fh);
return $this->status;
}
/**
* Like a Http:get request, but with custom User-Agent.
* @see Http:get
*/
public static function httpGet($url, $timeout = 'default', $options = array())
{
$options['timeout'] = $timeout;
/* Http::get */
$url = wfExpandUrl($url, PROTO_HTTP);
wfDebug("ForeignAPIRepo: HTTP GET: {$url}\n");
$options['method'] = "GET";
if (!isset($options['timeout'])) {
$options['timeout'] = 'default';
}
$req = MWHttpRequest::factory($url, $options);
$req->setUserAgent(ForeignAPIRepo::getUserAgent());
$status = $req->execute();
if ($status->isOK()) {
return $req->getContent();
} else {
return false;
}
}
/**
* Convert from/to wikitext/html via Parsoid.
*
* This will assume Parsoid is installed.
*
* @param string $from Format of content to convert: html|wikitext
* @param string $to Format to convert to: html|wikitext
* @param string $content
* @param Title $title
* @return string
* @throws NoParsoidException When parsoid configuration is not available
* @throws WikitextException When conversion is unsupported
*/
protected static function parsoid($from, $to, $content, Title $title)
{
list($parsoidURL, $parsoidPrefix, $parsoidTimeout, $parsoidForwardCookies) = self::parsoidConfig();
if ($from == 'html') {
$from = 'html';
} elseif (in_array($from, array('wt', 'wikitext'))) {
$from = 'wt';
} else {
throw new WikitextException('Unknown source format: ' . $from, 'process-wikitext');
}
$params = array($from => $content, 'body' => true);
if ($from === 'html') {
$params['scrubWikitext'] = 'true';
}
$prefixedDbTitle = $title->getPrefixedDBkey();
$request = \MWHttpRequest::factory($parsoidURL . '/' . $parsoidPrefix . '/' . urlencode($prefixedDbTitle), array('method' => 'POST', 'postData' => wfArrayToCgi($params), 'timeout' => $parsoidTimeout, 'connectTimeout' => 'default'));
if ($parsoidForwardCookies && !User::isEveryoneAllowed('read')) {
if (PHP_SAPI === 'cli') {
// From the command line we need to generate a cookie
$cookies = self::generateForwardedCookieForCli();
} else {
$cookies = RequestContext::getMain()->getRequest()->getHeader('Cookie');
}
$request->setHeader('Cookie', $cookies);
}
$status = $request->execute();
if (!$status->isOK()) {
$statusMsg = $status->getMessage()->text();
$msg = "Failed contacting Parsoid for title \"{$prefixedDbTitle}\": {$statusMsg}";
wfDebugLog('Flow', __METHOD__ . ": {$msg}");
throw new NoParsoidException("{$msg}", 'process-wikitext');
}
return $request->getContent();
}
