function http_get($opt)
{
$url = $this->base . '&opt=' . $opt;
if ($this->client_type == 'curl') {
return $this->http->get($url);
} else {
return HttpClient::quickGet($url);
}
}
/**
* 从url加载
*
* @param string $url
* @return Jquery
*/
public static function load($url, $time = 5)
{
$jq = new Jquery();
$str = HttpClient::quickGet($url, $time);
if (empty($str) || trim($str) == '') {
return false;
}
$jq->build($str);
$jq->rooturl = $url;
$bases = $jq->doms[0]->find('base');
foreach ($bases as $base) {
if (isset($base->href)) {
$jq->baseurl = $base->href;
break;
}
}
if ($jq->baseurl == null) {
$jq->baseurl = $jq->rooturl;
}
return $jq;
}
/**
* @static
* @param string $url
* @return bool|mixed|string
*/
public static function getRemoteContent($url)
{
if (ini_get("allow_url_fopen")) {
return file_get_contents($url);
} else {
if (function_exists("curl_init")) {
$ch = curl_init();
$timeout = 30;
// set to zero for no timeout
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
$return = curl_exec($ch);
curl_close($ch);
return $return;
} else {
$i = parse_url($url);
$httpClient = new HttpClient($i["host"]);
$httpClient->timeout = 30;
return $httpClient->quickGet($url);
}
}
}
/**
* @brief Retrieves a serialized object via HTTP and deserializes it
*
* Useful if UDP traffic isn't allowed
*
* @param url the URL of the object
* @return the deserialized object
*/
function unserializeFromURL($url)
{
require_once 'includes/HttpClient.class.php';
return gsQuery::unserialize(HttpClient::quickGet($url));
}
function doRequest($initrequest = null)
{
// Performs the actual HTTP request, returning true or false depending on outcome
if (!($fp = @fsockopen($this->host, $this->port, $errno, $errstr, $this->timeout))) {
// Set error message
switch ($errno) {
case -3:
$this->errormsg = 'Socket creation failed (-3)';
case -4:
$this->errormsg = 'DNS lookup failure (-4)';
case -5:
$this->errormsg = 'Connection refused or timed out (-5)';
default:
$this->errormsg = 'Connection failed (' . $errno . ')';
$this->errormsg .= ' ' . $errstr;
$this->debug($this->errormsg);
}
return false;
}
socket_set_timeout($fp, $this->timeout);
if ($initrequest === null) {
$request = $this->buildRequest();
} else {
$request = $initrequest;
}
$this->debug('Request', $request);
fwrite($fp, $request);
// Reset all the variables that should not persist between requests
$this->headers = array();
$this->content = '';
$this->errormsg = '';
// Set a couple of flags
$inHeaders = true;
$atStart = true;
$htmlval = true;
// Now start reading back the response
while (!feof($fp)) {
$line = fgets($fp);
if ($atStart) {
// Deal with first line of returned data
$atStart = false;
if (!preg_match('/HTTP\\/(\\d\\.\\d)\\s*(\\d+)\\s*(.*)/', $line, $m)) {
$this->errormsg = "Status code line invalid: " . htmlentities($line);
$this->debug($this->errormsg);
return false;
}
$http_version = $m[1];
// not used
$this->status = $m[2];
$status_string = $m[3];
// not used
if ($this->status == '404') {
$this->content = null;
fclose($fp);
return false;
}
$this->debug(trim($line));
continue;
}
if ($inHeaders) {
if (trim($line) == '') {
$inHeaders = false;
$this->debug('Received Headers', $this->headers);
if ($this->headers_only) {
break;
// Skip the rest of the input
}
continue;
}
if (!preg_match('/([^:]+):\\s*(.*)/', $line, $m)) {
// Skip to the next header
continue;
}
$key = strtolower(trim($m[1]));
$val = trim($m[2]);
// Deal with the possibility of multiple headers of same name
if (isset($this->headers[$key])) {
if (is_array($this->headers[$key])) {
$this->headers[$key][] = $val;
} else {
$this->headers[$key] = array($this->headers[$key], $val);
}
} else {
$this->headers[$key] = $val;
}
continue;
}
if ($htmlval && $this->justhtml) {
$head = $this->headers;
if (stripos($this->headers['content-type'], 'text/html') === false) {
$this->content = null;
fclose($fp);
return false;
}
}
$htmlval = false;
// We're not in the headers, so append the line to the contents
$this->content .= $line;
}
fclose($fp);
//判断页面编码
if (stripos($this->headers['content-type'], 'charset') !== false) {
$encode = String::find_first_string_by_reg($this->headers['content-type'], "/\\bcharset\\s*=\\s*\\w+[-]?[8]?\\b/");
$encode = explode('=', $encode);
if (isset($encode[1])) {
$this->charset = trim($encode[1]);
}
}
//解决重定向的问题
if ($this->status == '301' || $this->status == '302' || isset($this->headers['location']) && trim($this->headers['location']) != '') {
$mburl = $this->headers['location'];
$mburl = self::dealUrl($this->oldurl, $mburl);
$this->content = HttpClient::quickGet($mburl, 30, $this->cookies);
return true;
}
// If data is compressed, uncompress it
if (isset($this->headers['content-encoding']) && $this->headers['content-encoding'] == 'gzip') {
$this->debug('Content is gzip encoded, unzipping it');
$this->content = substr($this->content, 10);
// See http://www.php.net/manual/en/function.gzencode.php
$this->content = gzinflate($this->content);
}
// If $persist_cookies, deal with any cookies
if ($this->persist_cookies && isset($this->headers['set-cookie'])) {
$cookies = $this->headers['set-cookie'];
if (!is_array($cookies)) {
$cookies = array($cookies);
}
foreach ($cookies as $cookie) {
if (preg_match('/([^=]+)=(.+)/', $cookie, $m)) {
$value = explode(';', $m[2]);
$this->cookies[$m[1]] = $value[0];
//中继器的 Cookie还原
//setcookie($m[1],urldecode($m[2]));
}
}
// Record domain of cookies for security reasons
$this->cookie_host = $this->host;
}
// If $persist_referers, set the referer ready for the next request
if ($this->persist_referers) {
$this->debug('Persisting referer: ' . $this->getRequestURL());
$this->referer = $this->getRequestURL();
}
// Finally, if handle_redirects and a redirect is sent, do that
if ($this->handle_redirects) {
if (++$this->redirect_count >= $this->max_redirects) {
$this->errormsg = 'Number of redirects exceeded maximum (' . $this->max_redirects . ')';
$this->debug($this->errormsg);
$this->redirect_count = 0;
return false;
}
$location = isset($this->headers['location']) ? $this->headers['location'] : '';
$uri = isset($this->headers['uri']) ? $this->headers['uri'] : '';
if ($location || $uri) {
$url = parse_url($location . $uri);
// This will FAIL if redirect is to a different site
return $this->get($url['path']);
}
}
return true;
}
public function readContent($url)
{
$httpClient = new HttpClient("api.douban.com", 80);
return $httpClient->quickGet($url);
}
}
$httpClient->setReferer($refUrl);
$contentUrl = "http://epub.cnki.net" . $u;
$contentSize = 0;
do {
$httpClient->get($contentUrl);
$content = $httpClient->getContent();
//302页面
/*解析地址*/
$contentUrl = get_content_url($content);
echo $contentUrl . "\n";
$saveContent = $paperName . "\t" . $contentUrl . "\n";
save($mapFile, $saveContent, "a+");
//echo "save $saveContent\n";
/*抓取论文摘要内容*/
$content = $httpClient->quickGet($contentUrl);
$contentSize = strlen($content);
if ($contentSize > 300) {
save($cachedHtml, $content);
} else {
fakeSleep();
}
} while ($contentSize < 300);
} else {
$sleep = false;
echo "Hit\n";
$content = file_get_contents($localedCachedHtml);
if (strlen($content) < 300) {
delFile($cachedHtml);
echo "Empty abstract file \n";
}
function _doautocomplete(&$form, $inputtype, &$input, &$values)
{
global $request;
$input['class'] = "dropdown";
$input['acdropdown'] = "true";
//$input['autocomplete'] = "OFF";
$input['autocomplete_complete'] = "true";
// only match begin: autocomplete_matchbegin, or
$input['autocomplete_matchsubstring'] = "true";
if (empty($values)) {
if ($input['method']) {
if (empty($input['args'])) {
if (preg_match("/^(.*?) (.*)\$/", $input['method'], $m)) {
$input['method'] = $m[1];
$input['args'] = $m[2];
} else {
$input['args'] = null;
}
}
static $tmpArray = 'tmpArray00';
// deferred remote xmlrpc call
if (string_starts_with($input['method'], "dynxmlrpc:")) {
// how is server + method + args encoding parsed by acdropdown?
$input['autocomplete_list'] = substr($input['method'], 3);
if ($input['args']) {
$input['autocomplete_list'] .= " " . $input['args'];
}
// static xmlrpc call, local only
} elseif (string_starts_with($input['method'], "xmlrpc:")) {
include_once "lib/XmlRpcClient.php";
$values = wiki_xmlrpc_post(substr($input['method'], 7), $input['args']);
} elseif (string_starts_with($input['method'], "url:")) {
include_once "lib/HttpClient.php";
$html = HttpClient::quickGet(substr($input['method'], 4));
//TODO: how to parse the HTML result into a list?
} elseif (string_starts_with($input['method'], "dynurl:")) {
$input['autocomplete_list'] = substr($input['method'], 3);
} elseif (string_starts_with($input['method'], "plugin:")) {
$dbi = $request->getDbh();
$pluginName = substr($input['method'], 7);
$basepage = '';
require_once "lib/WikiPlugin.php";
$w = new WikiPluginLoader();
$p = $w->getPlugin($pluginName, false);
// second arg?
if (!is_object($p)) {
trigger_error("invalid input['method'] " . $input['method'], E_USER_WARNING);
}
$pagelist = $p->run($dbi, @$input['args'], $request, $basepage);
$values = array();
if (is_object($pagelist) and isa($pagelist, 'PageList')) {
foreach ($pagelist->_pages as $page) {
if (is_object($page)) {
$values[] = $page->getName();
} else {
$values[] = (string) $page;
}
}
}
} elseif (string_starts_with($input['method'], "array:")) {
// some predefined values (e.g. in a template or themeinfo.php)
$input['autocomplete_list'] = $input['method'];
} else {
trigger_error("invalid input['method'] " . $input['method'], E_USER_WARNING);
}
if (empty($input['autocomplete_list'])) {
$tmpArray++;
$input['autocomplete_list'] = "array:" . $tmpArray;
$svalues = empty($values) ? "" : join("','", $values);
$form->pushContent(JavaScript("var {$tmpArray} = new Array('" . $svalues . "')"));
}
if (count($values) == 1) {
$input['value'] = $values[0];
} else {
$input['value'] = "";
}
unset($input['method']);
unset($input['args']);
//unset($input['autocomplete']);
} elseif ($s = $request->getArg($input['name'])) {
$input['value'] = $s;
}
}
return true;
}
/**
* 获得手机号码位置
* @return array('国家','省','市','电信') | false
*/
public static function local_cell($tel)
{
$res = HttpClient::quickGet('http://life.tenpay.com/cgi-bin/mobile/MobileQueryAttribution.cgi?chgmobile=' . $tel, 1, false);
if ($res) {
$res = simplexml_load_string($res);
if ($res->retmsg == 'OK') {
if (trim($res->province) != '') {
return array('中国', trim($res->province), trim($res->city), trim($res->supplier));
}
}
}
return false;
}
// extract the first paragraph of the message only
preg_match_all($paragraphPattern, $message, $paragraphs);
$messagesOutput .= "<p style='font-family: Helvetica;'>" . $paragraphs[1][0] . "</p>";
}
}
// $messagesOutput .= "<p style='font-family: Helvetica; font-style:italic;'>[DLR Messages are in the next version of TubeStatus (1.1), I am waiting on Apple to load into the iTunes Store. Keep the feedback coming!]</p>";
$messagesOutput .= "]]></messages>\n";
$output .= $messagesOutput;
$output .= "</line>\n";
}
}
/*
* DLR Messages
*/
$dlrUrl = "http://www.tfl.gov.uk/tfl/livetravelnews/realtime/dlr/default.html";
$dlrContent = HttpClient::quickGet($dlrUrl);
// paragraph matching pattern
$paragraphPattern = "/\\<p>(.*?)<\\/p>/is";
// extract the messages from the dlr content
$messagePattern = "/\\<div class=\"message\">(.*?)<\\/div>/is";
preg_match_all($messagePattern, $dlrContent, $messages);
$messages = $messages[0];
$output .= " <line name='DLR' statustype='DLR' statustext='Docklands Light Railway'>\n";
// if the dlr has messages
$messagesOutput = "";
$messageCount = count($messages);
if ($messageCount > 0) {
$messagesOutput = "<messages><![CDATA[";
for ($indexMessage = 0; $indexMessage < $messageCount; $indexMessage++) {
$message = $messages[$indexMessage];
$message = str_replace('<div class="message">', "", $message);
<?php
include dirname(__FILE__) . '/http/HttpClient.class.php';
$web_url = $_POST["web_url"];
$charset = "utf-8";
$pageContents = "";
/* 如果前7个字符不是 "http://"，则补上 */
if (substr($web_url, 0, 7) != "http://" && substr($web_url, 0, 5) != "https" && $web_url != "") {
$web_url = "http://" . $web_url;
}
if ($web_url != "") {
$pageContents = HttpClient::quickGet($web_url);
/* 需要做转换的网页编码格式 */
$charset_list = array("gb2312", "gbk");
/* 查找是否存在指定编码 */
foreach ($charset_list as $charset_one) {
$charset_pos = stripos($pageContents, $charset_one, 0);
if ($charset_pos != false) {
/* 查找左括号的位置 */
$bracket_pos = strripos(substr($pageContents, 0, $charset_pos), "<", 0);
if ($bracket_pos != false) {
/* 查找字符编码所在标签是否是meta */
$meta_pos = stripos(substr($pageContents, $bracket_pos, $bracket_pos + 4), "meta");
if ($meta_pos != false) {
$charset = $charset_pos;
break;
}
}
}
}
}
/**
* wechat::getAccessToken()
* 获取access_token
* @return string
*/
public function getAccessToken()
{
$request = HttpClient::quickGet('https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid=' . $this->wechat_config['app_id'] . '&secret=' . $this->wechat_config['app_secret'], $this->wechat_config['cacert_url']);
$requestArray = json_decode($request, true);
//var_dump($requestArray);
if (isset($requestArray['errcode'])) {
return false;
}
$accessToken = $requestArray['access_token'];
return $accessToken;
}
<?php
include 'HttpClient.class.php';
//抓取页面的内容
$contents = HttpClient::quickGet('http://www.baidu.com/');
var_dump($contents);
//post请求某一个接口，返回的信息赋值给$res
$res = HttpClient::quickPost('http://example.com/sms.php', array('name' => 'kevin.liu', 'phone' => '18201042042'));
//可能有一些请求访问会出问题，则需要加一个userAgent
$client = new HttpClient('example.com');
$client->setDebug(true);
$client->setUserAgent('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.3a) Gecko/20021207');
if (!$client->get('/')) {
die('An error occurred: ' . $client->getError());
}
$contents = $client->getContent();
//还有一些情况是：在采集数据的时候必须先登陆，则可以先模拟登陆
$client = new HttpClient('example.com');
$client->post('/login.php', array('username' => 'kevin', 'password' => '123456'));
if (!$client->get('/private.php')) {
//采集数据的目标地址
die('An error occurred: ' . $client->getError());
}
$pageContents = $client->getContent();
