function http_get($opt) { $url = $this->base . '&opt=' . $opt; if ($this->client_type == 'curl') { return $this->http->get($url); } else { return HttpClient::quickGet($url); } }
/** * 从url加载 * * @param string $url * @return Jquery */ public static function load($url, $time = 5) { $jq = new Jquery(); $str = HttpClient::quickGet($url, $time); if (empty($str) || trim($str) == '') { return false; } $jq->build($str); $jq->rooturl = $url; $bases = $jq->doms[0]->find('base'); foreach ($bases as $base) { if (isset($base->href)) { $jq->baseurl = $base->href; break; } } if ($jq->baseurl == null) { $jq->baseurl = $jq->rooturl; } return $jq; }
/** * @static * @param string $url * @return bool|mixed|string */ public static function getRemoteContent($url) { if (ini_get("allow_url_fopen")) { return file_get_contents($url); } else { if (function_exists("curl_init")) { $ch = curl_init(); $timeout = 30; // set to zero for no timeout curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout); $return = curl_exec($ch); curl_close($ch); return $return; } else { $i = parse_url($url); $httpClient = new HttpClient($i["host"]); $httpClient->timeout = 30; return $httpClient->quickGet($url); } } }
/** * @brief Retrieves a serialized object via HTTP and deserializes it * * Useful if UDP traffic isn't allowed * * @param url the URL of the object * @return the deserialized object */ function unserializeFromURL($url) { require_once 'includes/HttpClient.class.php'; return gsQuery::unserialize(HttpClient::quickGet($url)); }
function doRequest($initrequest = null) { // Performs the actual HTTP request, returning true or false depending on outcome if (!($fp = @fsockopen($this->host, $this->port, $errno, $errstr, $this->timeout))) { // Set error message switch ($errno) { case -3: $this->errormsg = 'Socket creation failed (-3)'; case -4: $this->errormsg = 'DNS lookup failure (-4)'; case -5: $this->errormsg = 'Connection refused or timed out (-5)'; default: $this->errormsg = 'Connection failed (' . $errno . ')'; $this->errormsg .= ' ' . $errstr; $this->debug($this->errormsg); } return false; } socket_set_timeout($fp, $this->timeout); if ($initrequest === null) { $request = $this->buildRequest(); } else { $request = $initrequest; } $this->debug('Request', $request); fwrite($fp, $request); // Reset all the variables that should not persist between requests $this->headers = array(); $this->content = ''; $this->errormsg = ''; // Set a couple of flags $inHeaders = true; $atStart = true; $htmlval = true; // Now start reading back the response while (!feof($fp)) { $line = fgets($fp); if ($atStart) { // Deal with first line of returned data $atStart = false; if (!preg_match('/HTTP\\/(\\d\\.\\d)\\s*(\\d+)\\s*(.*)/', $line, $m)) { $this->errormsg = "Status code line invalid: " . htmlentities($line); $this->debug($this->errormsg); return false; } $http_version = $m[1]; // not used $this->status = $m[2]; $status_string = $m[3]; // not used if ($this->status == '404') { $this->content = null; fclose($fp); return false; } $this->debug(trim($line)); continue; } if ($inHeaders) { if (trim($line) == '') { $inHeaders = false; $this->debug('Received Headers', $this->headers); if ($this->headers_only) { break; // Skip the rest of the input } continue; } if (!preg_match('/([^:]+):\\s*(.*)/', $line, $m)) { // Skip to the next header continue; } $key = strtolower(trim($m[1])); $val = trim($m[2]); // Deal with the possibility of multiple headers of same name if (isset($this->headers[$key])) { if (is_array($this->headers[$key])) { $this->headers[$key][] = $val; } else { $this->headers[$key] = array($this->headers[$key], $val); } } else { $this->headers[$key] = $val; } continue; } if ($htmlval && $this->justhtml) { $head = $this->headers; if (stripos($this->headers['content-type'], 'text/html') === false) { $this->content = null; fclose($fp); return false; } } $htmlval = false; // We're not in the headers, so append the line to the contents $this->content .= $line; } fclose($fp); //判断页面编码 if (stripos($this->headers['content-type'], 'charset') !== false) { $encode = String::find_first_string_by_reg($this->headers['content-type'], "/\\bcharset\\s*=\\s*\\w+[-]?[8]?\\b/"); $encode = explode('=', $encode); if (isset($encode[1])) { $this->charset = trim($encode[1]); } } //解决重定向的问题 if ($this->status == '301' || $this->status == '302' || isset($this->headers['location']) && trim($this->headers['location']) != '') { $mburl = $this->headers['location']; $mburl = self::dealUrl($this->oldurl, $mburl); $this->content = HttpClient::quickGet($mburl, 30, $this->cookies); return true; } // If data is compressed, uncompress it if (isset($this->headers['content-encoding']) && $this->headers['content-encoding'] == 'gzip') { $this->debug('Content is gzip encoded, unzipping it'); $this->content = substr($this->content, 10); // See http://www.php.net/manual/en/function.gzencode.php $this->content = gzinflate($this->content); } // If $persist_cookies, deal with any cookies if ($this->persist_cookies && isset($this->headers['set-cookie'])) { $cookies = $this->headers['set-cookie']; if (!is_array($cookies)) { $cookies = array($cookies); } foreach ($cookies as $cookie) { if (preg_match('/([^=]+)=(.+)/', $cookie, $m)) { $value = explode(';', $m[2]); $this->cookies[$m[1]] = $value[0]; //中继器的 Cookie还原 //setcookie($m[1],urldecode($m[2])); } } // Record domain of cookies for security reasons $this->cookie_host = $this->host; } // If $persist_referers, set the referer ready for the next request if ($this->persist_referers) { $this->debug('Persisting referer: ' . $this->getRequestURL()); $this->referer = $this->getRequestURL(); } // Finally, if handle_redirects and a redirect is sent, do that if ($this->handle_redirects) { if (++$this->redirect_count >= $this->max_redirects) { $this->errormsg = 'Number of redirects exceeded maximum (' . $this->max_redirects . ')'; $this->debug($this->errormsg); $this->redirect_count = 0; return false; } $location = isset($this->headers['location']) ? $this->headers['location'] : ''; $uri = isset($this->headers['uri']) ? $this->headers['uri'] : ''; if ($location || $uri) { $url = parse_url($location . $uri); // This will FAIL if redirect is to a different site return $this->get($url['path']); } } return true; }
public function readContent($url) { $httpClient = new HttpClient("api.douban.com", 80); return $httpClient->quickGet($url); }
} $httpClient->setReferer($refUrl); $contentUrl = "http://epub.cnki.net" . $u; $contentSize = 0; do { $httpClient->get($contentUrl); $content = $httpClient->getContent(); //302页面 /*解析地址*/ $contentUrl = get_content_url($content); echo $contentUrl . "\n"; $saveContent = $paperName . "\t" . $contentUrl . "\n"; save($mapFile, $saveContent, "a+"); //echo "save $saveContent\n"; /*抓取论文摘要内容*/ $content = $httpClient->quickGet($contentUrl); $contentSize = strlen($content); if ($contentSize > 300) { save($cachedHtml, $content); } else { fakeSleep(); } } while ($contentSize < 300); } else { $sleep = false; echo "Hit\n"; $content = file_get_contents($localedCachedHtml); if (strlen($content) < 300) { delFile($cachedHtml); echo "Empty abstract file \n"; }
function _doautocomplete(&$form, $inputtype, &$input, &$values) { global $request; $input['class'] = "dropdown"; $input['acdropdown'] = "true"; //$input['autocomplete'] = "OFF"; $input['autocomplete_complete'] = "true"; // only match begin: autocomplete_matchbegin, or $input['autocomplete_matchsubstring'] = "true"; if (empty($values)) { if ($input['method']) { if (empty($input['args'])) { if (preg_match("/^(.*?) (.*)\$/", $input['method'], $m)) { $input['method'] = $m[1]; $input['args'] = $m[2]; } else { $input['args'] = null; } } static $tmpArray = 'tmpArray00'; // deferred remote xmlrpc call if (string_starts_with($input['method'], "dynxmlrpc:")) { // how is server + method + args encoding parsed by acdropdown? $input['autocomplete_list'] = substr($input['method'], 3); if ($input['args']) { $input['autocomplete_list'] .= " " . $input['args']; } // static xmlrpc call, local only } elseif (string_starts_with($input['method'], "xmlrpc:")) { include_once "lib/XmlRpcClient.php"; $values = wiki_xmlrpc_post(substr($input['method'], 7), $input['args']); } elseif (string_starts_with($input['method'], "url:")) { include_once "lib/HttpClient.php"; $html = HttpClient::quickGet(substr($input['method'], 4)); //TODO: how to parse the HTML result into a list? } elseif (string_starts_with($input['method'], "dynurl:")) { $input['autocomplete_list'] = substr($input['method'], 3); } elseif (string_starts_with($input['method'], "plugin:")) { $dbi = $request->getDbh(); $pluginName = substr($input['method'], 7); $basepage = ''; require_once "lib/WikiPlugin.php"; $w = new WikiPluginLoader(); $p = $w->getPlugin($pluginName, false); // second arg? if (!is_object($p)) { trigger_error("invalid input['method'] " . $input['method'], E_USER_WARNING); } $pagelist = $p->run($dbi, @$input['args'], $request, $basepage); $values = array(); if (is_object($pagelist) and isa($pagelist, 'PageList')) { foreach ($pagelist->_pages as $page) { if (is_object($page)) { $values[] = $page->getName(); } else { $values[] = (string) $page; } } } } elseif (string_starts_with($input['method'], "array:")) { // some predefined values (e.g. in a template or themeinfo.php) $input['autocomplete_list'] = $input['method']; } else { trigger_error("invalid input['method'] " . $input['method'], E_USER_WARNING); } if (empty($input['autocomplete_list'])) { $tmpArray++; $input['autocomplete_list'] = "array:" . $tmpArray; $svalues = empty($values) ? "" : join("','", $values); $form->pushContent(JavaScript("var {$tmpArray} = new Array('" . $svalues . "')")); } if (count($values) == 1) { $input['value'] = $values[0]; } else { $input['value'] = ""; } unset($input['method']); unset($input['args']); //unset($input['autocomplete']); } elseif ($s = $request->getArg($input['name'])) { $input['value'] = $s; } } return true; }
/** * 获得手机号码位置 * @return array('国家','省','市','电信') | false */ public static function local_cell($tel) { $res = HttpClient::quickGet('http://life.tenpay.com/cgi-bin/mobile/MobileQueryAttribution.cgi?chgmobile=' . $tel, 1, false); if ($res) { $res = simplexml_load_string($res); if ($res->retmsg == 'OK') { if (trim($res->province) != '') { return array('中国', trim($res->province), trim($res->city), trim($res->supplier)); } } } return false; }
// extract the first paragraph of the message only preg_match_all($paragraphPattern, $message, $paragraphs); $messagesOutput .= "<p style='font-family: Helvetica;'>" . $paragraphs[1][0] . "</p>"; } } // $messagesOutput .= "<p style='font-family: Helvetica; font-style:italic;'>[DLR Messages are in the next version of TubeStatus (1.1), I am waiting on Apple to load into the iTunes Store. Keep the feedback coming!]</p>"; $messagesOutput .= "]]></messages>\n"; $output .= $messagesOutput; $output .= "</line>\n"; } } /* * DLR Messages */ $dlrUrl = "http://www.tfl.gov.uk/tfl/livetravelnews/realtime/dlr/default.html"; $dlrContent = HttpClient::quickGet($dlrUrl); // paragraph matching pattern $paragraphPattern = "/\\<p>(.*?)<\\/p>/is"; // extract the messages from the dlr content $messagePattern = "/\\<div class=\"message\">(.*?)<\\/div>/is"; preg_match_all($messagePattern, $dlrContent, $messages); $messages = $messages[0]; $output .= " <line name='DLR' statustype='DLR' statustext='Docklands Light Railway'>\n"; // if the dlr has messages $messagesOutput = ""; $messageCount = count($messages); if ($messageCount > 0) { $messagesOutput = "<messages><![CDATA["; for ($indexMessage = 0; $indexMessage < $messageCount; $indexMessage++) { $message = $messages[$indexMessage]; $message = str_replace('<div class="message">', "", $message);
<?php include dirname(__FILE__) . '/http/HttpClient.class.php'; $web_url = $_POST["web_url"]; $charset = "utf-8"; $pageContents = ""; /* 如果前7个字符不是 "http://",则补上 */ if (substr($web_url, 0, 7) != "http://" && substr($web_url, 0, 5) != "https" && $web_url != "") { $web_url = "http://" . $web_url; } if ($web_url != "") { $pageContents = HttpClient::quickGet($web_url); /* 需要做转换的网页编码格式 */ $charset_list = array("gb2312", "gbk"); /* 查找是否存在指定编码 */ foreach ($charset_list as $charset_one) { $charset_pos = stripos($pageContents, $charset_one, 0); if ($charset_pos != false) { /* 查找左括号的位置 */ $bracket_pos = strripos(substr($pageContents, 0, $charset_pos), "<", 0); if ($bracket_pos != false) { /* 查找字符编码所在标签是否是meta */ $meta_pos = stripos(substr($pageContents, $bracket_pos, $bracket_pos + 4), "meta"); if ($meta_pos != false) { $charset = $charset_pos; break; } } } } }
/** * wechat::getAccessToken() * 获取access_token * @return string */ public function getAccessToken() { $request = HttpClient::quickGet('https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid=' . $this->wechat_config['app_id'] . '&secret=' . $this->wechat_config['app_secret'], $this->wechat_config['cacert_url']); $requestArray = json_decode($request, true); //var_dump($requestArray); if (isset($requestArray['errcode'])) { return false; } $accessToken = $requestArray['access_token']; return $accessToken; }
<?php include 'HttpClient.class.php'; //抓取页面的内容 $contents = HttpClient::quickGet('http://www.baidu.com/'); var_dump($contents); //post请求某一个接口,返回的信息赋值给$res $res = HttpClient::quickPost('http://example.com/sms.php', array('name' => 'kevin.liu', 'phone' => '18201042042')); //可能有一些请求访问会出问题,则需要加一个userAgent $client = new HttpClient('example.com'); $client->setDebug(true); $client->setUserAgent('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.3a) Gecko/20021207'); if (!$client->get('/')) { die('An error occurred: ' . $client->getError()); } $contents = $client->getContent(); //还有一些情况是:在采集数据的时候必须先登陆,则可以先模拟登陆 $client = new HttpClient('example.com'); $client->post('/login.php', array('username' => 'kevin', 'password' => '123456')); if (!$client->get('/private.php')) { //采集数据的目标地址 die('An error occurred: ' . $client->getError()); } $pageContents = $client->getContent();