function getItemdetails($url)
{
$dataResult = array();
$baseUrl = "http://www.dianping.com";
$totalUrl = $baseUrl . $url;
cls_curl::set_gzip(true);
$content = cls_curl::get($totalUrl);
preg_match_all('#<h1 class=\\"shop-name\\">\\s(.*?)\\s</h1>#s', $content, $shopname);
$name = preg_replace("/<(a.*?)>(.*?)<(\\/a.*?)>/si", "", $shopname[0][0]);
$name = preg_replace("/<(\\/?h1.*?)>/si", "", $name);
$dataResult['name'] = $name;
preg_match_all('#<span class="info-name">营业时间：(.*?)</span>(.*?)<span class="item">\\s(.*?)\\s</span>#s', $content, $hour);
$dataResult['hour'] = $hour[3][0];
preg_match_all('#<div class="expand-info address" itemprop="street-address">\\s(.*?)\\s</div>#s', $content, $address);
$addr = preg_replace("/<(\\/?div.*?)>/si", "", $address[0][0]);
$addr = preg_replace("/<(\\/?span.*?)>/si", "", $addr);
$addr = preg_replace("/<(\\/?a.*?)>/si", "", $addr);
$dataResult['address_detail'] = $addr;
preg_match_all('#<p class="expand-info tel">\\s(.*?)\\s</p>#s', $content, $phone);
$phoneNum = preg_replace("/<(\\/?p.*?)>/si", "", $phone[0][0]);
$phoneNum = preg_replace("/<(\\/?span.*?)>/si", "", $phoneNum);
$phoneNum = preg_replace("/<(a.*?)>(.*?)<(\\/a.*?)>/si", "", $phoneNum);
$dataResult['tel'] = $phoneNum;
$dataResult['province'] = '北京市';
$dataResult['city'] = '北京';
preg_match_all('#<span class="item">人均：(.*?)</span>#s', $content, $price);
$dataResult['price'] = $price[1][0];
return $dataResult;
}
public static function http_request($url, $type = 'get', $fields)
{
// 如果是 get 方式，直接拼凑一个 url 出来
if (strtolower($type) == 'get' && !empty($fields)) {
$url = $url . (strpos($url, "?") === false ? "?" : "&") . http_build_query($fields);
}
// 随机绑定 hosts，做负载均衡
if (self::$hosts) {
$parse_url = parse_url($url);
$host = $parse_url['host'];
$key = rand(0, count(self::$hosts) - 1);
$ip = self::$hosts[$key];
$url = str_replace($host, $ip, $url);
self::$headers = array_merge(array('Host:' . $host), self::$headers);
}
curl_setopt(self::$ch, CURLOPT_URL, $url);
// 如果是 post 方式
if (strtolower($type) == 'post') {
curl_setopt(self::$ch, CURLOPT_POST, true);
curl_setopt(self::$ch, CURLOPT_POSTFIELDS, $fields);
}
if (self::$useragent) {
curl_setopt(self::$ch, CURLOPT_USERAGENT, self::$useragent);
}
if (self::$cookie) {
curl_setopt(self::$ch, CURLOPT_COOKIE, self::$cookie);
}
if (self::$cookie_jar) {
curl_setopt(self::$ch, CURLOPT_COOKIEJAR, self::$cookie_jar);
}
if (self::$cookie_file) {
curl_setopt(self::$ch, CURLOPT_COOKIEFILE, self::$cookie_file);
}
if (self::$referer) {
curl_setopt(self::$ch, CURLOPT_REFERER, self::$referer);
}
if (self::$ip) {
self::$headers = array_merge(array('CLIENT-IP:' . self::$ip, 'X-FORWARDED-FOR:' . self::$ip), self::$headers);
}
if (self::$headers) {
curl_setopt(self::$ch, CURLOPT_HTTPHEADER, self::$headers);
}
if (self::$gzip) {
curl_setopt(self::$ch, CURLOPT_ENCODING, 'gzip');
}
if (self::$proxy) {
curl_setopt(self::$ch, CURLOPT_PROXY, self::$proxy);
}
if (self::$http_raw) {
curl_setopt(self::$ch, CURLOPT_HEADER, true);
}
$data = curl_exec(self::$ch);
self::$info = curl_getinfo(self::$ch);
if ($data === false) {
//echo date("Y-m-d H:i:s"), ' Curl error: ' . curl_error( self::$ch ), "\n";
}
// 关闭句柄
curl_close(self::$ch);
//$data = substr($data, 10);
//$data = gzinflate($data);
return $data;
}
public static function init()
{
if (empty(self::$ch)) {
self::$ch = curl_init();
curl_setopt(self::$ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt(self::$ch, CURLOPT_CONNECTTIMEOUT, self::$timeout);
curl_setopt(self::$ch, CURLOPT_HEADER, false);
curl_setopt(self::$ch, CURLOPT_USERAGENT, self::$useragent);
curl_setopt(self::$ch, CURLOPT_TIMEOUT, self::$timeout + 5);
}
return self::$ch;
}
/**
* 获取用户
*
* @param string $username
* @param string $user_type followees 、followers
* @return void
* @author seatle <*****@*****.**>
* @created time :2015-07-28 09:46
*/
function get_user_index($username, $user_type = 'followees', $worker)
{
$url = "http://www.zhihu.com/people/{$username}/{$user_type}";
set_cookie();
cls_curl::set_gzip(true);
$content = cls_curl::get($url);
if (empty($content)) {
return array();
}
$users = array();
// 用户不足20个的时候，从ajax取不到用户，所以首页这里还是要取一下
preg_match_all('#<h2 class="zm-list-content-title"><a data-tip=".*?" href="http://www.zhihu.com/people/(.*?)" class="zg-link" title=".*?">(.*?)</a></h2>#', $content, $out);
$count = count($out[1]);
for ($i = 0; $i < $count; $i++) {
$d_username = empty($out[1][$i]) ? '' : $out[1][$i];
$d_nickname = empty($out[2][$i]) ? '' : $out[2][$i];
if (!empty($d_username) && !empty($d_nickname)) {
$users[$d_username] = array('username' => $d_username, 'nickname' => $d_nickname);
}
}
$keyword = $user_type == 'followees' ? '关注了' : '关注者';
$worker->log("采集用户 --- " . $username . " --- {$keyword} --- 主页 --- 成功\n");
preg_match('#<span class="zg-gray-normal">' . $keyword . '</span><br />\\s<strong>(.*?)</strong><label> 人</label>#', $content, $out);
$user_count = empty($out[1]) ? 0 : intval($out[1]);
preg_match('#<input type="hidden" name="_xsrf" value="(.*?)"/>#', $content, $out);
$_xsrf = empty($out[1]) ? '' : trim($out[1]);
preg_match('#<div class="zh-general-list clearfix" data-init="(.*?)">#', $content, $out);
$url_params = empty($out[1]) ? '' : json_decode(html_entity_decode($out[1]), true);
if (!empty($_xsrf) && !empty($url_params) && is_array($url_params)) {
$url = "http://www.zhihu.com/node/" . $url_params['nodename'];
$params = $url_params['params'];
$j = 1;
for ($i = 0; $i < $user_count; $i = $i + 20) {
$params['offset'] = $i;
$post_data = array('method' => 'next', 'params' => json_encode($params), '_xsrf' => $_xsrf);
$content = cls_curl::post($url, $post_data);
if (empty($content)) {
$worker->log("采集用户 --- " . $username . " --- {$keyword} --- 第{$j}页 --- 失败\n");
continue;
}
$rows = json_decode($content, true);
if (empty($rows['msg']) || !is_array($rows['msg'])) {
$worker->log("采集用户 --- " . $username . " --- {$keyword} --- 第{$j}页 --- 失败\n");
continue;
}
$worker->log("采集用户 --- " . $username . " --- {$keyword} --- 第{$j}页 --- 成功\n");
foreach ($rows['msg'] as $row) {
preg_match_all('#<h2 class="zm-list-content-title"><a data-tip=".*?" href="http://www.zhihu.com/people/(.*?)" class="zg-link" title=".*?">(.*?)</a></h2>#', $row, $out);
$d_username = empty($out[1][0]) ? '' : $out[1][0];
$d_nickname = empty($out[2][0]) ? '' : $out[2][0];
if (!empty($d_username) && !empty($d_nickname)) {
$users[$d_username] = array('username' => $d_username, 'nickname' => $d_nickname);
}
}
$j++;
}
}
return $users;
}
<?php
date_default_timezone_set('Asia/Shanghai');
ini_set('display_errors', 1);
include "config.php";
include "cls_curl.php";
include "db.php";
include "cache.php";
include "worker.php";
include "../rolling/RollingCurl.php";
// an array of URL's to fetch
$urls = array("http://www.dianping.com/search/category/2/45/g146p2");
$content = cls_curl::get($urls[0]);
print_r($content);
// a function that will process the returned responses
function request_callback($response, $info, $request)
{
// parse the page title out of the returned HTML
if (preg_match("~<title>(.*?)</title>~i", $response, $out)) {
$title = $out[1];
}
echo "<b>{$title}</b><br />";
print_r($info);
echo "<br>";
echo "<hr>";
}
// create a new RollingCurl object and pass it the name of your custom callback function
$rc = new RollingCurl("request_callback");
// the window size determines how many simultaneous requests to allow.
$rc->window_size = 20;
foreach ($urls as $url) {
