public function __construct($url, $opts = false, $post_data = false)
{
$this->url = $url;
$retry = 0;
if (Crawler::$use_proxy) {
$proxy_name = Crawler::$proxy['name'];
$proxy_port = Crawler::$proxy['port'];
$proxy_user = Crawler::$proxy['user'];
$proxy_pass = Crawler::$proxy['pass'];
$request_url = $url;
$proxy_fp = @fsockopen($proxy_name, $proxy_port);
if (!$proxy_fp) {
throw new Exception('Gagal konek at ' . $url);
}
@fputs($proxy_fp, "GET {$request_url} HTTP/1.0\r\nHost: {$proxy_name}\r\n");
@fputs($proxy_fp, "Proxy-Authorization: Basic " . base64_encode("{$proxy_user}:{$proxy_pass}") . "\r\n\r\n");
$this->stream = $proxy_fp;
} else {
if (is_resource($url)) {
fseek($url, 0);
$this->stream = $url;
} else {
if ($opts && $opts['use_curl']) {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_ENCODING, "");
curl_setopt($ch, CURLOPT_TIMEOUT, 150);
if (@$opts['cookie']) {
curl_setopt($ch, CURLOPT_COOKIE, $opts['cookie']);
}
if (@$opts['referrer']) {
curl_setopt($ch, CURLOPT_REFERER, $opts['referrer']);
}
if (@$opts['agent']) {
curl_setopt($ch, CURLOPT_USERAGENT, $opts['agent']);
}
$fil = tmpfile();
$asdf = curl_exec($ch);
//print_r($asdf);
//file_put_contents('gehentai-gelo.txt', "URL:$url\n$asdf\n\n", FILE_APPEND);
fwrite($fil, $asdf);
curl_close($ch);
fseek($fil, 0);
$this->stream = $fil;
} else {
if ($opts || $post_data) {
$host = Crawler::site_name($url);
$url = Crawler::back_url($url);
$out = '';
if ($post_data) {
$out .= "POST {$url} HTTP/1.1\r\n";
} else {
$out .= "GET {$url} HTTP/1.1\r\n";
}
$out .= "Host: {$host}\r\n" . "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729)\r\n" . "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*" . "/" . "*;q=0.8\r\n" . "Accept-Language: en-us,en;q=0.5\r\n" . "Accept-Encoding: gzip,deflate\r\n" . "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" . "Keep-Alive: 3000\r\n";
if (is_array($opts) && count($opts)) {
foreach ($opts as $key => $value) {
$out .= "{$key}: {$value}\r\n";
}
}
if ($post_data) {
$out .= "Content-Type: application/x-www-form-urlencoded\r\n" . "Content-Length: " . strlen($post_data) . "\r\n" . "\r\n" . $post_data;
} else {
$out .= "\r\n";
}
$this->stream = @fsockopen($host, 80, $errno, $errstr, 150);
if (!$this->stream) {
die('Gagal konek at ' . $url);
}
//echo '<pre>'.$out.'</pre>';
@fwrite($this->stream, $out);
} else {
$this->stream = @fopen($url, 'r');
while (!$this->stream && $retry++ < 30) {
$this->stream = @fopen($url, 'r');
}
if (!$this->stream) {
throw new Exception('Gagal konek at ' . $url);
}
}
}
}
}
$this->readline();
//echo htmlentities($this->curline);flush();
}
