private function scrap_page($url)
{
$base_url = 'http://' . parse_url($url, PHP_URL_HOST);
$p = new Page($url);
$h = new simple_html_dom();
$h->load($p->content());
$boxes = $h->find('.textbox');
$result = array();
foreach ($boxes as $box) {
// image/url
$content = $box->find('.textbox-content', 0);
$url = $base_url . $content->find('a', 0)->href;
$thumb = $base_url . $content->find('img', 0)->src;
// other data
$label = $box->find('.webcss-label', 0);
$title = $label->find('p', 0)->find('a', 0)->innertext;
$title = html_entity_decode($title, ENT_COMPAT, 'UTF-8');
$h2 = $label->find('h2', 0);
$date = Text::create($h2->innertext)->cut_after('>:')->to_s();
$h5 = $label->find('h5', 0);
$tags = Text::create($h5->innertext)->strip_tags()->cut_after(':')->to_s();
$tags = array_filter(explode(',', $tags), 'trim');
$view = $label->find('.webcss_view', 0);
$m = Text::create($view->innertext)->regex_match('/(\\d+)/');
$pages = $m[1];
$item = array('title' => $title, 'url' => $url, 'date' => $date, 'pages' => $pages, 'thumb' => $thumb, 'tags' => '#' . implode('#', $tags) . '#');
$result[] = $item;
}
return array_reverse($result);
}
public function get_images($chapter_url, $prefix, $infix)
{
$ifx = Text::create($infix)->pad(3)->to_s();
$sitename = "http://mangahead.com";
$pref = Text::create($chapter_url);
if (!$pref->contain('index.php')) {
$pref = $pref->replace($sitename . '/Manga', $sitename . '/index.php/Manga');
}
$finish = false;
if ($pref->contain('?page=')) {
$page = (int) $pref->cut_after('?page=')->to_s();
$pref = $pref->cut_until('?page=');
} else {
$page = 1;
}
$pages = array();
while (!$finish) {
// file_put_contents('/tmp/head', $chapter_url."\n", FILE_APPEND);
$p = new Page($chapter_url);
$p->go_line('<blockquote>');
if ($p->curr_line()->contain(' ›')) {
$finish = false;
$chapter_url = $pref . '/?page=' . ++$page;
} else {
$finish = true;
}
$srcs = $p->curr_line()->extract_to_array('<img src="', '"');
foreach ($srcs as $src) {
$parturl = Text::create($src)->replace('index.php', 'mangas')->replace('?action=thumb', '')->to_s();
$name = basename($parturl);
$pages["{$prefix}-{$ifx}-{$name}"] = $sitename . $parturl;
}
}
return $pages;
}
public function get_images($chapter_url, $prefix, $infix)
{
$ifx = Text::create($infix)->pad(3)->to_s();
$p = new Page($chapter_url);
// grab total page
$p->go_line('id="top_bar"');
$p->go_line_regex('/of \\d+\\w+/');
$tot = $p->curr_line()->regex_match('/of (\\d+)/');
$tot = $tot[1];
// grab first image
$p->go_line('id="viewer"');
$p->next_line(2);
$src = $p->curr_line()->cut_between('src="', '"');
$name = basename($src);
$result = array("{$prefix}-{$ifx}-{$name}" => $src);
for ($i = 2; $i <= $tot; $i++) {
$p = new Page(dirname($chapter_url) . '/' . $i . '.html');
$p->go_line('id="viewer"');
$p->next_line(2);
$src = $p->curr_line()->cut_between('src="', '"');
$name = basename($src);
$result["{$prefix}-{$ifx}-{$name}"] = $src;
}
return $result;
}
public function crawl_chapter($v)
{
$ifx = Text::create($v['infix'])->pad(3)->to_s();
$prefix = $this->prefix;
$p = new Page($v['url']);
// grab total page
$p->go_line('id="top_bar"');
$p->go_line_regex('/of \\d+\\w+/');
$tot = $p->curr_line()->regex_match('/of (\\d+)/');
$tot = $tot[1];
// grab first image
$p->go_line('id="viewer"');
$p->next_line(2);
$src = $p->curr_line()->cut_between('src="', '"');
$name = basename($src);
echo "<a href='{$src}'>{$prefix}-{$ifx}-{$name}</a><br>\n";
// iterate
for ($i = 2; $i <= $tot; $i++) {
$p = new Page(dirname($v['url']) . '/' . $i . '.html');
$p->go_line('id="viewer"');
$p->next_line(2);
$src = $p->curr_line()->cut_between('src="', '"');
$name = basename($src);
echo "<a href='{$src}'>{$prefix}-{$ifx}-{$name}</a><br>\n";
}
}
public function crawl_chapter($v)
{
$ifx = Text::create($v['infix'])->pad(3)->to_s();
$p = new Page($v['url']);
// ada dua kemungkinan, 1 vol berisi n chap atau 1 chap saja
$p->go_line_or(array('data[volumechapter]', 'data[pages]'));
if ($p->curr_line()->contain('data[volumechapter]')) {
$volchaps = array();
do {
$line = $p->curr_line();
if ($line->contain('</option>')) {
$volchaps[$line->cut_between('value="', '"')->to_s()] = $line->cut_between('>', '</option')->to_s();
}
} while (!$p->next_line()->contain('</select><br'));
// traverse per chapter
foreach ($volchaps as $key2 => $val2) {
$url2 = dirname($v['url']) . '/' . $key2;
$this->crawl_page($url2, $ifx);
}
} else {
if ($p->curr_line()->contain('data[pages]')) {
$this->crawl_page($v['url'], $ifx);
}
}
}
/**
* Store a newly created resource in storage.
*
* @return Response
*/
public function store()
{
// get POST data
$input = Input::all();
// set validation rules
$rules = array('title' => 'required', 'text' => 'required', 'image' => 'required');
// validate input
$validation = Validator::make($input, $rules);
// if validation fails, return the user to the form w/ validation errors
if ($validation->fails()) {
return Redirect::back()->withErrors($validation)->withInput();
} else {
// create new Post instance
$post = Post::create(array('title' => $input['title']));
// create Text instance w/ text body
$text = Text::create(array('text' => $input['text']));
// save new Text and associate w/ new post
$post->text()->save($text);
// create the new Image
$image = Image::create(array('url' => $input['image']));
// save new Text and associate w/ new post
$post->image()->save($image);
if (isset($input['tags'])) {
foreach ($input['tags'] as $tagId) {
$tag = Tag::find($tagId);
$post->tags()->save($tag);
}
}
// associate the post with the current user
$post->author()->associate(Auth::user())->save();
// redirect to newly created post page
return Redirect::route('post.show', array($post->id));
}
}
public function crawl_page($p, $prefix, $ifx)
{
$h = new simple_html_dom();
$h->load($p->content());
$img = $h->find('#image', 0);
$src = Text::create($img->src)->cut_before('?');
$iname = $src->cut_rafter('/');
return array("{$prefix}-{$ifx}-{$iname}" => $src->to_s());
}
public function crawl_page($p, $prefix, $ifx, $i)
{
$p->go_line('id="manga-page"');
$img = $p->curr_line()->cut_between('src="', '"')->to_s();
$iname = urldecode(basename($img));
$ext = pathinfo($iname, PATHINFO_EXTENSION);
$iname = Text::create($i)->pad(3) . '.' . $ext;
return array("{$prefix}-{$ifx}-{$iname}", $img);
}
private function prepare_dir($dir, $list_url)
{
$dir = $dir ? $dir : $this->default_dir;
$tags = Text::create($list_url)->cut_after('tags=')->to_s();
$new_dir = rtrim($dir, '/') . '/' . $tags . '/';
if (!is_dir($new_dir)) {
mkdir($new_dir);
}
return $new_dir;
}
public function crawl_page($p, $prefix, $ifx)
{
$p->go_line('id="imgPage"');
$img = $p->next_line()->dup()->cut_between('src="', '"')->to_s();
$iname = urldecode(basename($img));
// 001_02_22_2012_10_46_59.jpg jadi 001.jpg
preg_match('/^([^_]+).*\\.(\\w+)$/', $iname, $m);
// $m[1] no urut, $m[2] extension
$iname = Text::create($m[1])->pad(3)->to_s() . '.' . $m[2];
return array("{$prefix}-{$ifx}-{$iname}" => $img);
}
public function crawl_page($p, $prefix, $ifx, $i)
{
$h = new simple_html_dom();
$h->load($p->content());
$main_img = $h->find('#mainImg', 0);
$img = $main_img->src;
$iname = Text::create($i)->pad(3)->to_s();
preg_match('/\\.(\\w+)$/', $img, $m);
$iname = $iname . '.' . $m[1];
return array("{$prefix}-{$ifx}-{$iname}" => $img);
}
public function crawl_page($p, $prefix, $ifx, $i)
{
$h = new simple_html_dom();
$h->load($p->content());
$img = $h->find('.manga-page', 0);
$src = $img->src;
$iname = urldecode(basename($src));
$ext = pathinfo($iname, PATHINFO_EXTENSION);
$iname = Text::create($i)->pad(3) . '.' . $ext;
return array("{$prefix}-{$ifx}-{$iname}", $src);
}
public function crawl_page($p, $ifx, $i)
{
$prefix = $this->prefix;
$p->go_line('id="manga-page"');
$img = $p->curr_line()->cut_between('src="', '"')->to_s();
$iname = urldecode(basename($img));
$ext = pathinfo($iname, PATHINFO_EXTENSION);
$iname = Text::create($i)->pad(3) . '.' . $ext;
// 12 karakter aneh
echo "<li><a href='{$img}'>{$prefix}-{$ifx}-{$iname}</a></li>\n";
$this->prev_iname = $iname;
}
public function crawl_page($p, $ifx)
{
$prefix = $this->prefix;
$p->go_line('id="imgPage"');
$img = $p->next_line()->dup()->cut_between('src="', '"')->to_s();
$iname = urldecode(basename($img));
// 001_02_22_2012_10_46_59.jpg jadi 001.jpg
preg_match('/^([^_]+).*\\.(\\w+)$/', $iname, $m);
// $m[1] no urut, $m[2] extension
$iname = Text::create($m[1])->pad(3)->to_s() . '.' . $m[2];
echo "<a href='{$img}'>{$prefix}-{$ifx}-{$iname}</a><br/>\n";
}
public function extract($columns, $s, $n, $url)
{
$result = array();
if (strpos($url, 'http://www.rlsbb.com') === 0) {
$url = str_replace('http://www.rlsbb.com', 'http://rlsbb.com', $url);
}
for ($i = $s; $i <= $n; $i++) {
$purl = rtrim($url, '/') . '/';
if ($i > 1) {
$purl .= 'page/' . $i . '/';
}
$p = new Page($purl);
$h = new simple_html_dom();
$h->load($p->content());
foreach ($h->find('div.post') as $post) {
$item = array();
$title_a = $post->find('.postTitle', 0)->find('a', 0);
$item['link'] = "<a href='{$title_a->href}'>link</a>";
$title_a = $post->find('.postTitle', 0)->find('a', 0);
$item['title'] = $title_a->innertext;
$subtitle = $post->find('.postSubTitle', 0);
$date = Text::create($subtitle->innertext)->regex_match('/Posted on (.*) in </');
$date = $date[1];
$item['date'] = $date;
$subtitle = $post->find('.postSubTitle', 0);
$categories = array();
foreach ($subtitle->find('a[rel=category tag]') as $c) {
$categories[] = $c->innertext;
}
$item['categories'] = implode(', ', $categories);
$content = $post->find('.postContent', 0);
if (!$content) {
$content = $post->find('.entry-content', 0);
}
$item['content'] = strip_tags($content->innertext, '<br>');
$item['description'] = $item['content'];
$content = $post->find('.postContent', 0);
if (!$content) {
$content = $post->find('.entry-content', 0);
}
$img = $content->find('img', 0);
$item['image'] = $img ? $img->outertext() : '';
$img2 = $content->find('img', 1);
$item['image2'] = $img2 ? $img2->outertext() : '';
$img3 = $content->find('img', 2);
$item['image3'] = $img3 ? $img3->outertext() : '';
$result[] = $item;
}
}
return $result;
}
public function crawl_chapter($v)
{
$ifx = Text::create($v['infix'])->pad(3)->to_s();
$p = new Page($v['url']);
// grab list of pages
$p->go_line('="changePage(');
$pages = $p->curr_line()->extract_to_array('href="', '"');
// grab current image
$this->crawl_page($p, $ifx);
array_shift($pages);
foreach ($pages as $purl) {
$this->crawl_page(new Page($purl), $ifx);
}
}
public function get_images($chapter_url, $prefix, $infix)
{
$ifx = Text::create($infix)->pad(3)->to_s();
$p = new Page($chapter_url);
$h = new simple_html_dom();
$h->load($p->content());
$select = $h->find('select[name="pagejump"]', 0);
$img = $h->find('#page', 0)->find('img', 0);
$srcdir = dirname($img->src);
$pages = array();
foreach ($select->find('option') as $opt) {
$pages["{$prefix}-{$infix}-{$opt->value}.jpg"] = $srcdir . '/' . $opt->value . '.jpg';
}
return $pages;
}
protected function crawl_page($p, $prefix, $ifx)
{
$p->go_line('id="img_mng_enl"');
$src = $p->curr_line()->cut_between('src="', '"')->to_s();
$name = Text::create(basename($src));
if ($name->contain('?')) {
$name = $name->cut_before('?');
}
if ($m = $name->regex_match('/(\\d+)_\\d+_\\d+_\\d+_\\d+_/')) {
$ext = $name->cut_rafter('.');
$name = $m[1] . '.' . $ext;
}
$name = urldecode($name);
return array("{$prefix}-{$ifx}-{$name}" => $src);
}
protected function crawl_page($p, $ifx)
{
$prefix = $this->prefix;
$p->go_line('id="img_mng_enl"');
$src = $p->curr_line()->cut_between('src="', '"')->to_s();
$name = Text::create(basename($src));
if ($name->contain('?')) {
$name = $name->cut_until('?');
}
if ($m = $name->regex_match('/(\\d+)_\\d+_\\d+_\\d+_\\d+_/')) {
$ext = $name->cut_rafter('.');
$name = $m[1] . '.' . $ext;
}
echo "<a href='{$src}'>{$prefix}-{$ifx}-{$name}</a><br>\n";
}
public function get_images($chapter_url, $prefix, $infix)
{
$ifx = Text::create($infix)->pad(3)->to_s();
$p = new Page($chapter_url);
// grab list of pages
$p->go_line('="changePage(');
$pages = $p->curr_line()->extract_to_array('href="', '"');
// grab current image
$result = $this->crawl_page($p, $prefix, $ifx);
array_shift($pages);
foreach ($pages as $purl) {
$result = $result + $this->crawl_page(new Page($purl), $prefix, $ifx);
}
return $result;
}
public function run()
{
DB::table('texts')->delete();
$lorem = <<<EOT
Lorem ipsum dolor sit amet, no vim virtute detracto comprehensam, iudico mentitum inimicus ad cum. Ne vocibus civibus corpora sea. Clita nominati ut est, at wisi accumsan cum. Voluptatum persequeris per an, ut aperiri delenit vix. Et agam eros omittantur cum, mutat cetero sed te. Iuvaret voluptaria sententiae ea qui, choro discere reprehendunt at nam.
Ex periculis rationibus usu. Esse omnes lucilius et mei. No vel debet deterruisset, quo an enim omnis dignissim. Errem choro invenire est te, vis semper mandamus intellegebat ei, cum dico omnium an. Cibo tollit causae quo in. Magna aeterno ponderum ad eos, libris fabulas cum no, has audire ceteros platonem ut.
Vim ne diam vitae. Et has maiorum pericula. No doming inermis eos, et prima disputando sea. Ei ludus aperiam tractatos mei. Vim zril debitis facilisi ad. Veri vidit choro mea ex.
Ut stet fabellas mediocritatem usu. Pri solet argumentum at. Cum no agam labitur, te animal neglegentur pro, eam clita iriure et. Veri erroribus ad mel.
EOT;
Text::create(array('post_id' => 1, 'text' => $lorem));
Text::create(array('post_id' => 2, 'text' => $lorem));
Text::create(array('post_id' => 3, 'text' => $lorem));
Text::create(array('post_id' => 4, 'text' => $lorem));
Text::create(array('post_id' => 5, 'text' => $lorem));
}
public function crawl_chapter($v)
{
$ifx = Text::create($v['infix'])->pad(3)->to_s();
$p = new Page($v['url']);
// grab list of pages
$p->go_line('id="page_select"');
$pages = $p->next_line()->extract_to_array('value="', '"');
// grab current image
$this->crawl_page($p, $ifx);
array_shift($pages);
foreach ($pages as $purl) {
$p = new Page($purl);
$this->crawl_page($p, $ifx);
}
/*
Manga_Crawler::multiProcess(4, $pages, array($this, 'crawl_page'), array($ifx));
*/
}
public function test_parse($page_url)
{
$p = new Page($page_url);
$h = new simple_html_dom();
$h->load($p->content());
foreach ($h->find('div.post') as $post) {
$title_a = $post->find('h3.postTitle', 0)->find('a', 0);
$subtitle = $post->find('div.postSubTitle', 0);
$date = Text::create($subtitle->innertext)->regex_match('/Posted on (.*) in </');
$date = $date[1];
$categories = array();
foreach ($subtitle->find('a[rel=category tag]') as $c) {
$categories[] = $c->innertext;
}
$content = $post->find('div.postContent', 0);
$row = array('url' => $title_a->href, 'title' => $title_a->innertext, 'date' => $date, 'category' => $categories, 'content' => htmlspecialchars($content->innertext));
print_r($row);
}
}
public function get_images($chapter_url, $prefix, $infix)
{
$ifx = Text::create($infix)->pad(3)->to_s();
$p = new Page($chapter_url);
$h = new simple_html_dom();
$h->load($p->content());
$DOMAIN = 'http://hugemanga.com/';
$first_image = $h->find('img.picture', 0);
$pattern = $DOMAIN . dirname($first_image->src) . '/' . str_replace('01', '###', basename($first_image->src));
$result = array();
$pages = $h->find('select[name="page"]', 0);
foreach ($pages->find('option') as $option) {
$page = Text::create($option->value)->pad(2)->to_s();
$full = str_replace('###', $page, $pattern);
$name = basename($full);
$result["{$prefix}-{$ifx}-{$name}"] = $full;
}
return $result;
}
public function crawl_page($p, $prefix, $ifx, $i)
{
$p->go_line('id="full_image"');
$img = $p->next_line(3)->dup()->cut_between('src="', '"')->to_s();
$iname = urldecode(basename($img));
$ext = pathinfo($iname, PATHINFO_EXTENSION);
// // 12 karakter aneh
// if (preg_match('/[0-9a-z]{13}\.\w+$/', $iname)) {
// $iname = preg_replace('/\w{13}\.(\w+)$/', '.$1', $iname);
// }
// if (preg_match('/_(\d+)_[a-zA-Z]+\.\w{3}$/', $iname, $m)) {
// $iname = $m[1].substr($iname, -4);
// } else {
// // ambil last 3 character
// $iname = substr($iname, -7);
// }
$iname = Text::create($i)->pad(3) . '.' . $ext;
return array("{$prefix}-{$ifx}-{$iname}" => $img);
}
public function crawl_chapter($v)
{
$ifx = Text::create($v['infix'])->pad(3)->to_s();
$p = new Page($v['url']);
$p->go_line('var pages');
$json = $p->curr_line()->cut_between(' = ', ';');
$list = json_decode($json);
foreach ($list as $page) {
$purl = new Text($page->url);
$name = new Text($page->filename);
if ($name->strlen() < 15) {
$name = $this->prefix . '-' . $ifx . '-' . $name;
}
if ($purl->contain('resize_img.php')) {
$purl = $purl->cut_between('resize_img.php?url=', '&width');
}
echo "<a href='{$purl}'>{$name}</a><br/>\n";
}
}
public function get_images($chapter_url, $prefix, $infix)
{
$basename = 'http://haven-reader.net';
$ifx = Text::create($infix)->pad(3)->to_s();
// open index page
$p = new Page($chapter_url . '&page=0');
$h = new simple_html_dom();
$h->load($p->content());
$main = $h->find('#main', 0);
// print_r($p->content());
$pages = array();
foreach ($main->find('img.PageLink') as $img) {
// convert thumb to img
$thumb = $img->src;
$real = Text::create($thumb)->replace('/thumbs/', '/')->regex_replace('/\\.jpg$/', '')->to_s();
$alt = $img->alt;
$pages["{$prefix}-{$ifx}-{$alt}"] = $basename . $real;
}
return $pages;
}
public function get_images($chapter_url, $prefix, $infix)
{
$ifx = Text::create($infix)->pad(3)->to_s();
$p = new Page($chapter_url);
$h = new simple_html_dom();
$h->load($p->content());
$pattern_url = Text::create($chapter_url)->regex_replace('#/p1$#', '/p%s')->to_s();
$select = $h->find('#e1', 0);
$result = array();
foreach ($select->find('option') as $option) {
$p2 = new Page(sprintf($pattern_url, $option->value));
$h2 = new simple_html_dom();
$h2->load($p2->content());
$img = $h2->find('.coverIssue', 0)->find('img', 0)->src;
$ext = pathinfo($img, PATHINFO_EXTENSION);
$iname = Text::create($option->value)->pad(3)->to_s() . '.' . $ext;
$result["{$prefix}-{$ifx}-{$iname}"] = $img;
}
return $result;
}
public function get_images($chapter_url, $prefix, $infix)
{
$ifx = Text::create($infix)->pad(3)->to_s();
$p = new Page($chapter_url, array('become_firefox' => true));
// grab list of pages
$p->go_line('var lstImages');
$i = 1;
$result = array();
do {
if ($p->curr_line()->contain('lstImages.push')) {
$line = $p->curr_line();
$img = $line->cut_between('push("', '")');
$iname = Text::create($i++)->pad(3)->to_s() . Text::create(basename($img))->cut_rfrom('.')->cut_before('?')->to_s();
$name = "{$prefix}-{$ifx}-{$iname}";
$result[$name] = $img;
}
} while (!$p->next_line()->contain('new Array()'));
$pages = $p->curr_line()->extract_to_array('href="', '"');
return $result;
}
private function download_images($images, $dir)
{
$i = 1;
$dir = substr($dir, 0, 120);
foreach ($images as $page_url) {
$ext = '.jpg';
$filename = Text::create($i)->pad(3)->to_s() . $ext;
$outpath = $dir . $filename;
if (!is_file($outpath)) {
$retry = false;
do {
$image_src = $this->get_image_src($page_url);
// $ext = Text::create($image_src)->cut_rafter('.')->to_s();
download_it($image_src, $outpath);
$retry = filesize($outpath) === 0;
} while ($retry);
}
$i++;
}
}
