Returns the children nodes of the current selection.
|
public children ( ) : |
||
| return | A Crawler instance with the children nodes | |
/**
* Get the selected value from a select field.
*
* @param \Symfony\Component\DomCrawler\Crawler $select
* @return array
*/
protected function getSelectedValueFromSelect(Crawler $select)
{
$selected = [];
foreach ($select->children() as $option) {
if ($option->nodeName === 'optgroup') {
foreach ($option->childNodes as $child) {
if ($child->hasAttribute('selected')) {
$selected[] = $this->getOptionValue($child);
}
}
} elseif ($option->hasAttribute('selected')) {
$selected[] = $this->getOptionValue($option);
}
}
return $selected;
}
public static function parse($contents)
{
$crawler = new Crawler();
$crawler->addHTMLContent($contents, 'UTF-8');
$items = $crawler->filter('td[style="padding-left: 5px;"] table[width="100%"]');
//Bypass Undefined variable error.
$staff = null;
$characters = null;
foreach ($items as $item) {
//Bypass to determine if the last table contains the staff members
$crawler = new Crawler($item);
if ($crawler->filter('td[width="27"]')->count() != 1) {
$staffitems = $crawler->children();
foreach ($staffitems as $staffitem) {
$staff[] = self::parseStaff($staffitem);
}
} else {
$characters[] = self::parseCharacters($crawler);
}
}
return array('Characters' => $characters, 'Staff' => $staff);
}
/**
* Get the selected value from a select field.
*
* @param \Symfony\Component\DomCrawler\Crawler $field
* @return string|null
*
* @throws \Exception
*/
protected function getSelectedValueFromSelect(Crawler $field)
{
if ($field->nodeName() !== 'select') {
throw new Exception('Given element is not a select element.');
}
foreach ($field->children() as $option) {
if ($option->hasAttribute('selected')) {
return $option->getAttribute('value');
}
}
return;
}
function authorGroup(Crawler $groupNode)
{
$data = ['link' => '', 'pages' => []];
$done = false;
$groupNode->children()->each(function (Crawler $node) use(&$data, &$done) {
if ($done) {
return;
}
switch (strtolower($node->nodeName())) {
case 'p':
$links = $node->filter('b a');
$name = firstNode($links, function (Crawler $node) {
return trim($node->attr('name')) != '';
});
$title = firstNode($links, function (Crawler $node) {
return trim($node->text()) != '';
});
$link = firstNode($links, function (Crawler $node) {
return trim($node->attr('href')) != '';
});
$data['idx'] = intval(str_replace('gr', '', $name->attr('name')));
$data['title'] = trim($title->text(), ' :');
$data['link'] = $link ? $link->attr('href') : null;
$about = $node->filter('font i')->first();
$data['annotation'] = trim($about->html());
break;
case 'dl':
$data['pages'][] = $this->groupPage($node);
break;
case 'h3':
$done = true;
}
});
return $data;
}
/**
* @param \Symfony\Component\DomCrawler\Crawler $node
* @param integer $amount
*/
public function seeNodeNumChildren($node, $amount, $filter = null)
{
$count = $filter !== null ? $node->filter($filter)->count() : $node->children()->count();
$this->assertEquals($amount, $count);
}
public function indexGames()
{
$crawler = new Crawler();
//--------------------extract all product details----------------------
global $products;
$products = array();
$html = file_get_contents("http://www.lelong.com.my/toys-and-games/game-console/");
$crawler->addContent($html);
//------------------extract retailer logo------------------------
//$retailer_logo = $crawler->filter('div#top1Logo img')->attr('src');
//---------------------------------------------------------------
//---------------------------filter category-------------------------------
$category = $crawler->filter('a[href="/toys-and-games/game-console/"]')->text();
//-------------------------------------------------------------------------
$crawler->filter('div.item4inline')->each(function ($crawler) {
for ($i = 2; $i <= 5;) {
$url = 'http://www.lelong.com.my/toys-and-games/game-console/?D=' . $i;
$html = file_get_contents($url);
$crawler->addContent($html);
global $products;
global $rank;
$rank = $crawler->filter('span.catalogTitle')->each(function ($crawler, $i) use(&$products) {
$products[$i]['title'] = $crawler->text();
$products[$i]['url'] = str_replace('//', '', $crawler->parents()->attr('href'));
});
$rank = $crawler->filter('div.catalogPrice b')->each(function ($crawler, $i) use(&$products) {
$toReplace = array('RM', ',');
$with = array('', '');
$products[$i]['price'] = str_replace($toReplace, $with, $crawler->text());
});
$rank = $crawler->filter('div.catalog-wrap')->each(function ($crawler, $i) use(&$products) {
$products[$i]['image'] = $crawler->parents()->attr('id');
});
$rank = $crawler->filter('div.catalogIcon')->each(function ($crawler, $i) use(&$products) {
$products[$i]['shipping'] = $crawler->children()->text();
});
++$rank;
$i++;
//print_r($products);
}
});
//--------------insert data using model-----------------
foreach ($products as $pro) {
$product = new Products();
if ($category == 'Game Console') {
$product->category_id = 6;
$product->condition_id = 3;
}
$arrProduct = explode(' ', $pro['title']);
$brands = \DB::table('brand')->whereIn('brand_title', $arrProduct)->get();
if ($brands) {
foreach ($brands as $brand) {
$product->brand_id = $brand->id;
}
} else {
$product->brand_id = 204;
}
$product->product_name = $pro['title'];
$product->shopper_link = $pro['url'];
$product->product_price = $pro['price'];
$product->picture_link = $pro['image'];
$product->product_shipping = $pro['shipping'];
$product->save();
}
//-------------------------------------------------------
}
/**
* Get the selected value from a select field.
*
* @param \Symfony\Component\DomCrawler\Crawler $field
* @return array
*
* @throws \Exception
*/
protected function getSelectedValueFromSelect(Crawler $field)
{
if ($field->nodeName() !== 'select') {
throw new Exception('Given element is not a select element.');
}
$selected = [];
foreach ($field->children() as $option) {
if ($option->nodeName === 'optgroup') {
foreach ($option->childNodes as $child) {
if ($child->hasAttribute('selected')) {
$selected[] = $child->getAttribute('value');
}
}
} elseif ($option->hasAttribute('selected')) {
$selected[] = $option->getAttribute('value');
}
}
return $selected;
}
/**
* @param Crawler $blocks
* @return array
*/
public function lookupStructure(Crawler $blocks)
{
$structure = array();
foreach ($blocks as $block) {
$block = new Crawler($block);
$children = $block->children();
$children_structure = null;
if ($children->getNode(0) != null) {
$children_structure = $this->lookupStructure($children);
}
$structure[] = ['tag' => $block->nodeName(), 'children' => $children_structure];
}
return $structure;
}
/**
* Get the price value from a pricePerUnit node
*
* @param Crawler $node
* @return string
*/
protected function getPrice(Crawler $node)
{
$children = '';
foreach ($node->children() as $child) {
$children .= $child->nodeValue;
}
$text = $node->text();
$price = mb_substr($text, 0, mb_strpos($text, $children));
// remove /unit
$price = trim($price, "£ \t\n\r\v");
// remove whitespace and pound @TODO handle different currencies
return $price;
}
public function indexGames()
{
$crawler = new Crawler();
//-------------extract all product details------------------
global $products;
$products = array();
$html = file_get_contents("http://www.mudah.my/Malaysia/Games-and-Consoles-for-sale-3120?lst=0&fs=1&cg=3120&w=3&so=1&st=s");
$crawler->addContent($html);
//------------------filter category------------------------
$category = $crawler->filter('a[title="See all ads in Games & Consoles category"] span')->text();
//----------------------------------------------------------
$crawler->filter('div.listing_thumbs')->each(function ($crawler) {
for ($i = 2; $i <= 5;) {
$url = 'http://www.mudah.my/Malaysia/Games-and-Consoles-for-sale-3120?o=' . $i . '&q=&so=1&th=1';
$html = file_get_contents($url);
$crawler->addContent($html);
global $rank;
global $products;
$rank = $crawler->filter('h2.list_title.truncate a')->each(function ($crawler, $i) use(&$products) {
$products[$i]['title'] = $crawler->text();
$products[$i]['url'] = $crawler->attr('href');
});
$rank = $crawler->filter('div.ads_price')->each(function ($crawler, $i) use(&$products) {
$toReplace = array(' ', 'RM');
$with = array('', '');
$products[$i]['price'] = str_replace($toReplace, $with, $crawler->text());
});
$rank = $crawler->filter('li.listing_thumbs_image img')->each(function ($crawler, $i) use(&$products) {
$products[$i]['image'] = $crawler->attr('src');
});
$rank = $crawler->filter('div.location')->each(function ($crawler, $i) use(&$products) {
$products[$i]['location'] = str_replace(' ', '', $crawler->children()->siblings()->text());
});
$rank = $crawler->filter('div[title="Condition"]')->each(function ($crawler, $i) use(&$products) {
$products[$i]['condition'] = $crawler->text();
});
++$rank;
$i++;
//print_r($products);
}
});
//--------------insert data using model-----------------
foreach ($products as $pro) {
$product = new Products();
if ($category == 'Games & Consoles') {
$product->category_id = 6;
}
$arrProduct = explode(' ', $pro['title']);
$brands = \DB::table('brand')->whereIn('brand_title', $arrProduct)->get();
if ($brands) {
foreach ($brands as $brand) {
$product->brand_id = $brand->id;
}
} else {
$product->brand_id = 1;
}
$product->product_name = $pro['title'];
$product->shopper_link = $pro['url'];
$product->product_price = $pro['price'];
$product->picture_link = str_replace('thumbs', 'images', $pro['image']);
$product->product_location = $pro['location'];
if ($pro['condition'] == 'New') {
$product->condition_id = 1;
} else {
$product->condition_id = 2;
}
$product->save();
}
//--------------------------------------------------------
return "<div class='alert alert-success'>Successfully crawler site</div>";
}
