Returns the parents nodes of the current selection.
|
public parents ( ) : |
||
| 리턴 | A Crawler instance with the parents nodes of the current selection | |
/**
* get itemprop given value && set the couple in Card
*
* if isItemref is set to true, then only itemprop who have no direct itemscope parent in node will be recorded
*
* @param Crawler $node
* @param iCard $card
* @param boolean $isItemref
* @return boolean
*/
private function setCardProperty(Crawler $node, CardInterface $card, $isItemref = false)
{
$property = $node->attr('itemprop');
$nd = $node;
if ($node->parents()->attr('itemscope') !== null) {
return false;
}
if ($this->nestedScope($node, $card, $property, $isItemref)) {
return true;
}
if ($this->manageRawProperty($node, $property, $card)) {
return true;
}
if ($this->manageImgProperty($node, $property, $card)) {
return true;
}
if ($this->manageLinkProperty($node, $property, $card)) {
return true;
}
if ($this->manageNumericProperty($node, $property, $card)) {
return true;
}
if ($isItemref) {
while (count($nd)) {
if ($nd->attr('itemscope') !== null) {
return true;
}
$nd = $nd->parents();
}
}
$card->{$property} = trim($node->text());
}
/**
* @param \Symfony\Component\DomCrawler\Crawler $node
*
* @return \Symfony\Component\DomCrawler\Form|\Symfony\Component\DomCrawler\Field\ChoiceFormField[]|\Symfony\Component\DomCrawler\Field\FileFormField[]
*/
protected function getFormFor($node)
{
$form = $node->parents()->filter('form')->first();
if (!$form) {
$this->fail('The selected node does not have a form ancestor.');
}
$action = $this->getFormUrl($form);
if (isset($this->forms[$action])) {
return $this->forms[$action];
}
/** @var \DOMElement $autoSubmit */
$autoSubmit = new \DOMElement('input');
$autoSubmit = $form->current()->appendChild($autoSubmit);
$autoSubmit->setAttribute('type', 'submit');
// for forms with no submits
$autoSubmit->setAttribute('name', 'codeception_added_auto_submit');
// Symfony2.1 DOM component requires name for each field.
$formSubmits = $form->filter('*[type=submit]');
$values = null;
// If there are more than one submit (+1 auto_added) in one form we should add value of actually clicked one
if ($formSubmits->count() > 2) {
$nodeItem = $node->getNode(0);
foreach ($formSubmits as $formSubmit) {
if ($formSubmit === $nodeItem) {
$values = array($nodeItem->getAttribute('name') => $nodeItem->getAttribute('value'));
break;
}
}
}
$form = $formSubmits->form($values);
$this->forms[$action] = $form;
return $this->forms[$action];
}
/**
* Returns the DomCrawler\Form object for the form pointed to by
* $node or its closes form parent.
*
* @param \Symfony\Component\DomCrawler\Crawler $node
* @return \Symfony\Component\DomCrawler\Form
*/
protected function getFormFor(Crawler $node)
{
if (strcasecmp($node->first()->getNode(0)->tagName, 'form') === 0) {
$form = $node->first();
} else {
$form = $node->parents()->filter('form')->first();
}
if (!$form) {
$this->fail('The selected node is not a form and does not have a form ancestor.');
}
$action = (string) $this->getFormUrl($form);
if (!isset($this->forms[$action])) {
$this->forms[$action] = $this->getFormFromCrawler($form, $action);
}
return $this->forms[$action];
}
/**
* @param Crawler $block
* @return array
*/
public function getSimilarSiblingsInfo(Crawler $block)
{
$all_siblings = $block->parents()->first()->children();
$siblings = [$block];
$sample_node = $block->getNode(0);
$sample_tag = $block->nodeName();
$sample_classes = explode(' ', $block->extract(['class'])[0]);
foreach ($all_siblings as $sibling) {
if ($sample_node->isSameNode($sibling)) {
continue;
}
$to_add = false;
if ($sibling->nodeName == $sample_tag) {
if ($sample_classes === FALSE) {
$to_add = true;
} else {
$sibling_classes = explode(' ', $sibling->getAttribute('class'));
$matched_classes = array_intersect($sample_classes, $sibling_classes);
if (!empty($matched_classes)) {
if ($matched_classes != $sample_classes) {
$sample_classes = $matched_classes;
}
$to_add = true;
}
}
}
if ($to_add) {
$siblings[] = new Crawler($sibling);
}
}
return [$siblings, ['tag' => $sample_tag, 'classes' => $sample_classes]];
}
/**
* @param \Symfony\Component\DomCrawler\Crawler $node
*
* @return \Symfony\Component\DomCrawler\Form|\Symfony\Component\DomCrawler\Field\ChoiceFormField[]|\Symfony\Component\DomCrawler\Field\FileFormField[]
*/
protected function getFormFor($node)
{
$form = $node->parents()->filter('form')->first();
if (!$form) {
$this->fail('The selected node does not have a form ancestor.');
}
$action = $this->getFormUrl($form);
if (isset($this->forms[$action])) {
return $this->forms[$action];
}
$formSubmits = $form->filter('*[type=submit]');
// Inject a submit button if there isn't one.
if ($formSubmits->count() == 0) {
$autoSubmit = new \DOMElement('input');
$form->rewind();
$autoSubmit = $form->current()->appendChild($autoSubmit);
$autoSubmit->setAttribute('type', 'submit');
// for forms with no submits
$autoSubmit->setAttribute('name', 'codeception_added_auto_submit');
}
// Retrieve the store the Form object.
$this->forms[$action] = $form->form();
return $this->forms[$action];
}
public function indexGames()
{
$crawler = new Crawler();
//--------------------extract all product details----------------------
global $products;
$products = array();
$html = file_get_contents("http://www.lelong.com.my/toys-and-games/game-console/");
$crawler->addContent($html);
//------------------extract retailer logo------------------------
//$retailer_logo = $crawler->filter('div#top1Logo img')->attr('src');
//---------------------------------------------------------------
//---------------------------filter category-------------------------------
$category = $crawler->filter('a[href="/toys-and-games/game-console/"]')->text();
//-------------------------------------------------------------------------
$crawler->filter('div.item4inline')->each(function ($crawler) {
for ($i = 2; $i <= 5;) {
$url = 'http://www.lelong.com.my/toys-and-games/game-console/?D=' . $i;
$html = file_get_contents($url);
$crawler->addContent($html);
global $products;
global $rank;
$rank = $crawler->filter('span.catalogTitle')->each(function ($crawler, $i) use(&$products) {
$products[$i]['title'] = $crawler->text();
$products[$i]['url'] = str_replace('//', '', $crawler->parents()->attr('href'));
});
$rank = $crawler->filter('div.catalogPrice b')->each(function ($crawler, $i) use(&$products) {
$toReplace = array('RM', ',');
$with = array('', '');
$products[$i]['price'] = str_replace($toReplace, $with, $crawler->text());
});
$rank = $crawler->filter('div.catalog-wrap')->each(function ($crawler, $i) use(&$products) {
$products[$i]['image'] = $crawler->parents()->attr('id');
});
$rank = $crawler->filter('div.catalogIcon')->each(function ($crawler, $i) use(&$products) {
$products[$i]['shipping'] = $crawler->children()->text();
});
++$rank;
$i++;
//print_r($products);
}
});
//--------------insert data using model-----------------
foreach ($products as $pro) {
$product = new Products();
if ($category == 'Game Console') {
$product->category_id = 6;
$product->condition_id = 3;
}
$arrProduct = explode(' ', $pro['title']);
$brands = \DB::table('brand')->whereIn('brand_title', $arrProduct)->get();
if ($brands) {
foreach ($brands as $brand) {
$product->brand_id = $brand->id;
}
} else {
$product->brand_id = 204;
}
$product->product_name = $pro['title'];
$product->shopper_link = $pro['url'];
$product->product_price = $pro['price'];
$product->picture_link = $pro['image'];
$product->product_shipping = $pro['shipping'];
$product->save();
}
//-------------------------------------------------------
}
private function fillAnnonceRepo($manager, &$annonces)
{
$numeroPage = 1;
$bddUpToDate = false;
$count = count($annonces);
$initRepo = false;
if ($count == 0) {
$initRepo = true;
}
$countNew = 0;
$newAnnonces = array();
// Atualise ou rempli pour la premiere fois la BDD
while (!$bddUpToDate) {
// Url contenant les informations a récuperer
$lbcUrl = "http://www.leboncoin.fr/animaux/offres/rhone_alpes/?o=" . $numeroPage;
$content = file_get_contents($lbcUrl);
$crawler = new Crawler($content);
$classListLbc = "list-lbc";
// Parsing de chaque div contenant chaque annonce
foreach ($crawler->filter('.lbc') as $lbcAnnonce) {
$tmpCrawler = new Crawler($lbcAnnonce);
$infoNeeded = array("title" => "", "placement" => "", "price" => "", "date" => "");
foreach ($infoNeeded as $key => $value) {
$tmpNode = $tmpCrawler->filter('.' . $key);
if (count($tmpNode) > 0) {
$infoNeeded[$key] = $tmpCrawler->filter('.' . $key)->text();
}
}
$lbcUrl = $tmpCrawler->parents()->attr("href");
$newAnnonce = new Annonce($infoNeeded['title'], $infoNeeded['placement'], $infoNeeded['price'], $infoNeeded['date'], $lbcUrl, $this->getImage($tmpCrawler));
// Si actualisation de la BDD, verification de l'existence ou non de l'annonce, si oui, fin de la boucle
if (!$initRepo) {
foreach ($annonces as $annonce) {
if ($annonce->Compare($newAnnonce)) {
$bddUpToDate = true;
break;
}
}
}
if ($bddUpToDate == true) {
break;
}
// Sinnon stockage de l'annonce
$newAnnonces[] = $newAnnonce;
$count++;
$countNew++;
if ($countNew > 100) {
$bddUpToDate = true;
break;
}
$manager->persist($newAnnonce);
}
$numeroPage++;
}
// Si actualisation de la BDD, vérification du nombre d'article, si celui-ci est supérieur a 100 suppresion des plus anciens
if ($initRepo == false) {
$annonces = array_merge($newAnnonces, $annonces);
$count = count($annonces);
if ($count > 100) {
$count = 100;
while (isset($annonces[$count])) {
$manager->remove($annonces[$count++]);
}
}
}
}
