/** * Execute the console command. * * @return mixed */ public function handle() { $client = new \Goutte\Client(); foreach (\MkmScraper\Card::all() as $card) { if ($card->todaysPrice()->count() < 1) { $crawler = $client->request('GET', 'https://www.magiccardmarket.eu/Products/Singles/' . rawurlencode($card->set) . '/' . rawurlencode($card->name)); \Log::info('https://www.magiccardmarket.eu/Products/Singles/' . rawurlencode($card->set) . '/' . rawurlencode($card->name)); $available = 0; $lowfinal = 0; $trendfinal = 0; try { if ($av = $crawler->filter('.sectioncontent .availTable .row_0 .cell_0_1')->first()) { $available = $av->text(); } if ($lo = $crawler->filter('.sectioncontent .availTable .row_1 .cell_1_1 span')->first()) { $low = $lo->text(); $lowfinal = str_replace(",", ".", $low); } if ($tr = $crawler->filter('.sectioncontent .availTable .row_2 .cell_2_1')->first()) { $trend = $tr->text(); $trendnumber = explode(" ", $trend); $trendfinal = str_replace(",", ".", $trendnumber[0]); } } catch (Exception $e) { print $e->getMessage(); } $cardPrice = \MkmScraper\CardPrice::create(array("id_card" => $card->id, "low" => $lowfinal, "trend" => $trendfinal, "sellers" => $available)); } } }
/** * Execute the console command. * * @return mixed */ public function handle() { $filename = "all.arff"; $handle = fopen($filename, 'w+'); $text = "@relation 'price'\n@attribute y {-1,1}\n@attribute x0 numeric\n@attribute x1 numeric\n@attribute x3 numeric\n@attribute x4 numeric\n@attribute x5 numeric\n@attribute x6 numeric\n@attribute x7 numeric\n@attribute x8 numeric\n@attribute x9 numeric\n@attribute x10 numeric\n@attribute x11 numeric\n@data\n\n"; $array = array(array()); foreach (\MkmScraper\Card::where("rarity", "Mythic")->get() as $card) { foreach ($card->graphPrices as $key => $price) { $row = array(); $row[] = $price->priceClass(); $row[] = $price->card->set->daysFromReleaseDate($price->date); $row[] = $price->card->set->daysFromRotationDate($price->date); $row[] = $price->tournamentDiffWeek(); $row[] = $price->tournamentDiffTwoWeek(); $row[] = $price->articlesDiffLastWeek(); $row[] = $price->articlesDiffLastTwoWeek(); $row[] = $price->otherCardMovementDay(); $row[] = $price->otherCardMovementWeek(); $row[] = $price->boostersOpen(); $array[] = $row; } } foreach ($array as $row) { foreach ($row as $item) { $text .= $item . ","; } $text .= "\n"; } file_put_contents($filename, $text); }
/** * Execute the console command. * * @return mixed */ public function handle() { foreach (\MkmScraper\Card::all() as $card) { $product = queryMKMAPI("product/" . $card->id); $articles = queryMKMAPI("articles/" . $card->id); $price = $product->product->priceGuide; $cardPrice = \MkmScraper\CardPrice::create(array("id_card" => $card->id, "sell" => $price->SELL, "low" => $price->LOW, "lowex" => $price->LOWEX, "lowfoil" => $price->LOWFOIL, "avg" => $price->AVG, "trend" => $price->TREND, "sellers" => sizeof($articles->article))); } }
/** * Execute the console command. * * @return mixed */ public function handle() { foreach (\MkmScraper\Set::all() as $set) { $result = queryMKMAPI("expansion/1/" . rawurlencode($set->name)); foreach ($result->card as $card) { $record = \MkmScraper\Card::find($card->idProduct); if (!$record) { \MkmScraper\Card::create(array("id" => $card->idProduct, "name" => $card->name[0]->productName, "id_set" => $set->id, "rarity" => $card->rarity)); } } } }
public function processAll() { $client = new \Goutte\Client(); foreach (\MkmScraper\Event::all() as $event) { $crawler = $client->request('GET', $event->link); $i = 1; $link = $crawler->filter("td>div>div a")->each(function ($node) use($client, $event, &$i) { if (strpos($node->attr('href'), "event") !== false) { $dlcrawler = $client->request('GET', $node->attr('href')); $link = $dlcrawler->filter("table table td .chosen_tr,table table td .hover_tr")->each(function ($node) use($event, $i) { $split = explode(" ", $node->text(), 2); $card = \MkmScraper\Card::where("name", "LIKE", $split[1] . "%")->first(); if ($card) { \MkmScraper\DecklistAppearance::create(array("id_card" => $card->id, "number" => $split[0], "place" => $i, "id_event" => $event->id)); //print $card->id." - ".$node->text()."<br/>"; } }); $i = $i + 1; } }); } }
/** * Execute the console command. * * @return mixed */ public function handle() { $client = new \Goutte\Client(); foreach (\MkmScraper\Card::all() as $card) { if (\MkmScraper\GraphPrice::where("id_card", $card->id)->where("created_at", ">", date('Y-m-d H:i:s', strtotime('-1 week')))->count() < 1) { $crawler = $client->request('GET', 'https://www.magiccardmarket.eu/Products/Singles/' . rawurlencode($card->set) . '/' . rawurlencode($card->name)); $available = $crawler->filter('#ProductInformation script')->first()->text(); $split = explode("chartData =", $available); $split2 = explode(";var ctx", $split[1]); $object = json_decode($split2[0]); $count = 0; foreach ($object->labels as $key => $label) { if (\MkmScraper\GraphPrice::where("id_card", $card->id)->where("date", date_format(date_create_from_format('d.m.y', $label), 'Y-m-d'))->count() < 1) { \MkmScraper\GraphPrice::create(array("id_card" => $card->id, "date" => date_format(date_create_from_format('d.m.y', $label), 'Y-m-d'), "sell" => $object->datasets[0]->data[$key])); $count++; } } if ($count == 0) { \MkmScraper\GraphPrice::create(array("id_card" => $card->id, "date" => "0000-00-00", "sell" => '')); } print $card->name . "\n"; } } }
public function exportAllNew() { $filename = "export.arff"; $handle = fopen($filename, 'w+'); $text = "@relation 'price'\n@attribute y {-1,1}\n@attribute x0 numeric\n@data\n\n"; $array = array(array()); foreach (\MkmScraper\Card::where("rarity", "Mythic")->get() as $card) { $date = "2015-09-03"; while ($date < "2015-12-10") { $row = array(); //$objects['sell'][]=array($dateexp[0], $dateexp[1],$dateexp[2],\MkmScraper\GraphPrice::getPriceWeek($date,$this)); //$objects['art'][]=array($dateexp[0], $dateexp[1],$dateexp[2],\MkmScraper\Article::getArticlesWeek($date,$this)); $articles = \MkmScraper\Article::getDiffWeek(date("Y-m-d", strtotime($date) + 21 * 24 * 60 * 60), $card); if ($articles > 0) { $row[] = \MkmScraper\DecklistAppearance::tournamentDiffLastWeek($date, $card); $row[] = \MkmScraper\GraphPrice::getPriceDiffWeekBoolean($date, $card); $row[] = $articles; $array[] = $row; } //$objects['art'][]=array($dateexp[0], $dateexp[1],$dateexp[2],\MkmScraper\Article::getDiffWeek($date,$this)); $date = date("Y-m-d", strtotime($date) + 7 * 24 * 60 * 60); } } foreach ($array as $row) { foreach ($row as $item) { $text .= $item . ","; } $text .= "\n"; } file_put_contents($filename, $text); return \Response::download($filename, 'export.arff', array('Content-Type' => 'text/aarf')); }
public function showCard($id) { $data['card'] = \MkmScraper\Card::find($id); return view("card", $data); }
Route::get("scrape", function () { $client = new \Goutte\Client(); $crawler = $client->request('GET', 'http://mtgtop8.com/event?e=10488'); $link = $crawler->filter("td>div>div a")->each(function ($node) use($client) { if (strpos($node->attr('href'), "event") !== false) { print "http://mtgtop8.com/" . $node->attr('href') . " - " . $node->text() . "<br>"; $dlcrawler = $client->request('GET', "http://mtgtop8.com/" . $node->attr('href')); $link = $dlcrawler->filter("table table td .chosen_tr,table table td .hover_tr")->each(function ($node) { print "<b>" . $node->text() . "</b><br>"; }); } }); }); Route::get("htmlscrape", function () { $client = new \Goutte\Client(); foreach (\MkmScraper\Card::all() as $card) { $time = microtime(); $crawler = $client->request('GET', 'https://www.magiccardmarket.eu/Products/Singles/' . rawurlencode($card->set) . '/' . rawurlencode($card->name)); $available = $crawler->filter('#ProductInformation script')->first()->text(); $split = explode("chartData =", $available); $split2 = explode(";var ctx", $split[1]); $object = json_decode($split2[0]); foreach ($object->labels as $key => $label) { if (\MkmScraper\GraphPrice::where("id_card", $card->id)->where("date", date_format(date_create_from_format('d.m.y', $label), 'Y-m-d'))->count() < 1) { \MkmScraper\GraphPrice::create(array("id_card" => $card->id, "date" => date_format(date_create_from_format('d.m.y', $label), 'Y-m-d'), "sell" => $object->datasets[0]->data[$key])); } } print $card->name . " - " . (microtime() - $time) . " ms </br>"; } }); Route::get("decklists/wizards", function () {