public function crawlPage($url, $depth) { // limit the URL's to 100 if (count($this->_seen) >= $this->_limitUrls) { return; } if (!$this->isValid($url, $depth)) { return; } // add to the seen URL $this->_seen[$url] = true; // get content an return code // $req = $this->executeCurl($url); $request = new AsyncWebRequest($url); if ($request->start()) { while ($request->isRunning()) { usleep(30); } if ($request->join()) { if ($request->response) { $links = $this->processLinks($request->response, $url, $depth); foreach ($links as $link) { if (count($this->_seen) >= $this->_limitUrls) { continue; } if (!$this->isValid($link, $depth)) { continue; } $this->_seen[$link] = true; $rq = new AsyncWebRequest($link); if ($rq->start()) { while ($rq->isRunning()) { usleep(30); } if ($rq->join()) { if ($rq->response) { $ls = $this->processLinks($rq->response, $link, $depth); foreach ($ls as $l) { if (count($this->_seen) >= $this->_limitUrls) { continue; } if (!$this->isValid($l, $depth)) { continue; } $this->_seen[$l] = true; } } } } } } } } // if($req->code == 200) // $this->processLinks($req->data,$url,$depth); }
$this->data = null; } public function run() { if ($this->url) { /* * If a large amount of data is being requested, you might want to * fsockopen and read using usleep in between reads */ $this->data = file_get_contents($this->url); } else { printf("Thread #%lu was not provided a URL\n", $this->getThreadId()); } return strlen($this->data); } } $t = microtime(true); $g = new AsyncWebRequest(sprintf("http://www.google.com/?q=%s", rand() * 10)); if ($g->start()) { printf("Request took %f seconds to start ", microtime(true) - $t); while ($g->isBusy()) { echo "."; usleep(500); } if ($g->join()) { printf(" ... joined ... "); printf(" and %f seconds to finish receiving %d bytes\n", microtime(true) - $t, strlen($g->data)); } else { printf(" and %f seconds to finish, request failed\n", microtime(true) - $t); } }
/** * Permite obtener los datos del taxista usando hilos * @param array $identificadores arreglo con los identificadores de los taxistas relacionados con el taxi * @return array arreglo con los datos de los taxistas */ private function obtenerDatosTaxistaHilos($identificadores) { $urls = []; for ($i = 0; $i < count($identificadores); $i++) { $url = "http://datos.labplc.mx/movilidad/taxis/conductor.json?identificador=" . $identificadores[$i]["identificador"]; array_push($urls, $url); } $g = new AsyncWebRequest($urls); $g->start(); return $g->arrayT; }