public function crawlPage($url, $depth)
{
// limit the URL's to 100
if (count($this->_seen) >= $this->_limitUrls) {
return;
}
if (!$this->isValid($url, $depth)) {
return;
}
// add to the seen URL
$this->_seen[$url] = true;
// get content an return code
// $req = $this->executeCurl($url);
$request = new AsyncWebRequest($url);
if ($request->start()) {
while ($request->isRunning()) {
usleep(30);
}
if ($request->join()) {
if ($request->response) {
$links = $this->processLinks($request->response, $url, $depth);
foreach ($links as $link) {
if (count($this->_seen) >= $this->_limitUrls) {
continue;
}
if (!$this->isValid($link, $depth)) {
continue;
}
$this->_seen[$link] = true;
$rq = new AsyncWebRequest($link);
if ($rq->start()) {
while ($rq->isRunning()) {
usleep(30);
}
if ($rq->join()) {
if ($rq->response) {
$ls = $this->processLinks($rq->response, $link, $depth);
foreach ($ls as $l) {
if (count($this->_seen) >= $this->_limitUrls) {
continue;
}
if (!$this->isValid($l, $depth)) {
continue;
}
$this->_seen[$l] = true;
}
}
}
}
}
}
}
}
// if($req->code == 200)
// $this->processLinks($req->data,$url,$depth);
}
$this->data = null;
}
public function run()
{
if ($this->url) {
/*
* If a large amount of data is being requested, you might want to
* fsockopen and read using usleep in between reads
*/
$this->data = file_get_contents($this->url);
} else {
printf("Thread #%lu was not provided a URL\n", $this->getThreadId());
}
return strlen($this->data);
}
}
$t = microtime(true);
$g = new AsyncWebRequest(sprintf("http://www.google.com/?q=%s", rand() * 10));
if ($g->start()) {
printf("Request took %f seconds to start ", microtime(true) - $t);
while ($g->isBusy()) {
echo ".";
usleep(500);
}
if ($g->join()) {
printf(" ... joined ... ");
printf(" and %f seconds to finish receiving %d bytes\n", microtime(true) - $t, strlen($g->data));
} else {
printf(" and %f seconds to finish, request failed\n", microtime(true) - $t);
}
}
$this->url = $url;
}
public function run()
{
if ($this->url) {
/*
* If a large amount of data is being requested, you might want to
* fsockopen and read using usleep in between reads
*/
$this->data = file_get_contents($this->url);
} else {
printf("Thread #%lu was not provided a URL\n", $this->getThreadId());
}
return strlen($this->data);
}
}
$t = microtime(true);
$g = new AsyncWebRequest(sprintf("http://www.google.com/?q=%s", rand() * 10));
/* starting synchronized */
if ($g->start()) {
printf("Request took %f seconds to start ", microtime(true) - $t);
while ($g->isRunning()) {
echo ".";
usleep(100);
}
if ($g->join()) {
printf(" and %f seconds to finish receiving %d bytes\n", microtime(true) - $t, strlen($g->data));
} else {
printf(" and %f seconds to finish, request failed\n", microtime(true) - $t);
}
}
/**
* Permite obtener los datos del taxista usando hilos
* @param array $identificadores arreglo con los identificadores de los taxistas relacionados con el taxi
* @return array arreglo con los datos de los taxistas
*/
private function obtenerDatosTaxistaHilos($identificadores)
{
$urls = [];
for ($i = 0; $i < count($identificadores); $i++) {
$url = "http://datos.labplc.mx/movilidad/taxis/conductor.json?identificador=" . $identificadores[$i]["identificador"];
array_push($urls, $url);
}
$g = new AsyncWebRequest($urls);
$g->start();
return $g->arrayT;
}
