/**
* @param GAQuery $pQuery
* @return array
*/
public function retrieveData(GAQuery $pQuery)
{
if (!$this->isAuth()) {
return null;
}
$r = new Request($pQuery->get());
$header[] = 'Authorization: GoogleLogin auth=' . $this->getAuth();
$r->setOption(CURLOPT_SSL_VERIFYPEER, 0);
$r->setOption(CURLOPT_HTTPHEADER, $header);
$r->setOption(CURLOPT_HEADER, false);
return SimpleJSON::decode($r->execute());
}
/**
* Méthode de demande d'authentication pour le service google souhaité
* @return void
*/
private function login()
{
$r = new Request(self::URL_LOGIN);
$r->setOption(CURLOPT_FOLLOWLOCATION, true);
$r->setOption(CURLOPT_SSL_VERIFYPEER, 0);
$data = array('accountType' => $this->account_type, 'Email' => $this->email, 'Passwd' => $this->password, 'source' => self::SOURCE, 'service' => $this->service);
$r->setDataPost($data);
$retour = $r->execute();
$retour = preg_replace("/(\n|\r)/", "", $retour);
$toParse = array("Auth", "LSID", "SID");
foreach ($toParse as $var) {
if (preg_match("/" . $var . "=(.*)\$/", $retour, $extract, PREG_OFFSET_CAPTURE)) {
try {
$this->{$var} = $extract[1][0];
$retour = preg_replace("/" . $var . "=.*\$/", "", $retour);
} catch (Exception $e) {
return;
}
} else {
return;
}
}
}
/**
* @return bool
*/
private function next()
{
$message = "" . count($this->urlsDone) . " done" . PHP_EOL;
$message .= "" . count($this->urlsToCrawl) . " left" . PHP_EOL;
if (count($this->urlsToCrawl) == 0) {
return false;
}
$url = array_shift($this->urlsToCrawl);
if (in_array($url, $this->urlsDone)) {
return true;
}
$message .= "Loading : " . $url . PHP_EOL;
$this->dispatchEvent(new SimpleCrawlerEvent(SimpleCrawlerEvent::OUTPUT, $message));
$r = new Request($url);
try {
$d = $r->execute();
} catch (\Exception $e) {
$d = false;
}
if (!$d) {
$this->urlsDone[] = $url;
$redirect = $r->getRedirectURL();
if ($redirect) {
$this->urlsToCrawl[] = $redirect;
} else {
$this->log($url, $r->getResponseHTTPCode(), $r->getRedirectURL());
}
return true;
}
$baseHref = $this->extract('/\\<base href="([^"]+)"/', $d);
$title = $this->extract('/\\<title\\>([^<]+)/', $d);
$description = $this->extract('/\\<meta name="description" content="([^"]+)"/', $d);
preg_match_all('/href\\="([^"]+)"/', $d, $matches);
if (isset($matches[1]) && !empty($matches[1])) {
foreach ($matches[1] as $u) {
if (strpos($u, 'http://') === 0 || strpos($u, 'https://') === 0 || strpos($u, 'javascript:') === 0 || strpos($u, '#') === 0 || $u === "/") {
continue;
}
if (strpos($u, "/") === 0) {
$u = substr($u, 1, strlen($u));
}
$u = $baseHref . $u;
if ($this->deepRunning && !in_array($u, $this->urlsToCrawl) && !in_array($u, $this->urlsDone)) {
$this->urlsToCrawl[] = $u;
}
}
}
$this->urlsDone[] = $url;
$this->log($url, $title, $description);
return true;
}
/**
* @throws \Exception
*/
public function retrieve()
{
/**
* Check get vars
*/
$need = Core::checkRequiredGetVars("need") ? explode(self::NEED_SEPARATOR, $_GET["need"]) : array();
if (empty($need)) {
$this->output($this->log("No lib to load", "warn"));
}
$needs = array();
$this->calculateNeeds($need, $needs);
$needs = array_unique($needs);
/**
* Get lib contents
*/
foreach ($needs as $lib) {
if (isset($this->manifest[$lib])) {
if (!isset($this->manifest[$lib][$this->type]) || !is_array($this->manifest[$lib][$this->type])) {
$this->output .= $this->log($lib . " is not available", "warn");
continue;
}
$files = $this->manifest[$lib][$this->type];
for ($i = 0, $max = count($files); $i < $max; $i++) {
$absolute_link = preg_match('/^http\\:\\/\\//', $files[$i], $matches);
if (!$absolute_link) {
$files[$i] = dirname(self::MANIFEST) . "/" . $this->configuration["relative"] . $files[$i];
$content = File::read($files[$i]);
self::$current_folder = dirname($files[$i]);
if ($this->type == self::TYPE_CSS) {
$content = preg_replace_callback('/(url\\(\\")([^\\"]+)/', 'core\\tools\\Dependencies::correctUrls', $content);
}
$this->output .= $content . "\r\n";
} else {
$this->output .= Request::load($files[$i]);
}
}
} else {
$this->output .= $this->log($lib . " is not available", "warn");
}
}
/**
* Minified / Uglyflied / gzip
*/
$accept_gzip = preg_match('/gzip/', $_SERVER['HTTP_ACCEPT_ENCODING'], $matches) && !Core::checkRequiredGetVars("output");
if ($accept_gzip) {
Header::content_encoding("gzip");
$this->output = gzencode($this->output);
}
$this->output($this->output);
}
/**
* Exécute un ensemble de requête GET via les méthodes curl_multi_*
* @param string[] $pUrlArr
* @return array
*/
public static function multiLoad($pUrlArr)
{
$requests = array();
foreach ($pUrlArr as $url) {
/** @var Request $r */
$r = new Request($url);
$r->setOption(CURLOPT_RETURNTRANSFER, 1);
$requests[] = $r;
}
$mh = curl_multi_init();
foreach ($requests as $r) {
curl_multi_add_handle($mh, $r->getResource());
}
$active = null;
//execute the handles
do {
$mrc = curl_multi_exec($mh, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active && $mrc == CURLM_OK) {
if (curl_multi_select($mh) != -1) {
do {
$mrc = curl_multi_exec($mh, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
$return = array();
foreach ($requests as $r) {
$return[] = curl_multi_getcontent($r->getResource());
}
foreach ($requests as $r) {
curl_multi_remove_handle($mh, $r->getResource());
}
curl_multi_close($mh);
return $return;
}
public function __construct($pUrl)
{
$this->url = $pUrl;
$this->content = Request::load($pUrl);
$this->parse();
}
