Fetcher::$spanHosts = isset($opts['span-hosts']) || isset($opts['H']); Fetcher::$quiet = isset($opts['quiet']) || isset($opts['q']); Fetcher::$recursive = isset($opts['recursive']) || isset($opts['r']); Fetcher::$verbose = isset($opts['verbose']) || isset($opts['v']); if (isset($opts['connect-timeout'])) { Fetcher::$connectTimeout = $opts['connect-timeout']; } if (isset($opts['max-redirect'])) { Fetcher::$maxRedirect = $opts['max-redirect']; } if (isset($opts['level']) || isset($opts['l'])) { Fetcher::$maxDepth = isset($opts['level']) ? $opts['level'] : $opts['l']; } if (isset($opts['output-directory']) || isset($opts['o'])) { Fetcher::$outputDirectory = isset($opts['output-directory']) ? $opts['output-directory'] : $opts['o']; } if (isset($opts['timeout']) || isset($opts['T'])) { Fetcher::$timeout = isset($opts['timeout']) ? $opts['timeout'] : $opts['T']; } if (isset($opts['wait']) || isset($opts['w'])) { Fetcher::$wait = isset($opts['wait']) ? $opts['wait'] : $opts['w']; } try { Fetcher::fetch($url); } catch (Exception $e) { if (!Fetcher::$quiet) { fwrite(STDERR, '[error] ' . $e->getMessage() . "\n"); } exit(1); } exit(0);
} try { $actions = array(); foreach ($res->args['actions'] as $action) { if ($action == 'crawl') { $crawler = new Crawler(); $crawler->setShowLinksOnly($res->options['showLinksOnly']); $actions[$action] = $crawler; } else { if ($action == 'index') { $actions[$action] = new Indexer(); } } } $fetcher = new Fetcher(); $retrieved = $fetcher->fetch($url, $actions, $res->options['force']); if ($retrieved === false) { exit(0); } $update = false; foreach ($actions as $key => $action) { Log::info("step: {$key}"); $update |= $action->run($retrieved); } if ($update) { //FIXME: update index if it exists already $fetcher->storeDoc($retrieved->url, $retrieved->esDoc); } else { Log::info("Not updating"); } } catch (\Exception $e) {