Fetcher::$spanHosts = isset($opts['span-hosts']) || isset($opts['H']);
Fetcher::$quiet = isset($opts['quiet']) || isset($opts['q']);
Fetcher::$recursive = isset($opts['recursive']) || isset($opts['r']);
Fetcher::$verbose = isset($opts['verbose']) || isset($opts['v']);
if (isset($opts['connect-timeout'])) {
Fetcher::$connectTimeout = $opts['connect-timeout'];
}
if (isset($opts['max-redirect'])) {
Fetcher::$maxRedirect = $opts['max-redirect'];
}
if (isset($opts['level']) || isset($opts['l'])) {
Fetcher::$maxDepth = isset($opts['level']) ? $opts['level'] : $opts['l'];
}
if (isset($opts['output-directory']) || isset($opts['o'])) {
Fetcher::$outputDirectory = isset($opts['output-directory']) ? $opts['output-directory'] : $opts['o'];
}
if (isset($opts['timeout']) || isset($opts['T'])) {
Fetcher::$timeout = isset($opts['timeout']) ? $opts['timeout'] : $opts['T'];
}
if (isset($opts['wait']) || isset($opts['w'])) {
Fetcher::$wait = isset($opts['wait']) ? $opts['wait'] : $opts['w'];
}
try {
Fetcher::fetch($url);
} catch (Exception $e) {
if (!Fetcher::$quiet) {
fwrite(STDERR, '[error] ' . $e->getMessage() . "\n");
}
exit(1);
}
exit(0);
}
try {
$actions = array();
foreach ($res->args['actions'] as $action) {
if ($action == 'crawl') {
$crawler = new Crawler();
$crawler->setShowLinksOnly($res->options['showLinksOnly']);
$actions[$action] = $crawler;
} else {
if ($action == 'index') {
$actions[$action] = new Indexer();
}
}
}
$fetcher = new Fetcher();
$retrieved = $fetcher->fetch($url, $actions, $res->options['force']);
if ($retrieved === false) {
exit(0);
}
$update = false;
foreach ($actions as $key => $action) {
Log::info("step: {$key}");
$update |= $action->run($retrieved);
}
if ($update) {
//FIXME: update index if it exists already
$fetcher->storeDoc($retrieved->url, $retrieved->esDoc);
} else {
Log::info("Not updating");
}
} catch (\Exception $e) {
