private static function get_from_file() { if (JURI::root(true) !== '/') { return null; } $path = JPATH_ROOT . '/robots.txt'; if (JFile::exists($path)) { $txt = file_get_contents($path); if ($robots_txt !== false) { AimySitemapLogger::debug('Crawl-Init: Got robots.txt from filesystem'); } return $txt; } return null; }
public function crawl_ajax() { JSession::checkToken() or jexit(JText::_('INVALID TOKEN')); $rights = AimySitemapRightsHelper::getRights(); if (!$rights->get('aimysitemap.crawl')) { jexit(JText::_('JLIB_APPLICATION_ERROR_ACCESS_FORBIDDEN')); } header('Content-Type: application/json; charset=UTF-8'); $crawler = new AimySitemapCrawler(); $res = null; try { $res = $crawler->crawl(); } catch (Exception $e) { AimySitemapLogger::error("Crawl-Ctrl: Exception: {$e}"); error_log("{$e}"); echo json_encode(array('ok' => false, 'msg' => $e->getMessage())), "\n"; return JFactory::getApplication()->close(); } if (isset($res['url'])) { AimySitemapLogger::debug('Crawl: done' . (isset($res['msg']) ? ': ' . $res['msg'] : '')); } if (isset($res['abort']) && $res['abort']) { AimySitemapLogger::debug('Crawl: ' . (isset($res['ok']) && $res['ok'] ? 'finished' : 'aborted')); if (isset($res['ok']) && $res['ok']) { if (defined('JDEBUG') && JDEBUG) { $res = array('ok' => 1, 'abort' => 1, 'msg' => JText::_('AIMY_SM_MSG_VIEW_LOG'), 'stats' => array()); } else { require_once JPATH_COMPONENT . DIRECTORY_SEPARATOR . 'helpers' . DIRECTORY_SEPARATOR . 'kvstore.php'; if (AimySitemapKVStore::get('rebuilding')) { $res = array('ok' => 1, 'msg' => JText::_('AIMY_SM_MSG_REBUILDING'), 'rebuilding' => 1); } else { AimySitemapKVStore::set('rebuilding', 1); require_once JPATH_COMPONENT . DIRECTORY_SEPARATOR . 'Sitemap.php'; $sm = new AimySitemapSitemap(); $res['stats'] = $sm->rebuild(); AimySitemapCrawler::delete_crawl_data(); AimySitemapKVStore::delete('rebuilding'); } } } } echo json_encode($res), "\n"; JFactory::getApplication()->close(); }
public static function ping($se) { $se = strtolower($se); if (!isset(self::$ses[$se])) { return false; } $cfg = new AimySitemapConfigHelper(); $sm_url = JUri::root() . $cfg->get('xml_path', '/sitemap.xml'); $req_url = sprintf(self::$ses[$se], urlencode($sm_url)); $u = new AimySitemapUri($req_url); $resp = null; AimySitemapHttpClient::set_ua_name('AimySitemapNotifier/3.16.0'); $resp = AimySitemapHttpClient::get_url($u); if (is_array($resp) && isset($resp['head']) && $resp['head']['code'] == '200') { AimySitemapLogger::debug("Notifier: Sending ping to {$se}: OK"); return true; } AimySitemapLogger::debug("Notifier: Sending ping to {$se}: FAILED ({$req_url})"); return false; }
private static function parse_head($s) { $a = explode("\r\n", trim($s)); $h = array_shift($a); $r = array(); preg_match('#^HTTP/1.\\d (\\d{3}) #', $h, $m); if (count($m) != 2) { AimySitemapLogger::debug("Crawl: HTTP HEAD response: [{$s}]"); AimySitemapLogger::debug("Crawl: parse_head failed: [{$h}]"); throw new RuntimeException('Failed to parse head: status code not found', self::MSG_PARSE_HTTP); } $r['code'] = $m[1]; while (!empty($a)) { $l = array_pop($a); $p = explode(':', $l, 2); if (count($p) != 2) { throw new RuntimeException('Failed to parse head: failed to split fields', self::MSG_PARSE_HTTP); } $r[$p[0]] = trim($p[1]); } return $r; }