public function __construct()
{
require_once 'HTTP/Client.php';
// /netcat/require/lib
require_once 'HTTP/Request/Listener.php';
$headers = array('User-Agent' => nc_search::get_setting('CrawlerUserAgent'));
$this->http_client = new HTTP_Client(null, $headers);
$this->http_client->enableHistory(false);
$this->http_client->setMaxRedirects(nc_search::get_setting('CrawlerMaxRedirects'));
$max_doc_size = nc_search::get_setting('CrawlerMaxDocumentSize');
$db = nc_Core::get_object()->db;
$accepted_mime_types = $db->get_col("SELECT DISTINCT `ContentType`\n FROM `Search_Extension`\n WHERE `ExtensionInterface` = 'nc_search_document_parser'");
// $accepted_mime_types = nc_search::load_all('nc_search_extension_rule')
// ->where('extension_interface', 'nc_search_document_parser')
// ->each('get', 'content_type');
// $accepted_mime_types = array_unique($accepted_mime_types);
$listener = new nc_search_indexer_crawler_listener($max_doc_size, $accepted_mime_types);
$this->http_client->attach($listener, true);
}