$ds = new PovDataService($conf['dsn'], $conf['dbname'], $conf['dbpassword'], array(PDO::ATTR_PERSISTENT => false));
//$lastTimestamp=$ds->getD4CrawlerTimestamp();
$nmDataService = new NodemanagerDataService($conf['d4nmdsn'], $conf['d4nmdbname'], $conf['d4nmdbpassword'], array(PDO::ATTR_PERSISTENT => false));
$forums = $ds->getAllForums();
$logService->log('D4CRAWLER', 'Forums length=' . sizeof($forums), var_log($forums, '$forums'), 'd4crawler');
foreach ($forums as $forum) {
$logService->log('D4CRAWLER', 'Starting forum=' . $forum['forum'], var_log($forum, '$forum'), 'd4crawler');
$forumid = $forum['forumid'];
$categoryid = $forum['categoryid'];
$lastCrawled = $forum['last_crawled'];
$d4ds = $nmDataService->getForumHubDataService($forumid);
$threads = $d4ds->getNewThreads($forumid, $lastCrawled);
$createdat = 0;
foreach ($threads as $t) {
$url = $t['url'];
$exist = $ds->getThreadByUrl($url);
if (!$exist) {
$current_createdat = $t['createdat'];
if ($current_createdat > $createdat) {
$createdat = $current_createdat;
}
libxml_use_internal_errors(true);
$start = time();
$logService->log('D4CRAWLER', 'd4crawler new thread forumid=' . $forumid, var_log($url, '$url'), 'd4crawler');
try {
@crawl($url, '', '', '', '', '', false);
} catch (Exception $x) {
echo $x;
$logService->log('ERROR', 'd4 crawler exception', log_var($x, '$x'), 'd4crawler');
}
} else {
