function discover_device($device, $options = NULL)
{
global $config, $valid, $exec_status, $discovered_devices;
// Initialise variables
$valid = array();
// Reset $valid array
$modules = array();
$cache_discovery = array();
// Specific discovery cache for exchange snmpwalk data between modules (memory/storage/sensors/etc)
$attribs = get_entity_attribs('device', $device['device_id']);
$device_start = utime();
// Start counting device poll time
// Check if device discovery already running
$pid_info = check_process_run($device);
if ($pid_info) {
// Process ID exist in DB
print_message("%rAnother " . $pid_info['process_name'] . " process (PID: " . $pid_info['PID'] . ", UID: " . $pid_info['UID'] . ", STARTED: " . $pid_info['STARTED'] . ") already running for device " . $device['hostname'] . " (" . $device['device_id'] . ").%n", 'color');
return FALSE;
}
add_process_info($device);
// Store process info
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "]", 1);
$detect_os = TRUE;
// Set TRUE or FALSE for module 'os' (exclude double os detection)
if ($device['os'] == 'generic' || isset($options['h']) && $options['h'] == 'new') {
$detect_os = FALSE;
$old_os = $device['os'];
$device['os'] = get_device_os($device);
if ($device['os'] != $old_os) {
print_cli_data("Device OS changed", $old_os . " -> " . $device['os'], 1);
log_event('OS changed: ' . $old_os . ' -> ' . $device['os'], $device, 'device', $device['device_id'], 'warning');
dbUpdate(array('os' => $device['os']), 'devices', '`device_id` = ?', array($device['device_id']));
}
}
print_cli_data("OS Type", $device['os'], 1);
if ($config['os'][$device['os']]['group']) {
$device['os_group'] = $config['os'][$device['os']]['group'];
print_cli_data("OS Group", $device['os_group'], 1);
}
print_cli_data("SNMP Version", $device['snmp_version'], 1);
print_cli_data("Last discovery", $device['last_discovered'], 1);
print_cli_data("Last duration", $device['last_discovered_timetaken'] . " seconds", 1);
echo PHP_EOL;
// Either only run the modules specified on the commandline, or run all modules in config.
if ($options['m']) {
foreach (explode(",", $options['m']) as $module) {
$modules[$module] = TRUE;
}
} else {
if ($device['force_discovery'] && $options['h'] == 'new' && isset($attribs['force_discovery_modules'])) {
// Forced discovery specific modules
foreach (json_decode($attribs['force_discovery_modules'], TRUE) as $module) {
$modules[$module] = TRUE;
}
log_event('Forced discovery module(s): ' . implode(', ', array_keys($modules)), $device, 'device', $device['device_id'], 'debug');
} else {
$modules = $config['discovery_modules'];
}
}
// Use os specific modules order
//print_vars($modules);
if (isset($config['os'][$device['os']]['discovery_order'])) {
//print_vars($config['os'][$device['os']]['discovery_order']);
foreach ($config['os'][$device['os']]['discovery_order'] as $module => $module_order) {
if (array_key_exists($module, $modules)) {
$module_status = $modules[$module];
switch ($module_order) {
case 'last':
// add to end of modules list
unset($modules[$module]);
$modules[$module] = $module_status;
break;
case 'first':
// add to begin of modules list, but not before os/system
$new_modules = array();
if ($modules['os']) {
$new_modules['os'] = $modules['os'];
unset($modules['os']);
}
if ($modules['system']) {
$new_modules['system'] = $modules['system'];
unset($modules['system']);
}
$new_modules[$module] = $module_status;
unset($modules[$module]);
$modules = $new_modules + $modules;
break;
default:
// add into specific place (after module name in $module_order)
// yes, this is hard and magically
if (array_key_exists($module_order, $modules)) {
unset($modules[$module]);
$new_modules = array();
foreach ($modules as $new_module => $new_status) {
array_shift($modules);
$new_modules[$new_module] = $new_status;
if ($new_module == $module_order) {
$new_modules[$module] = $module_status;
break;
}
}
$modules = array_merge($new_modules, (array) $modules);
}
}
}
}
//print_vars($modules);
}
foreach ($modules as $module => $module_status) {
if (discovery_module_excluded($device, $module) === FALSE) {
if ($attribs['discover_' . $module] || $module_status && !isset($attribs['discover_' . $module])) {
$m_start = utime();
$GLOBALS['module_stats'][$module] = array();
print_cli_heading("Module Start: %R" . $module . "");
include "includes/discovery/{$module}.inc.php";
$m_end = utime();
$GLOBALS['module_stats'][$module]['time'] = round($m_end - $m_start, 4);
print_module_stats($device, $module);
echo PHP_EOL;
//print_cli_heading("Module End: %R".$module."");
} elseif (isset($attribs['discover_' . $module]) && $attribs['discover_' . $module] == "0") {
print_debug("Module [ {$module} ] disabled on host.");
} else {
print_debug("Module [ {$module} ] disabled globally.");
}
}
}
// Set type to a predefined type for the OS if it's not already set
if ($device['type'] == "unknown" || $device['type'] == "") {
if ($config['os'][$device['os']]['type']) {
$device['type'] = $config['os'][$device['os']]['type'];
}
}
$device_end = utime();
$device_run = $device_end - $device_start;
$device_time = substr($device_run, 0, 5);
dbUpdate(array('last_discovered' => array('NOW()'), 'type' => $device['type'], 'last_discovered_timetaken' => $device_time, 'force_discovery' => 0), 'devices', '`device_id` = ?', array($device['device_id']));
if (isset($attribs['force_discovery_modules'])) {
del_entity_attrib('device', $device['device_id'], 'force_discovery_modules');
}
// Put performance into devices_perftimes table
// Not worth putting discovery data into rrd. it's not done every 5 mins :)
dbInsert(array('device_id' => $device['device_id'], 'operation' => 'discover', 'start' => $device_start, 'duration' => $device_run), 'devices_perftimes');
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "] completed discovery modules at " . date("Y-m-d H:i:s"), 1);
print_cli_data("Discovery time", $device_time . " seconds", 1);
echo PHP_EOL;
$discovered_devices++;
// Clean
del_process_info($device);
// Remove process info
unset($cache_discovery);
}
$schema_insert = $db_rev == 0 && !dbQuery('SELECT 1 FROM `devices` LIMIT 1;');
if ($mysql_status) {
echo ' done.' . PHP_EOL;
} else {
echo ' FALSE.' . PHP_EOL;
if (!$schema_insert) {
print_error("Error during installation initial schema, but tables exist. Run update again.");
// Not should happen NEVER
exit(2);
}
}
}
}
if ($db_rev > 272) {
// Check if discovery -u already running
$pid_info = check_process_run(-1);
if ($pid_info) {
// Process ID exist in DB
print_message("%rAnother " . $pid_info['process_name'] . " process (PID: " . $pid_info['PID'] . ", UID: " . $pid_info['UID'] . ", STARTED: " . $pid_info['STARTED'] . ") already running for update.%n", 'color');
// Not sure what better, return and process all other discovery operations
// or complete stop discovery
// (stop can produce 6 hour latency for devices discovery, but return can broke all discovery devices after update)
return FALSE;
//exit(2);
}
add_process_info(-1);
// Store process info
}
$updating = 0;
// Only numeric filenames (001.sql, 013.php)
$sql_regexp = "/^\\d+\\.sql\$/";
function poll_device($device, $options)
{
global $config, $device, $polled_devices, $db_stats, $exec_status, $alert_rules, $alert_table, $graphs, $attribs;
$alert_metrics = array();
$oid_cache = array();
$old_device_state = unserialize($device['device_state']);
$attribs = get_entity_attribs('device', $device['device_id']);
$pid_info = check_process_run($device);
// This just clear stalled DB entries
add_process_info($device);
// Store process info
$alert_rules = cache_alert_rules();
$alert_table = cache_device_alert_table($device['device_id']);
if (OBS_DEBUG > 1 && (count($alert_rules) || count($alert_table))) {
print_vars($alert_rules);
print_vars($alert_table);
}
$status = 0;
$device_start = utime();
// Start counting device poll time
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "]", 1);
print_cli_data("OS", $device['os'], 1);
if ($config['os'][$device['os']]['group']) {
$device['os_group'] = $config['os'][$device['os']]['group'];
print_cli_data("OS Group", $device['os_group'], 1);
}
if (is_numeric($device['last_polled_timetaken'])) {
print_cli_data("Last poll duration", $device['last_polled_timetaken'] . " seconds", 1);
}
print_cli_data("Last Polled", $device['last_polled'], 1);
print_cli_data("SNMP Version", $device['snmp_version'], 1);
//unset($poll_update); unset($poll_update_query); unset($poll_separator);
$update_array = array();
$host_rrd_dir = $config['rrd_dir'] . "/" . $device['hostname'];
if (!is_dir($host_rrd_dir)) {
mkdir($host_rrd_dir);
echo "Created directory : {$host_rrd_dir}\n";
}
$flags = OBS_DNS_ALL;
if ($device['snmp_transport'] == 'udp6' || $device['snmp_transport'] == 'tcp6') {
$flags = $flags ^ OBS_DNS_A;
}
$attribs['ping_skip'] = isset($attribs['ping_skip']) && $attribs['ping_skip'];
if ($attribs['ping_skip']) {
$flags = $flags | OBS_PING_SKIP;
// Add skip ping flag
}
$device['pingable'] = isPingable($device['hostname'], $flags);
if ($device['pingable']) {
$device['snmpable'] = isSNMPable($device);
if ($device['snmpable']) {
$ping_msg = $attribs['ping_skip'] ? '' : 'PING (' . $device['pingable'] . 'ms) and ';
print_cli_data("Device status", "Device is reachable by " . $ping_msg . "SNMP (" . $device['snmpable'] . "ms)", 1);
$status = "1";
$status_type = '';
} else {
print_cli_data("Device status", "Device is not responding to SNMP requests", 1);
$status = "0";
$status_type = 'snmp';
}
} else {
print_cli_data("Device status", "Device is not responding to PINGs", 1);
$status = "0";
$status_type = 'ping';
}
if ($device['status'] != $status) {
dbUpdate(array('status' => $status), 'devices', 'device_id = ?', array($device['device_id']));
// dbInsert(array('importance' => '0', 'device_id' => $device['device_id'], 'message' => "Device is " .($status == '1' ? 'up' : 'down')), 'alerts');
$event_msg = 'Device status changed to ';
if ($status == '1') {
// Device Up, Severity Warning (4)
$event_msg .= 'Up';
$event_severity = 4;
} else {
// Device Down, Severity Error (3)!
$event_msg .= 'Down';
$event_severity = 3;
}
if ($status_type != '') {
$event_msg .= ' (' . $status_type . ')';
}
log_event($event_msg, $device, 'device', $device['device_id'], $event_severity);
}
rrdtool_update_ng($device, 'status', array('status' => $status));
if (!$attribs['ping_skip']) {
// Ping response RRD database.
rrdtool_update_ng($device, 'ping', array('ping' => $device['pingable'] ? $device['pingable'] : 'U'));
}
// SNMP response RRD database.
rrdtool_update_ng($device, 'ping_snmp', array('ping_snmp' => $device['snmpable'] ? $device['snmpable'] : 'U'));
$alert_metrics['device_status'] = $status;
$alert_metrics['device_status_type'] = $status_type;
$alert_metrics['device_ping'] = $device['pingable'];
// FIXME, when ping skipped, here always 0.001
$alert_metrics['device_snmp'] = $device['snmpable'];
if ($status == "1") {
// Arrays for store and check enabled/disabled graphs
$graphs = array();
$graphs_db = array();
foreach (dbFetchRows("SELECT * FROM `device_graphs` WHERE `device_id` = ?", array($device['device_id'])) as $entry) {
$graphs_db[$entry['graph']] = isset($entry['enabled']) ? (bool) $entry['enabled'] : TRUE;
}
if (!$attribs['ping_skip']) {
// Enable Ping graphs
$graphs['ping'] = TRUE;
}
// Enable SNMP graphs
$graphs['ping_snmp'] = TRUE;
// Run these base modules always and before all other modules!
$poll_modules = array('system', 'os');
$mods_disabled_global = array();
$mods_disabled_device = array();
$mods_excluded = array();
if ($options['m']) {
foreach (explode(',', $options['m']) as $module) {
$module = trim($module);
if (in_array($module, $poll_modules)) {
continue;
}
// Skip already added modules
if ($module == 'unix-agent') {
array_unshift($poll_modules, $module);
// Add 'unix-agent' before all
continue;
}
if (is_file($config['install_dir'] . "/includes/polling/{$module}.inc.php")) {
$poll_modules[] = $module;
}
}
} else {
foreach ($config['poller_modules'] as $module => $module_status) {
if (in_array($module, $poll_modules)) {
continue;
}
// Skip already added modules
if ($attribs['poll_' . $module] || $module_status && !isset($attribs['poll_' . $module])) {
if (poller_module_excluded($device, $module)) {
$mods_excluded[] = $module;
//print_warning("Module [ $module ] excluded for device.");
continue;
}
if ($module == 'unix-agent') {
array_unshift($poll_modules, $module);
// Add 'unix-agent' before all
continue;
}
if (is_file($config['install_dir'] . "/includes/polling/{$module}.inc.php")) {
$poll_modules[] = $module;
}
} elseif (isset($attribs['poll_' . $module]) && !$attribs['poll_' . $module]) {
$mods_disabled_device[] = $module;
//print_warning("Module [ $module ] disabled on device.");
} else {
$mods_disabled_global[] = $module;
//print_warning("Module [ $module ] disabled globally.");
}
}
}
if (count($mods_excluded)) {
print_cli_data("Modules Excluded", implode(", ", $mods_excluded), 1);
}
if (count($mods_disabled_global)) {
print_cli_data("Disabled Globally", implode(", ", $mods_disabled_global), 1);
}
if (count($mods_disabled_device)) {
print_cli_data("Disabled Device", implode(", ", $mods_disabled_global), 1);
}
if (count($poll_modules)) {
print_cli_data("Modules Enabled", implode(", ", $poll_modules), 1);
}
echo PHP_EOL;
foreach ($poll_modules as $module) {
print_debug(PHP_EOL . "including: includes/polling/{$module}.inc.php");
print_cli_heading("Module Start: %R" . $module . "");
$m_start = utime();
include $config['install_dir'] . "/includes/polling/{$module}.inc.php";
$m_end = utime();
$m_run = round($m_end - $m_start, 4);
$device_state['poller_mod_perf'][$module] = $m_run;
print_cli_data("Module time", number_format($m_run, 4) . "s");
echo PHP_EOL;
}
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "] completed poller modules at " . date("Y-m-d H:i:s"), 1);
// Check and update graphs DB
$graphs_stat = array();
if (!isset($options['m'])) {
// Hardcoded poller performance
$graphs['poller_perf'] = TRUE;
// Delete not exists graphs from DB (only if poller run without modules option)
foreach ($graphs_db as $graph => $value) {
if (!isset($graphs[$graph])) {
dbDelete('device_graphs', "`device_id` = ? AND `graph` = ?", array($device['device_id'], $graph));
unset($graphs_db[$graph]);
$graphs_stat['deleted'][] = $graph;
}
}
}
// Add or update graphs in DB
foreach ($graphs as $graph => $value) {
if (!isset($graphs_db[$graph])) {
dbInsert(array('device_id' => $device['device_id'], 'graph' => $graph, 'enabled' => $value), 'device_graphs');
$graphs_stat['added'][] = $graph;
} else {
if ($value != $graphs_db[$graph]) {
dbUpdate(array('enabled' => $value), 'device_graphs', '`device_id` = ? AND `graph` = ?', array($device['device_id'], $graph));
$graphs_stat['updated'][] = $graph;
} else {
$graphs_stat['checked'][] = $graph;
}
}
}
// Print graphs stats
foreach ($graphs_stat as $key => $stat) {
if (count($stat)) {
print_cli_data('Graphs [' . $key . ']', implode(', ', $stat), 1);
}
}
$device_end = utime();
$device_run = $device_end - $device_start;
$device_time = round($device_run, 4);
$update_array['last_polled'] = array('NOW()');
$update_array['last_polled_timetaken'] = $device_time;
$update_array['device_state'] = serialize($device_state);
#echo("$device_end - $device_start; $device_time $device_run");
print_cli_data("Poller time", $device_time . " seconds", 1);
//print_message(PHP_EOL."Polled in $device_time seconds");
// Only store performance data if we're not doing a single-module poll
if (!$options['m']) {
dbInsert(array('device_id' => $device['device_id'], 'operation' => 'poll', 'start' => $device_start, 'duration' => $device_run), 'devices_perftimes');
rrdtool_update_ng($device, 'perf-poller', array('val' => $device_time));
}
if (OBS_DEBUG) {
echo "Updating " . $device['hostname'] . " - ";
print_vars($update_array);
echo " \n";
}
$updated = dbUpdate($update_array, 'devices', '`device_id` = ?', array($device['device_id']));
if ($updated) {
print_cli_data("Updated Data", implode(", ", array_keys($update_array)), 1);
}
$alert_metrics['device_uptime'] = $device['uptime'];
$alert_metrics['device_rebooted'] = $rebooted;
// 0 - not rebooted, 1 - rebooted
$alert_metrics['device_duration_poll'] = $device['last_polled_timetaken'];
unset($cache_storage);
// Clear cache of hrStorage ** MAYBE FIXME? ** (ok, later)
unset($cache);
// Clear cache (unify all things here?)
}
check_entity('device', $device, $alert_metrics);
echo PHP_EOL;
// Clean
del_process_info($device);
// Remove process info
unset($alert_metrics);
}
function process_alerts($device)
{
global $config, $alert_rules, $alert_assoc;
$pid_info = check_process_run($device);
// This just clear stalled DB entries
add_process_info($device);
// Store process info
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "]", 1);
$alert_table = cache_device_alert_table($device['device_id']);
$sql = "SELECT * FROM `alert_table`";
//$sql .= " LEFT JOIN `alert_table-state` ON `alert_table`.`alert_table_id` = `alert_table-state`.`alert_table_id`";
$sql .= " WHERE `device_id` = ? AND `alert_status` IS NOT NULL;";
foreach (dbFetchRows($sql, array($device['device_id'])) as $entry) {
print_cli_data_field('Alert: ' . $entry['alert_table_id']);
print_cli('Status: [' . $entry['alert_status'] . '] ', 'color');
// If the alerter is now OK and has previously alerted, send an recovery notice.
if ($entry['alert_status'] == '1' && $entry['has_alerted'] == '1') {
$alert = $alert_rules[$entry['alert_test_id']];
if (!$alert['suppress_recovery']) {
alert_notifier($entry, "recovery");
$log_id = log_alert('Recovery notification sent', $device, $entry, 'RECOVER_NOTIFY');
} else {
echo 'Recovery suppressed.';
$log_id = log_alert('Recovery notification suppressed', $device, $entry, 'RECOVER_SUPPRESSED');
}
$update_array['last_recovered'] = time();
$update_array['has_alerted'] = 0;
dbUpdate($update_array, 'alert_table', '`alert_table_id` = ?', array($entry['alert_table_id']));
}
if ($entry['alert_status'] == '0') {
echo 'Alert tripped. ';
// Has this been alerted more frequently than the alert interval in the config?
/// FIXME -- this should be configurable per-entity or per-checker
if (time() - $entry['last_alerted'] < $config['alerts']['interval'] && !isset($GLOBALS['spam'])) {
$entry['suppress_alert'] = TRUE;
}
// Don't re-alert if interval set to 0
if ($config['alerts']['interval'] == 0 && $entry['last_alerted'] != 0) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until set.
if (is_numeric($entry['ignore_until']) && $entry['ignore_until'] > time()) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until_ok set.
if (is_numeric($entry['ignore_until_ok']) && $entry['ignore_until_ok'] == '1') {
$entry['suppress_alert'] = TRUE;
}
if ($entry['suppress_alert'] != TRUE) {
echo 'Requires notification. ';
alert_notifier($entry, "alert");
$log_id = log_alert('Alert notification sent', $device, $entry, 'ALERT_NOTIFY');
$update_array['last_alerted'] = time();
$update_array['has_alerted'] = 1;
dbUpdate($update_array, 'alert_table', '`alert_table_id` = ?', array($entry['alert_table_id']));
} else {
echo "No notification required. " . (time() - $entry['last_alerted']);
}
} else {
if ($entry['alert_status'] == '1') {
echo "Status: OK. ";
} else {
if ($entry['alert_status'] == '2') {
echo "Status: Notification Delayed. ";
} else {
if ($entry['alert_status'] == '3') {
echo "Status: Notification Suppressed. ";
} else {
echo "Unknown status.";
}
}
}
}
echo PHP_EOL;
}
echo PHP_EOL;
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "] completed notifications at " . date("Y-m-d H:i:s"), 1);
// Clean
del_process_info($device);
// Remove process info
}
