function discover_device($device, $options = NULL) { global $config, $valid, $exec_status, $discovered_devices; // Initialise variables $valid = array(); // Reset $valid array $modules = array(); $cache_discovery = array(); // Specific discovery cache for exchange snmpwalk data between modules (memory/storage/sensors/etc) $attribs = get_entity_attribs('device', $device['device_id']); $device_start = utime(); // Start counting device poll time // Check if device discovery already running $pid_info = check_process_run($device); if ($pid_info) { // Process ID exist in DB print_message("%rAnother " . $pid_info['process_name'] . " process (PID: " . $pid_info['PID'] . ", UID: " . $pid_info['UID'] . ", STARTED: " . $pid_info['STARTED'] . ") already running for device " . $device['hostname'] . " (" . $device['device_id'] . ").%n", 'color'); return FALSE; } add_process_info($device); // Store process info print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "]", 1); $detect_os = TRUE; // Set TRUE or FALSE for module 'os' (exclude double os detection) if ($device['os'] == 'generic' || isset($options['h']) && $options['h'] == 'new') { $detect_os = FALSE; $old_os = $device['os']; $device['os'] = get_device_os($device); if ($device['os'] != $old_os) { print_cli_data("Device OS changed", $old_os . " -> " . $device['os'], 1); log_event('OS changed: ' . $old_os . ' -> ' . $device['os'], $device, 'device', $device['device_id'], 'warning'); dbUpdate(array('os' => $device['os']), 'devices', '`device_id` = ?', array($device['device_id'])); } } print_cli_data("OS Type", $device['os'], 1); if ($config['os'][$device['os']]['group']) { $device['os_group'] = $config['os'][$device['os']]['group']; print_cli_data("OS Group", $device['os_group'], 1); } print_cli_data("SNMP Version", $device['snmp_version'], 1); print_cli_data("Last discovery", $device['last_discovered'], 1); print_cli_data("Last duration", $device['last_discovered_timetaken'] . " seconds", 1); echo PHP_EOL; // Either only run the modules specified on the commandline, or run all modules in config. if ($options['m']) { foreach (explode(",", $options['m']) as $module) { $modules[$module] = TRUE; } } else { if ($device['force_discovery'] && $options['h'] == 'new' && isset($attribs['force_discovery_modules'])) { // Forced discovery specific modules foreach (json_decode($attribs['force_discovery_modules'], TRUE) as $module) { $modules[$module] = TRUE; } log_event('Forced discovery module(s): ' . implode(', ', array_keys($modules)), $device, 'device', $device['device_id'], 'debug'); } else { $modules = $config['discovery_modules']; } } // Use os specific modules order //print_vars($modules); if (isset($config['os'][$device['os']]['discovery_order'])) { //print_vars($config['os'][$device['os']]['discovery_order']); foreach ($config['os'][$device['os']]['discovery_order'] as $module => $module_order) { if (array_key_exists($module, $modules)) { $module_status = $modules[$module]; switch ($module_order) { case 'last': // add to end of modules list unset($modules[$module]); $modules[$module] = $module_status; break; case 'first': // add to begin of modules list, but not before os/system $new_modules = array(); if ($modules['os']) { $new_modules['os'] = $modules['os']; unset($modules['os']); } if ($modules['system']) { $new_modules['system'] = $modules['system']; unset($modules['system']); } $new_modules[$module] = $module_status; unset($modules[$module]); $modules = $new_modules + $modules; break; default: // add into specific place (after module name in $module_order) // yes, this is hard and magically if (array_key_exists($module_order, $modules)) { unset($modules[$module]); $new_modules = array(); foreach ($modules as $new_module => $new_status) { array_shift($modules); $new_modules[$new_module] = $new_status; if ($new_module == $module_order) { $new_modules[$module] = $module_status; break; } } $modules = array_merge($new_modules, (array) $modules); } } } } //print_vars($modules); } foreach ($modules as $module => $module_status) { if (discovery_module_excluded($device, $module) === FALSE) { if ($attribs['discover_' . $module] || $module_status && !isset($attribs['discover_' . $module])) { $m_start = utime(); $GLOBALS['module_stats'][$module] = array(); print_cli_heading("Module Start: %R" . $module . ""); include "includes/discovery/{$module}.inc.php"; $m_end = utime(); $GLOBALS['module_stats'][$module]['time'] = round($m_end - $m_start, 4); print_module_stats($device, $module); echo PHP_EOL; //print_cli_heading("Module End: %R".$module.""); } elseif (isset($attribs['discover_' . $module]) && $attribs['discover_' . $module] == "0") { print_debug("Module [ {$module} ] disabled on host."); } else { print_debug("Module [ {$module} ] disabled globally."); } } } // Set type to a predefined type for the OS if it's not already set if ($device['type'] == "unknown" || $device['type'] == "") { if ($config['os'][$device['os']]['type']) { $device['type'] = $config['os'][$device['os']]['type']; } } $device_end = utime(); $device_run = $device_end - $device_start; $device_time = substr($device_run, 0, 5); dbUpdate(array('last_discovered' => array('NOW()'), 'type' => $device['type'], 'last_discovered_timetaken' => $device_time, 'force_discovery' => 0), 'devices', '`device_id` = ?', array($device['device_id'])); if (isset($attribs['force_discovery_modules'])) { del_entity_attrib('device', $device['device_id'], 'force_discovery_modules'); } // Put performance into devices_perftimes table // Not worth putting discovery data into rrd. it's not done every 5 mins :) dbInsert(array('device_id' => $device['device_id'], 'operation' => 'discover', 'start' => $device_start, 'duration' => $device_run), 'devices_perftimes'); print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "] completed discovery modules at " . date("Y-m-d H:i:s"), 1); print_cli_data("Discovery time", $device_time . " seconds", 1); echo PHP_EOL; $discovered_devices++; // Clean del_process_info($device); // Remove process info unset($cache_discovery); }
} else { echo ' Could not open file!' . PHP_EOL; // Critical errors, stop update logfile('update-errors.log', "====== Schema update " . sprintf("%03d", $db_rev) . " -> " . sprintf("%03d", $filename) . " =============="); logfile('update-errors.log', "Error: Could not open file {$filepath}!"); del_process_info(-1); // Remove process info exit(1); } } } $updating++; $db_rev = $filename; } } if ($updating) { // $GLOBALS['cache']['db_version'] = $db_rev; // Cache new db version // if ($schema_insert) // { // dbInsert(array('attrib_type' => 'dbSchema', 'attrib_value' => $db_rev), 'observium_attribs'); // } else { // dbUpdate(array('attrib_value' => $db_rev), 'observium_attribs', 'attrib_type = ?', array('dbSchema')); // } echo '-- Done.' . PHP_EOL; } else { echo '-- Database is up to date.' . PHP_EOL; } // Clean del_process_info(-1); // Remove process info // EOF
function poll_device($device, $options) { global $config, $device, $polled_devices, $db_stats, $exec_status, $alert_rules, $alert_table, $graphs, $attribs; $alert_metrics = array(); $oid_cache = array(); $old_device_state = unserialize($device['device_state']); $attribs = get_entity_attribs('device', $device['device_id']); $pid_info = check_process_run($device); // This just clear stalled DB entries add_process_info($device); // Store process info $alert_rules = cache_alert_rules(); $alert_table = cache_device_alert_table($device['device_id']); if (OBS_DEBUG > 1 && (count($alert_rules) || count($alert_table))) { print_vars($alert_rules); print_vars($alert_table); } $status = 0; $device_start = utime(); // Start counting device poll time print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "]", 1); print_cli_data("OS", $device['os'], 1); if ($config['os'][$device['os']]['group']) { $device['os_group'] = $config['os'][$device['os']]['group']; print_cli_data("OS Group", $device['os_group'], 1); } if (is_numeric($device['last_polled_timetaken'])) { print_cli_data("Last poll duration", $device['last_polled_timetaken'] . " seconds", 1); } print_cli_data("Last Polled", $device['last_polled'], 1); print_cli_data("SNMP Version", $device['snmp_version'], 1); //unset($poll_update); unset($poll_update_query); unset($poll_separator); $update_array = array(); $host_rrd_dir = $config['rrd_dir'] . "/" . $device['hostname']; if (!is_dir($host_rrd_dir)) { mkdir($host_rrd_dir); echo "Created directory : {$host_rrd_dir}\n"; } $flags = OBS_DNS_ALL; if ($device['snmp_transport'] == 'udp6' || $device['snmp_transport'] == 'tcp6') { $flags = $flags ^ OBS_DNS_A; } $attribs['ping_skip'] = isset($attribs['ping_skip']) && $attribs['ping_skip']; if ($attribs['ping_skip']) { $flags = $flags | OBS_PING_SKIP; // Add skip ping flag } $device['pingable'] = isPingable($device['hostname'], $flags); if ($device['pingable']) { $device['snmpable'] = isSNMPable($device); if ($device['snmpable']) { $ping_msg = $attribs['ping_skip'] ? '' : 'PING (' . $device['pingable'] . 'ms) and '; print_cli_data("Device status", "Device is reachable by " . $ping_msg . "SNMP (" . $device['snmpable'] . "ms)", 1); $status = "1"; $status_type = ''; } else { print_cli_data("Device status", "Device is not responding to SNMP requests", 1); $status = "0"; $status_type = 'snmp'; } } else { print_cli_data("Device status", "Device is not responding to PINGs", 1); $status = "0"; $status_type = 'ping'; } if ($device['status'] != $status) { dbUpdate(array('status' => $status), 'devices', 'device_id = ?', array($device['device_id'])); // dbInsert(array('importance' => '0', 'device_id' => $device['device_id'], 'message' => "Device is " .($status == '1' ? 'up' : 'down')), 'alerts'); $event_msg = 'Device status changed to '; if ($status == '1') { // Device Up, Severity Warning (4) $event_msg .= 'Up'; $event_severity = 4; } else { // Device Down, Severity Error (3)! $event_msg .= 'Down'; $event_severity = 3; } if ($status_type != '') { $event_msg .= ' (' . $status_type . ')'; } log_event($event_msg, $device, 'device', $device['device_id'], $event_severity); } rrdtool_update_ng($device, 'status', array('status' => $status)); if (!$attribs['ping_skip']) { // Ping response RRD database. rrdtool_update_ng($device, 'ping', array('ping' => $device['pingable'] ? $device['pingable'] : 'U')); } // SNMP response RRD database. rrdtool_update_ng($device, 'ping_snmp', array('ping_snmp' => $device['snmpable'] ? $device['snmpable'] : 'U')); $alert_metrics['device_status'] = $status; $alert_metrics['device_status_type'] = $status_type; $alert_metrics['device_ping'] = $device['pingable']; // FIXME, when ping skipped, here always 0.001 $alert_metrics['device_snmp'] = $device['snmpable']; if ($status == "1") { // Arrays for store and check enabled/disabled graphs $graphs = array(); $graphs_db = array(); foreach (dbFetchRows("SELECT * FROM `device_graphs` WHERE `device_id` = ?", array($device['device_id'])) as $entry) { $graphs_db[$entry['graph']] = isset($entry['enabled']) ? (bool) $entry['enabled'] : TRUE; } if (!$attribs['ping_skip']) { // Enable Ping graphs $graphs['ping'] = TRUE; } // Enable SNMP graphs $graphs['ping_snmp'] = TRUE; // Run these base modules always and before all other modules! $poll_modules = array('system', 'os'); $mods_disabled_global = array(); $mods_disabled_device = array(); $mods_excluded = array(); if ($options['m']) { foreach (explode(',', $options['m']) as $module) { $module = trim($module); if (in_array($module, $poll_modules)) { continue; } // Skip already added modules if ($module == 'unix-agent') { array_unshift($poll_modules, $module); // Add 'unix-agent' before all continue; } if (is_file($config['install_dir'] . "/includes/polling/{$module}.inc.php")) { $poll_modules[] = $module; } } } else { foreach ($config['poller_modules'] as $module => $module_status) { if (in_array($module, $poll_modules)) { continue; } // Skip already added modules if ($attribs['poll_' . $module] || $module_status && !isset($attribs['poll_' . $module])) { if (poller_module_excluded($device, $module)) { $mods_excluded[] = $module; //print_warning("Module [ $module ] excluded for device."); continue; } if ($module == 'unix-agent') { array_unshift($poll_modules, $module); // Add 'unix-agent' before all continue; } if (is_file($config['install_dir'] . "/includes/polling/{$module}.inc.php")) { $poll_modules[] = $module; } } elseif (isset($attribs['poll_' . $module]) && !$attribs['poll_' . $module]) { $mods_disabled_device[] = $module; //print_warning("Module [ $module ] disabled on device."); } else { $mods_disabled_global[] = $module; //print_warning("Module [ $module ] disabled globally."); } } } if (count($mods_excluded)) { print_cli_data("Modules Excluded", implode(", ", $mods_excluded), 1); } if (count($mods_disabled_global)) { print_cli_data("Disabled Globally", implode(", ", $mods_disabled_global), 1); } if (count($mods_disabled_device)) { print_cli_data("Disabled Device", implode(", ", $mods_disabled_global), 1); } if (count($poll_modules)) { print_cli_data("Modules Enabled", implode(", ", $poll_modules), 1); } echo PHP_EOL; foreach ($poll_modules as $module) { print_debug(PHP_EOL . "including: includes/polling/{$module}.inc.php"); print_cli_heading("Module Start: %R" . $module . ""); $m_start = utime(); include $config['install_dir'] . "/includes/polling/{$module}.inc.php"; $m_end = utime(); $m_run = round($m_end - $m_start, 4); $device_state['poller_mod_perf'][$module] = $m_run; print_cli_data("Module time", number_format($m_run, 4) . "s"); echo PHP_EOL; } print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "] completed poller modules at " . date("Y-m-d H:i:s"), 1); // Check and update graphs DB $graphs_stat = array(); if (!isset($options['m'])) { // Hardcoded poller performance $graphs['poller_perf'] = TRUE; // Delete not exists graphs from DB (only if poller run without modules option) foreach ($graphs_db as $graph => $value) { if (!isset($graphs[$graph])) { dbDelete('device_graphs', "`device_id` = ? AND `graph` = ?", array($device['device_id'], $graph)); unset($graphs_db[$graph]); $graphs_stat['deleted'][] = $graph; } } } // Add or update graphs in DB foreach ($graphs as $graph => $value) { if (!isset($graphs_db[$graph])) { dbInsert(array('device_id' => $device['device_id'], 'graph' => $graph, 'enabled' => $value), 'device_graphs'); $graphs_stat['added'][] = $graph; } else { if ($value != $graphs_db[$graph]) { dbUpdate(array('enabled' => $value), 'device_graphs', '`device_id` = ? AND `graph` = ?', array($device['device_id'], $graph)); $graphs_stat['updated'][] = $graph; } else { $graphs_stat['checked'][] = $graph; } } } // Print graphs stats foreach ($graphs_stat as $key => $stat) { if (count($stat)) { print_cli_data('Graphs [' . $key . ']', implode(', ', $stat), 1); } } $device_end = utime(); $device_run = $device_end - $device_start; $device_time = round($device_run, 4); $update_array['last_polled'] = array('NOW()'); $update_array['last_polled_timetaken'] = $device_time; $update_array['device_state'] = serialize($device_state); #echo("$device_end - $device_start; $device_time $device_run"); print_cli_data("Poller time", $device_time . " seconds", 1); //print_message(PHP_EOL."Polled in $device_time seconds"); // Only store performance data if we're not doing a single-module poll if (!$options['m']) { dbInsert(array('device_id' => $device['device_id'], 'operation' => 'poll', 'start' => $device_start, 'duration' => $device_run), 'devices_perftimes'); rrdtool_update_ng($device, 'perf-poller', array('val' => $device_time)); } if (OBS_DEBUG) { echo "Updating " . $device['hostname'] . " - "; print_vars($update_array); echo " \n"; } $updated = dbUpdate($update_array, 'devices', '`device_id` = ?', array($device['device_id'])); if ($updated) { print_cli_data("Updated Data", implode(", ", array_keys($update_array)), 1); } $alert_metrics['device_uptime'] = $device['uptime']; $alert_metrics['device_rebooted'] = $rebooted; // 0 - not rebooted, 1 - rebooted $alert_metrics['device_duration_poll'] = $device['last_polled_timetaken']; unset($cache_storage); // Clear cache of hrStorage ** MAYBE FIXME? ** (ok, later) unset($cache); // Clear cache (unify all things here?) } check_entity('device', $device, $alert_metrics); echo PHP_EOL; // Clean del_process_info($device); // Remove process info unset($alert_metrics); }
function process_alerts($device) { global $config, $alert_rules, $alert_assoc; $pid_info = check_process_run($device); // This just clear stalled DB entries add_process_info($device); // Store process info print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "]", 1); $alert_table = cache_device_alert_table($device['device_id']); $sql = "SELECT * FROM `alert_table`"; //$sql .= " LEFT JOIN `alert_table-state` ON `alert_table`.`alert_table_id` = `alert_table-state`.`alert_table_id`"; $sql .= " WHERE `device_id` = ? AND `alert_status` IS NOT NULL;"; foreach (dbFetchRows($sql, array($device['device_id'])) as $entry) { print_cli_data_field('Alert: ' . $entry['alert_table_id']); print_cli('Status: [' . $entry['alert_status'] . '] ', 'color'); // If the alerter is now OK and has previously alerted, send an recovery notice. if ($entry['alert_status'] == '1' && $entry['has_alerted'] == '1') { $alert = $alert_rules[$entry['alert_test_id']]; if (!$alert['suppress_recovery']) { alert_notifier($entry, "recovery"); $log_id = log_alert('Recovery notification sent', $device, $entry, 'RECOVER_NOTIFY'); } else { echo 'Recovery suppressed.'; $log_id = log_alert('Recovery notification suppressed', $device, $entry, 'RECOVER_SUPPRESSED'); } $update_array['last_recovered'] = time(); $update_array['has_alerted'] = 0; dbUpdate($update_array, 'alert_table', '`alert_table_id` = ?', array($entry['alert_table_id'])); } if ($entry['alert_status'] == '0') { echo 'Alert tripped. '; // Has this been alerted more frequently than the alert interval in the config? /// FIXME -- this should be configurable per-entity or per-checker if (time() - $entry['last_alerted'] < $config['alerts']['interval'] && !isset($GLOBALS['spam'])) { $entry['suppress_alert'] = TRUE; } // Don't re-alert if interval set to 0 if ($config['alerts']['interval'] == 0 && $entry['last_alerted'] != 0) { $entry['suppress_alert'] = TRUE; } // Check if alert has ignore_until set. if (is_numeric($entry['ignore_until']) && $entry['ignore_until'] > time()) { $entry['suppress_alert'] = TRUE; } // Check if alert has ignore_until_ok set. if (is_numeric($entry['ignore_until_ok']) && $entry['ignore_until_ok'] == '1') { $entry['suppress_alert'] = TRUE; } if ($entry['suppress_alert'] != TRUE) { echo 'Requires notification. '; alert_notifier($entry, "alert"); $log_id = log_alert('Alert notification sent', $device, $entry, 'ALERT_NOTIFY'); $update_array['last_alerted'] = time(); $update_array['has_alerted'] = 1; dbUpdate($update_array, 'alert_table', '`alert_table_id` = ?', array($entry['alert_table_id'])); } else { echo "No notification required. " . (time() - $entry['last_alerted']); } } else { if ($entry['alert_status'] == '1') { echo "Status: OK. "; } else { if ($entry['alert_status'] == '2') { echo "Status: Notification Delayed. "; } else { if ($entry['alert_status'] == '3') { echo "Status: Notification Suppressed. "; } else { echo "Unknown status."; } } } } echo PHP_EOL; } echo PHP_EOL; print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "] completed notifications at " . date("Y-m-d H:i:s"), 1); // Clean del_process_info($device); // Remove process info }