function thold_check_threshold($rra_id, $data_id, $name, $currentval, $cdef)
{
global $config, $plugins, $debug;
thold_debug("Checking Threshold: DS:{$name} RRA_ID:{$rra_id} DATA_ID:{$data_id} VALUE:{$currentval}");
$debug = false;
// Do not proceed if we have chosen to globally disable all alerts
if (read_config_option('thold_disable_all') == 'on') {
thold_debug('Threshold checking is disabled globally');
return;
}
$alert_exempt = read_config_option('alert_exempt');
/* check for exemptions */
$weekday = date('l');
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* Get all the info about the item from the database */
$item = db_fetch_assoc("SELECT * FROM thold_data WHERE thold_enabled='on' AND data_id=" . $data_id);
/* return if the item doesn't exist, which means its disabled */
if (!isset($item[0])) {
thold_debug('Threshold is disabled');
return;
}
$item = $item[0];
/* check for the weekend exemption on the threshold level */
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $item['exempt'] == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* don't alert for this host if it's selected for maintenance */
if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
include_once $config["base_path"] . '/plugins/maint/functions.php';
if (plugin_maint_check_cacti_host($item['host_id'])) {
thold_debug('Threshold checking is disabled by maintenance schedule');
return;
}
}
$graph_id = $item['graph_id'];
/* only alert if Host is in UP mode (not down, unknown, or recovering) */
$h = db_fetch_row('SELECT * FROM host WHERE id=' . $item['host_id']);
if ($h['status'] != 3) {
thold_debug('Threshold checking halted by Host Status (' . $h['status'] . ')');
return;
}
/* pull the cached name, if not present, it means that the graph hasn't polled yet */
$t = db_fetch_assoc('SELECT id, name, name_cache
FROM data_template_data
WHERE local_data_id = ' . $rra_id . '
ORDER BY id
LIMIT 1');
/* pull a few default settings */
$global_alert_address = read_config_option('alert_email');
$global_notify_enabled = read_config_option('alert_notify_default') == 'on';
$logset = read_config_option('alert_syslog') == 'on';
$deadnotify = read_config_option('alert_deadnotify') == 'on';
$realert = read_config_option('alert_repeat');
$alert_trigger = read_config_option('alert_trigger');
$alert_bl_trigger = read_config_option('alert_bl_trigger');
$httpurl = read_config_option('alert_base_url');
$thold_show_datasource = read_config_option('thold_show_datasource');
$thold_send_text_only = read_config_option('thold_send_text_only');
$thold_alert_text = read_config_option('thold_alert_text');
$thold_warning_text = read_config_option('thold_warning_text');
/* remove this after adding an option for it */
$thold_show_datasource = true;
$trigger = $item['thold_fail_trigger'] == '' ? $alert_trigger : $item['thold_fail_trigger'];
$warning_trigger = $item['thold_warning_fail_trigger'] == '' ? $alert_trigger : $item['thold_warning_fail_trigger'];
$alertstat = $item['thold_alert'];
/* make sure the alert text has been set */
if (!isset($thold_alert_text) || $thold_alert_text == '') {
$thold_alert_text = "<html><body>An alert has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
}
/* make sure the warning text has been set */
if (!isset($thold_warning_text) || $thold_warning_text == '') {
$thold_warning_text = "<html><body>A warning has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
}
$hostname = db_fetch_row('SELECT description, hostname from host WHERE id = ' . $item['host_id']);
$rows = db_fetch_assoc('SELECT plugin_thold_contacts.data
FROM plugin_thold_contacts, plugin_thold_threshold_contact
WHERE plugin_thold_contacts.id=plugin_thold_threshold_contact.contact_id
AND plugin_thold_threshold_contact.thold_id = ' . $item['id']);
$alert_emails = '';
if (read_config_option('thold_disable_legacy') != 'on') {
$alert_emails = array();
if (count($rows)) {
foreach ($rows as $row) {
$alert_emails[] = $row['data'];
}
}
$alert_emails = implode(',', $alert_emails);
if ($alert_emails != '') {
$alert_emails .= ',' . $item['notify_extra'];
} else {
$alert_emails = $item['notify_extra'];
}
}
$alert_emails .= (strlen($alert_emails) ? "," : "") . get_thold_notification_emails($item['notify_alert']);
$alert_phones = '';
if (read_config_option('thold_disable_legacy') != 'on') {
// $alert_phones = array();
// if (count($rows)) {
// foreach ($rows as $row) {
// $alert_phones[] = $row['data'];
// }
// }
// $alert_phones = implode(',', $alert_phones);
if ($alert_phones != '') {
$alert_phones .= ',' . $thold['alert_phones_extra'];
} else {
$alert_phones = $item['alert_phones_extra'];
}
}
$alert_phones .= (strlen($alert_phones) ? "," : "") . get_thold_notification_phones($item['notify_alert']);
$warning_emails = '';
if (read_config_option('thold_disable_legacy') != 'on') {
$warning_emails = $item['notify_warning_extra'];
}
$warning_emails .= (strlen($warning_emails) ? "," : "") . get_thold_notification_emails($item['notify_warning']);
$warning_phones = '';
if (read_config_option('thold_disable_legacy') != 'on') {
$warning_phones = $item['warning_phones_extra'];
}
$alert_command = '';
$alert_command = $item['alert_command'];
thold_debug('Alert Command: ' . $alert_command);
$warning_command = $item['warning_command'];
$types = array('High/Low', 'Baseline Deviation', 'Time Based');
// Do some replacement of variables
$thold_alert_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_alert_text);
$thold_alert_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_alert_text);
$thold_alert_text = str_replace('<TIME>', time(), $thold_alert_text);
$thold_alert_text = str_replace('<GRAPHID>', $graph_id, $thold_alert_text);
$thold_alert_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_alert_text);
$thold_alert_text = str_replace('<CURRENTVALUE>', $currentval, $thold_alert_text);
$thold_alert_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_alert_text);
$thold_alert_text = str_replace('<DSNAME>', $name, $thold_alert_text);
$thold_alert_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_alert_text);
$thold_alert_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_hi'] : ''), $thold_alert_text);
$thold_alert_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_low'] : ''), $thold_alert_text);
$thold_alert_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_fail_trigger'] : ''), $thold_alert_text);
$thold_alert_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_fail_length'], 'time') : '', $thold_alert_text);
$thold_alert_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_alert_text);
$thold_alert_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_alert_text);
// Do some replacement of variables
$thold_warning_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_warning_text);
$thold_warning_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_warning_text);
$thold_warning_text = str_replace('<TIME>', time(), $thold_warning_text);
$thold_warning_text = str_replace('<GRAPHID>', $graph_id, $thold_warning_text);
$thold_warning_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_warning_text);
$thold_warning_text = str_replace('<CURRENTVALUE>', $currentval, $thold_warning_text);
$thold_warning_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_warning_text);
$thold_warning_text = str_replace('<DSNAME>', $name, $thold_warning_text);
$thold_warning_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_warning_text);
$thold_warning_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_warning_hi'] : ''), $thold_warning_text);
$thold_warning_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_warning_low'] : ''), $thold_warning_text);
$thold_warning_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_warning_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_warning_fail_trigger'] : ''), $thold_warning_text);
$thold_warning_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_warning_fail_length'], 'time') : '', $thold_warning_text);
$thold_warning_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_warning_text);
$thold_warning_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_warning_text);
$msg = $thold_alert_text;
$warn_msg = $thold_warning_text;
if ($thold_send_text_only == 'on') {
$file_array = '';
} else {
$file_array = array(0 => array('local_graph_id' => $graph_id, 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id={$graph_id}&rra_id=0&view_type=tree", 'mimetype' => 'image/png', 'filename' => $graph_id));
}
$url = $httpurl . "/graph.php?local_graph_id=" . $graph_id . "&rra_id=all";
switch ($item['thold_type']) {
case 0:
/* hi/low */
if ($currentval != '') {
$breach_up = $item['thold_hi'] != '' && $currentval > $item['thold_hi'];
$breach_down = $item['thold_low'] != '' && $currentval < $item['thold_low'];
$warning_breach_up = $item['thold_warning_hi'] != '' && $currentval > $item['thold_warning_hi'];
$warning_breach_down = $item['thold_warning_low'] != '' && $currentval < $item['thold_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
/* is in alert status */
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold HI / Low check breached HI:' . $item['thold_hi'] . ' LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
$item['thold_fail_count']++;
$item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
/* Re-Alert? */
$ra = $item['thold_fail_count'] > $trigger && $item['repeat_alert'] != 0 && $item['thold_fail_count'] % $item['repeat_alert'] == 0;
if ($item['thold_fail_count'] == $trigger || $ra) {
$notify = true;
}
$subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still ' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $trigger, $item['thold_fail_count'], $url);
}
if ($alert_command != '') {
exec_script($alert_command);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count=" . $item['thold_fail_count'] . ",\r\n\t\t\t\tthold_warning_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
thold_debug('Threshold HI / Low Warning check breached HI:' . $item['thold_warning_hi'] . ' LOW:' . $item['thold_warning_low'] . ' VALUE:' . $currentval);
$item['thold_warning_fail_count']++;
$item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
/* re-alert? */
$ra = $item['thold_warning_fail_count'] > $warning_trigger && $item['repeat_alert'] != 0 && $item['thold_warning_fail_count'] % $item['repeat_alert'] == 0;
if ($item['thold_warning_fail_count'] == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low']) . " with {$currentval}";
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($item['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
}
if ($warning_command != '') {
exec_script($warning_command);
}
if (trim($warning_phones) != '') {
thold_sms($warning_phones, $subject);
}
if (trim($warning_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
} elseif ($item['thold_warning_fail_count'] >= $warning_trigger && $item['thold_fail_count'] >= $trigger) {
$subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Changed to Warning Threshold with Value {$currentval}";
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count=" . $item['thold_warning_fail_count'] . ",\r\n\t\t\t\tthold_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} else {
thold_debug('Threshold HI / Low check is normal HI:' . $item['thold_hi'] . ' LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
/* if we were at an alert status before */
if ($alertstat != 0) {
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Restored to Normal Threshold with Value {$currentval}";
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_fail_count=0, thold_warning_fail_count=0\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
if ($item['thold_warning_fail_count'] >= $warning_trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
}
if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($warning_phones, $subject);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
} elseif ($item['thold_fail_count'] >= $trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
}
if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
}
}
break;
case 1:
/* baseline */
$bl_alert_prev = $item['bl_alert'];
$bl_count_prev = $item['bl_fail_count'];
$bl_fail_trigger = $item['bl_fail_trigger'] == '' ? $alert_bl_trigger : $item['bl_fail_trigger'];
$item['bl_alert'] = thold_check_baseline($rra_id, $name, $currentval, $item);
switch ($item['bl_alert']) {
case -2:
/* exception is active, Future Release 'todo' */
break;
case -1:
/* reference value not available, Future Release 'todo' */
break;
case 0:
/* all clear */
/* if we were at an alert status before */
if ($alertstat != 0) {
thold_debug('Threshold Baseline check is normal');
if ($item['bl_fail_count'] >= $bl_fail_trigger && $item['restored_alert'] != 'on') {
thold_debug('Threshold Baseline check returned to normal');
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
}
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
}
$item['bl_fail_count'] = 0;
break;
case 1:
/* value is below calculated threshold */
/* value is below calculated threshold */
case 2:
/* value is above calculated threshold */
$item['bl_fail_count']++;
$breach_up = $item['bl_alert'] == STAT_HI;
$breach_down = $item['bl_alert'] == STAT_LO;
thold_debug('Threshold Baseline check breached');
/* re-alert? */
$ra = $item['bl_fail_count'] > $bl_fail_trigger && $item['bl_fail_count'] % ($item['repeat_alert'] == '' ? $realert : $item['repeat_alert']) == 0;
if ($item['bl_fail_count'] == $bl_fail_trigger || $ra) {
thold_debug('Alerting is necessary');
$subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . " calculated baseline threshold " . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
if ($logset == 1) {
logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
}
if ($alert_command != '') {
exec_script($alert_command);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
} else {
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
break;
}
db_execute("UPDATE thold_data SET thold_alert=0, thold_fail_count=0,\r\n\t\t\tbl_alert='" . $item['bl_alert'] . "',\r\n\t\t\tbl_fail_count='" . $item['bl_fail_count'] . "',\r\n\t\t\tthold_low='" . $item['thold_low'] . "',\r\n\t\t\tthold_hi='" . $item['thold_hi'] . "',\r\n\t\t\tbl_thold_valid='" . $item['bl_thold_valid'] . "'\r\n\t\t\tWHERE rra_id='{$rra_id}' AND data_id=" . $item['data_id']);
break;
case 2:
/* time based */
if ($currentval != '') {
$breach_up = $item['time_hi'] != '' && $currentval > $item['time_hi'];
$breach_down = $item['time_low'] != '' && $currentval < $item['time_low'];
$warning_breach_up = $item['time_warning_hi'] != '' && $currentval > $item['time_warning_hi'];
$warning_breach_down = $item['time_warning_low'] != '' && $currentval < $item['time_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
$step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $rra_id, FALSE);
/* alerts */
$trigger = $item['time_fail_trigger'];
$time = time() - $item['time_fail_length'] * $step;
$failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_TRIGGERA . "," . ST_NOTIFYRA . "," . ST_NOTIFYAL . ") AND time>" . $time);
/* warnings */
$warning_trigger = $item['time_warning_fail_trigger'];
$warning_time = time() - $item['time_warning_fail_length'] * $step;
$warning_failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_NOTIFYWA . "," . ST_TRIGGERW . ") AND time>" . $warning_time) + $failures;
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold Time Based check breached HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
$item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
$item['thold_fail_count'] = $failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($item['repeat_alert'] - 1) * $step;
$lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYAL . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
$ra = $failures > $trigger && $item['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$failures++;
thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($failures == $trigger || $ra) {
$notify = true;
}
$subject = ($notify ? "ALERT: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['time_hi'] : $item['time_low']) . " with {$currentval}";
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($item['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $item['time_hi'] : $item['time_low'], $currentval, $trigger, $failures, $url);
}
if ($alert_command != '') {
exec_script($alert_command);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
$item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
$item['thold_warning_fail_count'] = $warning_failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($item['time_warning_fail_length'] - 1) * $step;
$lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYWA . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
$ra = $warning_failures > $warning_trigger && $item['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$warning_failures++;
thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($warning_failures == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low']) . " with {$currentval}";
if ($notify) {
if ($logset == 1) {
logger($item['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low'], $currentval, $warning_trigger, $warning_failures, $url);
}
if ($warning_command != '') {
exec_script($warning_command);
}
if (trim($warning_phones) != '') {
thold_sms($warning_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
} elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
$subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count={$warning_failures},\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} else {
thold_debug('Threshold Time Based check is normal HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
if ($alertstat != 0 && $warning_failures < $warning_trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
}
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $msg, $file_array);
}
if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($warning_phones, $subject);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} elseif ($alertstat != 0 && $failures < $trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
}
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} else {
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_fail_count={$failures},\r\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
}
}
break;
}
}
function thold_update_host_status () {
global $config;
// Return if we aren't set to notify
$deadnotify = (read_config_option('alert_deadnotify') == 'on');
if (!$deadnotify) return 0;
include_once($config['base_path'] . '/plugins/thold/thold_functions.php');
if (api_plugin_is_enabled('maint')) {
include_once($config["base_path"] . '/plugins/maint/functions.php');
}
$alert_email = read_config_option('alert_email');
$ping_failure_count = read_config_option('ping_failure_count');
// Lets find hosts that were down, but are now back up
$failed = db_fetch_assoc("SELECT * FROM plugin_thold_host_failed");
if (sizeof($failed)) {
foreach($failed as $fh) {
if (!empty($fh['host_id'])) {
if (api_plugin_is_enabled('maint')) {
if (plugin_maint_check_cacti_host ($fh['host_id'])) {
continue;
}
}
$host = db_fetch_row('SELECT * FROM host WHERE id = ' . $fh['host_id']);
if ($host['status'] == HOST_UP) {
$snmp_system = '';
$snmp_hostname = '';
$snmp_location = '';
$snmp_contact = '';
$snmp_uptime = '';
$uptimelong = '';
$downtimemsg = '';
if (($host["snmp_community"] == "" && $host["snmp_username"] == "") || $host["snmp_version"] == 0) {
// SNMP not in use
$snmp_system = 'SNMP not in use';
} else {
$snmp_system = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.1.0", $host["snmp_version"],
$host["snmp_username"], $host["snmp_password"],
$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);
if (substr_count($snmp_system, "00:")) {
$snmp_system = str_replace("00:", "", $snmp_system);
$snmp_system = str_replace(":", " ", $snmp_system);
}
if ($snmp_system != "") {
$snmp_uptime = cacti_snmp_get($host['hostname'], $host['snmp_community'], ".1.3.6.1.2.1.1.3.0", $host['snmp_version'],
$host['snmp_username'], $host['snmp_password'],
$host['snmp_auth_protocol'], $host['snmp_priv_passphrase'], $host['snmp_priv_protocol'],
$host['snmp_context'], $host['snmp_port'], $host['snmp_timeout'], read_config_option("snmp_retries"), SNMP_WEBUI);
$snmp_hostname = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.5.0", $host["snmp_version"],
$host["snmp_username"], $host["snmp_password"],
$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);
$snmp_location = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.6.0", $host["snmp_version"],
$host["snmp_username"], $host["snmp_password"],
$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);
$snmp_contact = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.4.0", $host["snmp_version"],
$host["snmp_username"], $host["snmp_password"],
$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);
$days = intval($snmp_uptime / (60*60*24*100));
$remainder = $snmp_uptime % (60*60*24*100);
$hours = intval($remainder / (60*60*100));
$remainder = $remainder % (60*60*100);
$minutes = intval($remainder / (60*100));
$uptimelong = $days . "d " . $hours . "h " . $minutes . "m";
}
$downtime = time() - strtotime($host['status_fail_date']);
$downtime_days = floor($downtime/86400);
$downtime_hours = floor(($downtime - ($downtime_days * 86400))/3600);
$downtime_minutes = floor(($downtime - ($downtime_days * 86400) - ($downtime_hours * 3600))/60);
$downtime_seconds = $downtime - ($downtime_days * 86400) - ($downtime_hours * 3600) - ($downtime_minutes * 60);
if ($downtime_days > 0 ) {
$downtimemsg = $downtime_days . "d " . $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s ";
} elseif ($downtime_hours > 0 ) {
$downtimemsg = $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s";
} elseif ($downtime_minutes > 0 ) {
$downtimemsg = $downtime_minutes . "m " . $downtime_seconds . "s";
} else {
$downtimemsg = $downtime_seconds . "s ";
}
}
$subject = read_config_option('thold_up_subject');
if ($subject == '') {
$subject = 'Host Notice: <DESCRIPTION> (<HOSTNAME>) returned from DOWN state';
}
$subject = str_replace('<HOSTNAME>', !$snmp_hostname ? $host['hostname'] : $snmp_hostname, $subject);
$subject = str_replace('<DESCRIPTION>', $host['description'], $subject);
$subject = str_replace('<DOWN/UP>', 'UP', $subject);
$subject = strip_tags($subject);
$msg = read_config_option('thold_up_text');
if ($msg == '') {
$msg = '<br>System <DESCRIPTION> (<HOSTNAME>) status: <DOWN/UP><br><br>Current ping response: <CUR_TIME> ms<br>Average system response : <AVG_TIME> ms<br>System availability: <AVAILABILITY><br>Total Checks Since Clear: <TOT_POLL><br>Total Failed Checks: <FAIL_POLL><br>Last Date Checked UP: <LAST_FAIL><br>Host Previously DOWN for: <DOWNTIME><br><br>Snmp Info:<br>Name - <SNMP_HOSTNAME><br>Location - <SNMP_LOCATION><br>Uptime - <UPTIMETEXT> (<UPTIME> ms)<br>System - <SNMP_SYSTEM><br><br>NOTE: <NOTES>';
}
$msg = str_replace('<SUBJECT>', $subject, $msg);
$msg = str_replace('<HOSTNAME>', $host['hostname'], $msg);
$msg = str_replace('<DESCRIPTION>', $host['description'], $msg);
$msg = str_replace('<UPTIME>', $snmp_uptime, $msg);
$msg = str_replace('<UPTIMETEXT>', $uptimelong, $msg);
$msg = str_replace('<DOWNTIME>', $downtimemsg, $msg);
$msg = str_replace('<MESSAGE>', '', $msg);
$msg = str_replace('<DOWN/UP>', 'UP', $msg);
$msg = str_replace('<SNMP_HOSTNAME>', $snmp_hostname, $msg);
$msg = str_replace('<SNMP_LOCATION>', $snmp_location, $msg);
$msg = str_replace('<SNMP_CONTACT>', $snmp_contact, $msg);
$msg = str_replace('<SNMP_SYSTEM>', html_split_string($snmp_system), $msg);
$msg = str_replace('<LAST_FAIL>', $host["status_fail_date"], $msg);
$msg = str_replace('<AVAILABILITY>', round(($host["availability"]), 2) . ' %', $msg);
$msg = str_replace('<TOT_POLL>', $host["total_polls"], $msg);
$msg = str_replace('<FAIL_POLL>', $host["failed_polls"], $msg);
$msg = str_replace('<CUR_TIME>', round(($host["cur_time"]), 2), $msg);
$msg = str_replace('<AVG_TIME>', round(($host["avg_time"]), 2), $msg);
$msg = str_replace('<NOTES>', $host["notes"], $msg);
$msg = str_replace("\n", '<br>', $msg);
switch($host['thold_send_email']) {
case '0': // Disabled
$alert_email = '';
break;
case '1': // Global List
break;
case '2': // Host List Only
$alert_email = get_thold_notification_emails($host['thold_host_email']);
break;
case '3': // Global and Host List
$alert_email = $alert_email . ',' . get_thold_notification_emails($host['thold_host_email']);
break;
}
if ($alert_email == '' && $host['thold_send_email'] > 0) {
cacti_log('THOLD: Can not send a Host Recovering Email since the \'Alert Email\' setting is not set !', true, 'POLLER');
} elseif ($host['thold_send_email'] == '0') {
cacti_log('THOLD: Did not send a Host Recovering Email, disabled per host setting : ' . $host['description'] . ' !', true, 'POLLER');
} elseif ($alert_email != '') {
thold_mail($alert_email, '', $subject, $msg, '');
}
thold_log(array(
"type" => 3,
"time" => time(),
"host_id" => $host['id'],
"graph_id" => 0,
"threshold_id" => 0,
"threshold_value" => 'Up/Down',
"current" => 'Up',
"status" => 5,
"description" => 'Host Notice: '.$host['hostname'].' returned from DOWN state',
"emails" => $alert_email
)
);
}
}
}
}
// Lets find hosts that are down
$hosts = db_fetch_assoc('SELECT *
FROM host
WHERE disabled=""
AND status=' . HOST_DOWN . '
AND status_event_count=' . $ping_failure_count);
$total_hosts = sizeof($hosts);
if (count($hosts)) {
foreach($hosts as $host) {
if (api_plugin_is_enabled('maint')) {
if (plugin_maint_check_cacti_host ($host['id'])) {
continue;
}
}
$downtime = time() - strtotime($host['status_rec_date']);
$downtime_days = floor($downtime/86400);
$downtime_hours = floor(($downtime - ($downtime_days * 86400))/3600);
$downtime_minutes = floor(($downtime - ($downtime_days * 86400) - ($downtime_hours * 3600))/60);
$downtime_seconds = $downtime - ($downtime_days * 86400) - ($downtime_hours * 3600) - ($downtime_minutes * 60);
if ($downtime_days > 0 ) {
$downtimemsg = $downtime_days . "d " . $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s ";
} elseif ($downtime_hours > 0 ) {
$downtimemsg = $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s";
} elseif ($downtime_minutes > 0 ) {
$downtimemsg = $downtime_minutes . "m " . $downtime_seconds . "s";
} else {
$downtimemsg = $downtime_seconds . "s ";
}
/*$subject = read_config_option('thold_down_subject');
if ($subject == '') {
$subject = 'Host Error: <DESCRIPTION> (<HOSTNAME>) is DOWN';
}
$subject = str_replace('<HOSTNAME>', $host['hostname'], $subject); */
$subject = read_config_option('thold_down_subject');
if ($subject == '') {
$subject = 'Host Error: <DESCRIPTION> (<HOSTNAME>) is DOWN';
}
$snmpsys = db_fetch_assoc('SELECT snmp_sysName, snmp_sysLocation, snmp_sysDescr FROM mac_track_devices WHERE hostname = \''.$host['hostname'].'\'');
$subject = str_replace('<HOSTNAME>', !$snmpsys ? $host['hostname'] : $snmpsys[0]['snmp_sysName'], $subject);
$subject = str_replace('<DESCRIPTION>', $host['description'], $subject);
$subject = str_replace('<DOWN/UP>', 'DOWN', $subject);
$subject = strip_tags($subject);
$msg = read_config_option('thold_down_text');
if ($msg == '') {
$msg = 'System Error : <DESCRIPTION> (<HOSTNAME>) is <DOWN/UP><br>Reason: <MESSAGE><SNMP_HOSTNAME><SNMP_LOCATION><SNMP_SYSTEM><br><br>Average system response : <AVG_TIME> ms<br>System availability: <AVAILABILITY><br>Total Checks Since Clear: <TOT_POLL><br>Total Failed Checks: <FAIL_POLL><br>Last Date Checked DOWN : <LAST_FAIL><br>Host Previously UP for: <DOWNTIME><br>NOTE: <NOTES>';
}
$msg = str_replace('<SUBJECT>', $subject, $msg);
$msg = str_replace('<HOSTNAME>', $host['hostname'], $msg);
$msg = str_replace('<DESCRIPTION>', $host['description'], $msg);
$msg = str_replace('<UPTIME>', '', $msg);
$msg = str_replace('<DOWNTIME>', $downtimemsg, $msg);
$msg = str_replace('<MESSAGE>', $host['status_last_error'], $msg);
$msg = str_replace('<DOWN/UP>', 'DOWN', $msg);
$msg = str_replace('<SNMP_HOSTNAME>', !$snmpsys ? '' : '<br><br>Device name: '.$snmpsys[0]['snmp_sysName'], $msg);
$msg = str_replace('<SNMP_LOCATION>', !$snmpsys ? '' : '<br>Location: '.$snmpsys[0]['snmp_sysLocation'], $msg);
$msg = str_replace('<SNMP_CONTACT>', '', $msg);
$msg = str_replace('<SNMP_SYSTEM>', !$snmpsys ? '' : '<br>Description: '.$snmpsys[0]['snmp_sysDescr'], $msg);
$msg = str_replace('<LAST_FAIL>', $host["status_fail_date"], $msg);
$msg = str_replace('<AVAILABILITY>', round(($host["availability"]), 2) . ' %', $msg);
$msg = str_replace('<CUR_TIME>', round(($host["cur_time"]), 2), $msg);
$msg = str_replace('<TOT_POLL>', $host["total_polls"], $msg);
$msg = str_replace('<FAIL_POLL>', $host["failed_polls"], $msg);
$msg = str_replace('<AVG_TIME>', round(($host["avg_time"]), 2), $msg);
$msg = str_replace('<NOTES>', $host["notes"], $msg);
$msg = str_replace("\n", '<br>', $msg);
switch($host['thold_send_email']) {
case '0': // Disabled
$alert_email = '';
break;
case '1': // Global List
break;
case '2': // Host List Only
$alert_email = get_thold_notification_emails($host['thold_host_email']);
break;
case '3': // Global and Host List
$alert_email = $alert_email . ',' . get_thold_notification_emails($host['thold_host_email']);
break;
}
if ($alert_email == '' && $host['thold_send_email'] > 0) {
cacti_log('THOLD: Can not send a Host Down Email since the \'Alert Email\' setting is not set !', true, 'POLLER');
} elseif ($host['thold_send_email'] == '0') {
cacti_log('THOLD: Did not send a Host Down Email, disabled per host setting : ' . $host['description'] . ' !', true, 'POLLER');
} elseif ($alert_email != '') {
thold_mail($alert_email, '', $subject, $msg, '');
}
thold_log(array(
"type" => 3,
"time" => time(),
"host_id" => $host['id'],
"graph_id" => 0,
"threshold_id" => 0,
"threshold_value" => 'Up/Down',
"current" => 'Down',
"status" => 4,
"description" => 'Host Error: '.$host['hostname'].' is DOWN',
"emails" => $alert_email
)
);
}
}
// Now lets record all failed hosts
db_execute("TRUNCATE TABLE plugin_thold_host_failed");
$hosts = db_fetch_assoc('SELECT id
FROM host
WHERE disabled=""
AND status!=' . HOST_UP);
$failed = '';
if (sizeof($hosts)) {
foreach ($hosts as $host) {
if (api_plugin_is_enabled('maint')) {
if (plugin_maint_check_cacti_host ($host['id'])) {
continue;
}
}
$failed .= (strlen($failed) ? "), (":"(") . $host['id'];
}
$failed .= ")";
db_execute("INSERT INTO plugin_thold_host_failed (host_id) VALUES $failed");
}
return $total_hosts;
}
function thold_check_threshold(&$thold_data)
{
global $config, $plugins, $debug, $thold_types;
$name = db_fetch_cell_prepared('SELECT data_source_name FROM data_template_rrd WHERE id = ?', array($thold_data['data_template_rrd_id']));
thold_debug('Checking Threshold:' . ' Name: ' . $name . ', local_data_id: ' . $thold_data['local_data_id'] . ', data_template_rrd_id: ' . $thold_data['data_template_rrd_id'] . ', value: ' . $thold_data['lastread']);
$debug = false;
// Do not proceed if we have chosen to globally disable all alerts
if (read_config_option('thold_disable_all') == 'on') {
thold_debug('Threshold checking is disabled globally');
return;
}
$alert_exempt = read_config_option('alert_exempt');
/* check for exemptions */
$weekday = date('l');
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* check for the weekend exemption on the threshold level */
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $thold_data['exempt'] == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* don't alert for this host if it's selected for maintenance */
if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
include_once $config['base_path'] . '/plugins/maint/functions.php';
if (plugin_maint_check_cacti_host($thold_data['host_id'])) {
thold_debug('Threshold checking is disabled by maintenance schedule');
return;
}
}
$local_graph_id = $thold_data['local_graph_id'];
/* only alert if Device is in UP mode (not down, unknown, or recovering) */
$h = db_fetch_row('SELECT * FROM host WHERE id=' . $thold_data['host_id']);
if (sizeof($h) && $h['status'] != 3) {
thold_debug('Threshold checking halted by Device Status (' . $h['status'] . ')');
return;
}
/* pull the cached name, if not present, it means that the graph hasn't polled yet */
$t = db_fetch_assoc('SELECT id, name, name_cache
FROM data_template_data
WHERE local_data_id = ' . $thold_data['local_data_id'] . '
ORDER BY id
LIMIT 1');
/* pull a few default settings */
$global_alert_address = read_config_option('alert_email');
$global_notify_enabled = read_config_option('alert_notify_default') == 'on';
$logset = read_config_option('alert_syslog') == 'on';
$deadnotify = read_config_option('alert_deadnotify') == 'on';
$realert = read_config_option('alert_repeat');
$alert_trigger = read_config_option('alert_trigger');
$alert_bl_trigger = read_config_option('alert_bl_trigger');
$httpurl = read_config_option('base_url');
$thold_send_text_only = read_config_option('thold_send_text_only');
$thold_snmp_traps = read_config_option('thold_alert_snmp') == 'on';
$thold_snmp_warning_traps = read_config_option('thold_alert_snmp_warning') != 'on';
$thold_snmp_normal_traps = read_config_option('thold_alert_snmp_normal') != 'on';
$cacti_polling_interval = read_config_option('poller_interval');
/* remove this after adding an option for it */
$thold_show_datasource = thold_datasource_required($thold_data['name'], $name);
$trigger = $thold_data['thold_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_fail_trigger'];
$warning_trigger = $thold_data['thold_warning_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_warning_fail_trigger'];
$alertstat = $thold_data['thold_alert'];
$alert_emails = get_thold_alert_emails($thold_data);
$warning_emails = get_thold_warning_emails($thold_data);
$alert_msg = get_thold_alert_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
$warn_msg = get_thold_warning_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
$thold_snmp_data = get_thold_snmp_data($name, $thold_data, $h, $thold_data['lastread']);
$file_array = '';
if ($thold_send_text_only != 'on') {
if (!empty($thold_data['local_graph_id'])) {
$file_array[] = array('local_graph_id' => $thold_data['local_graph_id'], 'local_data_id' => $thold_data['local_data_id'], 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id=" . $thold_data['local_graph_id'] . '&rra_id=0&view_type=tree', 'mimetype' => 'image/png', 'filename' => clean_up_name($thold_data['name']));
}
}
$url = $httpurl . '/graph.php?local_graph_id=' . $thold_data['local_graph_id'] . '&rra_id=all';
switch ($thold_data['thold_type']) {
case 0:
/* hi/low */
if ($thold_data['lastread'] != '') {
$breach_up = $thold_data['thold_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_hi'];
$breach_down = $thold_data['thold_low'] != '' && $thold_data['lastread'] < $thold_data['thold_low'];
$warning_breach_up = $thold_data['thold_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_warning_hi'];
$warning_breach_down = $thold_data['thold_warning_low'] != '' && $thold_data['lastread'] < $thold_data['thold_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
/* is in alert status */
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold HI / Low check breached HI:' . $thold_data['thold_hi'] . ' LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
$thold_data['thold_fail_count']++;
$thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
/* Re-Alert? */
$ra = $thold_data['thold_fail_count'] > $trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_fail_count'] % $thold_data['repeat_alert'] == 0;
if ($thold_data['thold_fail_count'] == $trigger || $ra) {
$notify = true;
}
$subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps) {
$thold_snmp_data['eventClass'] = 3;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['thold_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['thold_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ',
thold_fail_count=' . $thold_data['thold_fail_count'] . ",\n\t\t\t\tthold_warning_fail_count=0\n\t\t\t\tWHERE id=" . $thold_data['id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
thold_debug('Threshold HI / Low Warning check breached HI:' . $thold_data['thold_warning_hi'] . ' LOW:' . $thold_data['thold_warning_low'] . ' VALUE:' . $thold_data['lastread']);
$thold_data['thold_warning_fail_count']++;
$thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
/* re-alert? */
$ra = $thold_data['thold_warning_fail_count'] > $warning_trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_warning_fail_count'] % $thold_data['repeat_alert'] == 0;
if ($thold_data['thold_warning_fail_count'] == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($thold_data['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
}
if (trim($warning_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_warning_traps) {
$thold_snmp_data['eventClass'] = 2;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailDurationTrigger'] = $warning_trigger * $cacti_polling_interval;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
} elseif ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['thold_fail_count'] >= $trigger) {
$subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Changed to Warning Threshold with Value ' . $thold_data['lastread'];
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_warning_traps) {
$thold_snmp_data['eventClass'] = 2;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYAW + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ',
thold_warning_fail_count=' . $thold_data['thold_warning_fail_count'] . ',
thold_fail_count=0
WHERE id=' . $thold_data['id']);
} else {
thold_debug('Threshold HI / Low check is normal HI:' . $thold_data['thold_hi'] . ' LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
/* if we were at an alert status before */
if ($alertstat != 0) {
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Restored to Normal Threshold with Value ' . $thold_data['lastread'];
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_fail_count=0, \n\t\t\t\t\tthold_warning_fail_count=0\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
if ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
}
if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
} elseif ($thold_data['thold_fail_count'] >= $trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
}
if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
}
}
}
break;
case 1:
/* baseline */
$bl_alert_prev = $thold_data['bl_alert'];
$bl_count_prev = $thold_data['bl_fail_count'];
$bl_fail_trigger = $thold_data['bl_fail_trigger'] == '' ? $alert_bl_trigger : $thold_data['bl_fail_trigger'];
$thold_data['bl_alert'] = thold_check_baseline($local_data_id, $name, $thold_data['lastread'], $thold_data);
switch ($thold_data['bl_alert']) {
case -2:
/* exception is active, Future Release 'todo' */
break;
case -1:
/* reference value not available, Future Release 'todo' */
break;
case 0:
/* all clear */
/* if we were at an alert status before */
if ($alertstat != 0) {
thold_debug('Threshold Baseline check is normal');
if ($thold_data['bl_fail_count'] >= $bl_fail_trigger && $thold_data['restored_alert'] != 'on') {
thold_debug('Threshold Baseline check returned to normal');
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
}
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$hold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
}
}
$thold_data['bl_fail_count'] = 0;
break;
case 1:
/* value is below calculated threshold */
/* value is below calculated threshold */
case 2:
/* value is above calculated threshold */
$thold_data['bl_fail_count']++;
$breach_up = $thold_data['bl_alert'] == STAT_HI;
$breach_down = $thold_data['bl_alert'] == STAT_LO;
thold_debug('Threshold Baseline check breached');
/* re-alert? */
$ra = $thold_data['bl_fail_count'] > $bl_fail_trigger && $thold_data['bl_fail_count'] % ($thold_data['repeat_alert'] == '' ? $realert : $thold_data['repeat_alert']) == 0;
if ($thold_data['bl_fail_count'] == $bl_fail_trigger || $ra) {
thold_debug('Alerting is necessary');
$subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' calculated baseline threshold ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
if ($logset == 1) {
logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps) {
$thold_snmp_data['eventClass'] = 3;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['bl_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['bl_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['bl_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailCountTrigger'] = $bl_fail_trigger;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
} else {
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
}
break;
}
db_execute("UPDATE thold_data \n\t\t\tSET thold_alert=0, \n\t\t\tthold_fail_count=0,\n\t\t\tbl_alert='" . $thold_data['bl_alert'] . "',\n\t\t\tbl_fail_count='" . $thold_data['bl_fail_count'] . "',\n\t\t\tthold_low='" . $thold_data['thold_low'] . "',\n\t\t\tthold_hi='" . $thold_data['thold_hi'] . "',\n\t\t\tbl_thold_valid='" . $thold_data['bl_thold_valid'] . "'\n\t\t\tWHERE id=" . $thold_data['id']);
break;
case 2:
/* time based */
if ($thold_data['lastread'] != '') {
$breach_up = $thold_data['time_hi'] != '' && $thold_data['lastread'] > $thold_data['time_hi'];
$breach_down = $thold_data['time_low'] != '' && $thold_data['lastread'] < $thold_data['time_low'];
$warning_breach_up = $thold_data['time_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['time_warning_hi'];
$warning_breach_down = $thold_data['time_warning_low'] != '' && $thold_data['lastread'] < $thold_data['time_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
$step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $local_data_id, FALSE);
/* alerts */
$trigger = $thold_data['time_fail_trigger'];
$time = time() - $thold_data['time_fail_length'] * $step;
$failures = db_fetch_cell('SELECT count(id)
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_TRIGGERA . ',' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ')
AND time>' . $time);
/* warnings */
$warning_trigger = $thold_data['time_warning_fail_trigger'];
$warning_time = time() - $thold_data['time_warning_fail_length'] * $step;
$warning_failures = db_fetch_cell('SELECT count(id)
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_NOTIFYWA . ',' . ST_TRIGGERW . ')
AND time>' . $warning_time) + $failures;
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold Time Based check breached HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
$thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
$thold_data['thold_fail_count'] = $failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($thold_data['repeat_alert'] - 1) * $step;
$lastemailtime = db_fetch_cell('SELECT time
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ')
ORDER BY time DESC
LIMIT 1', FALSE);
$ra = $failures > $trigger && $thold_data['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$failures++;
thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($failures == $trigger || $ra) {
$notify = true;
}
$subject = ($notify ? 'ALERT: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['time_hi'] : $thold_data['time_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($thold_data['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], $thold_data['lastread'], $trigger, $failures, $url);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps) {
$thold_snmp_data['eventClass'] = 3;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA) + 1;
$thold_snmp_data['eventFailCount'] = $failures;
$thold_snmp_data['eventFailCountTrigger'] = $trigger;
$thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
$thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
$thold_data['thold_warning_fail_count'] = $warning_failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($thold_data['time_warning_fail_length'] - 1) * $step;
$lastemailtime = db_fetch_cell('SELECT time
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYWA . ')
ORDER BY time DESC
LIMIT 1', FALSE);
$ra = $warning_failures > $warning_trigger && $thold_data['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$warning_failures++;
thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($warning_failures == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
if ($logset == 1) {
logger($thold_data['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low'], $thold_data['lastread'], $warning_trigger, $warning_failures, $url);
}
if (trim($alert_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_warning_traps) {
$thold_snmp_data['eventClass'] = 2;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
$thold_snmp_data['eventFailCount'] = $warning_failures;
$thold_snmp_data['eventFailCountTrigger'] = $warning_trigger;
$thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
} elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
$subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_warning_fail_count={$warning_failures},\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
} else {
thold_debug('Threshold Time Based check is normal HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
if ($alertstat != 0 && $warning_failures < $warning_trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
}
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
} elseif ($alertstat != 0 && $failures < $trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
}
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
} else {
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_fail_count={$failures},\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
}
}
break;
}
}