function thold_check_threshold($rra_id, $data_id, $name, $currentval, $cdef)
{
global $config, $plugins, $debug;
thold_debug("Checking Threshold: DS:{$name} RRA_ID:{$rra_id} DATA_ID:{$data_id} VALUE:{$currentval}");
$debug = false;
// Do not proceed if we have chosen to globally disable all alerts
if (read_config_option('thold_disable_all') == 'on') {
thold_debug('Threshold checking is disabled globally');
return;
}
$alert_exempt = read_config_option('alert_exempt');
/* check for exemptions */
$weekday = date('l');
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* Get all the info about the item from the database */
$item = db_fetch_assoc("SELECT * FROM thold_data WHERE thold_enabled='on' AND data_id=" . $data_id);
/* return if the item doesn't exist, which means its disabled */
if (!isset($item[0])) {
thold_debug('Threshold is disabled');
return;
}
$item = $item[0];
/* check for the weekend exemption on the threshold level */
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $item['exempt'] == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* don't alert for this host if it's selected for maintenance */
if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
include_once $config["base_path"] . '/plugins/maint/functions.php';
if (plugin_maint_check_cacti_host($item['host_id'])) {
thold_debug('Threshold checking is disabled by maintenance schedule');
return;
}
}
$graph_id = $item['graph_id'];
/* only alert if Host is in UP mode (not down, unknown, or recovering) */
$h = db_fetch_row('SELECT * FROM host WHERE id=' . $item['host_id']);
if ($h['status'] != 3) {
thold_debug('Threshold checking halted by Host Status (' . $h['status'] . ')');
return;
}
/* pull the cached name, if not present, it means that the graph hasn't polled yet */
$t = db_fetch_assoc('SELECT id, name, name_cache
FROM data_template_data
WHERE local_data_id = ' . $rra_id . '
ORDER BY id
LIMIT 1');
/* pull a few default settings */
$global_alert_address = read_config_option('alert_email');
$global_notify_enabled = read_config_option('alert_notify_default') == 'on';
$logset = read_config_option('alert_syslog') == 'on';
$deadnotify = read_config_option('alert_deadnotify') == 'on';
$realert = read_config_option('alert_repeat');
$alert_trigger = read_config_option('alert_trigger');
$alert_bl_trigger = read_config_option('alert_bl_trigger');
$httpurl = read_config_option('alert_base_url');
$thold_show_datasource = read_config_option('thold_show_datasource');
$thold_send_text_only = read_config_option('thold_send_text_only');
$thold_alert_text = read_config_option('thold_alert_text');
$thold_warning_text = read_config_option('thold_warning_text');
/* remove this after adding an option for it */
$thold_show_datasource = true;
$trigger = $item['thold_fail_trigger'] == '' ? $alert_trigger : $item['thold_fail_trigger'];
$warning_trigger = $item['thold_warning_fail_trigger'] == '' ? $alert_trigger : $item['thold_warning_fail_trigger'];
$alertstat = $item['thold_alert'];
/* make sure the alert text has been set */
if (!isset($thold_alert_text) || $thold_alert_text == '') {
$thold_alert_text = "<html><body>An alert has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
}
/* make sure the warning text has been set */
if (!isset($thold_warning_text) || $thold_warning_text == '') {
$thold_warning_text = "<html><body>A warning has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
}
$hostname = db_fetch_row('SELECT description, hostname from host WHERE id = ' . $item['host_id']);
$rows = db_fetch_assoc('SELECT plugin_thold_contacts.data
FROM plugin_thold_contacts, plugin_thold_threshold_contact
WHERE plugin_thold_contacts.id=plugin_thold_threshold_contact.contact_id
AND plugin_thold_threshold_contact.thold_id = ' . $item['id']);
$alert_emails = '';
if (read_config_option('thold_disable_legacy') != 'on') {
$alert_emails = array();
if (count($rows)) {
foreach ($rows as $row) {
$alert_emails[] = $row['data'];
}
}
$alert_emails = implode(',', $alert_emails);
if ($alert_emails != '') {
$alert_emails .= ',' . $item['notify_extra'];
} else {
$alert_emails = $item['notify_extra'];
}
}
$alert_emails .= (strlen($alert_emails) ? "," : "") . get_thold_notification_emails($item['notify_alert']);
$alert_phones = '';
if (read_config_option('thold_disable_legacy') != 'on') {
// $alert_phones = array();
// if (count($rows)) {
// foreach ($rows as $row) {
// $alert_phones[] = $row['data'];
// }
// }
// $alert_phones = implode(',', $alert_phones);
if ($alert_phones != '') {
$alert_phones .= ',' . $thold['alert_phones_extra'];
} else {
$alert_phones = $item['alert_phones_extra'];
}
}
$alert_phones .= (strlen($alert_phones) ? "," : "") . get_thold_notification_phones($item['notify_alert']);
$warning_emails = '';
if (read_config_option('thold_disable_legacy') != 'on') {
$warning_emails = $item['notify_warning_extra'];
}
$warning_emails .= (strlen($warning_emails) ? "," : "") . get_thold_notification_emails($item['notify_warning']);
$warning_phones = '';
if (read_config_option('thold_disable_legacy') != 'on') {
$warning_phones = $item['warning_phones_extra'];
}
$alert_command = '';
$alert_command = $item['alert_command'];
thold_debug('Alert Command: ' . $alert_command);
$warning_command = $item['warning_command'];
$types = array('High/Low', 'Baseline Deviation', 'Time Based');
// Do some replacement of variables
$thold_alert_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_alert_text);
$thold_alert_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_alert_text);
$thold_alert_text = str_replace('<TIME>', time(), $thold_alert_text);
$thold_alert_text = str_replace('<GRAPHID>', $graph_id, $thold_alert_text);
$thold_alert_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_alert_text);
$thold_alert_text = str_replace('<CURRENTVALUE>', $currentval, $thold_alert_text);
$thold_alert_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_alert_text);
$thold_alert_text = str_replace('<DSNAME>', $name, $thold_alert_text);
$thold_alert_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_alert_text);
$thold_alert_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_hi'] : ''), $thold_alert_text);
$thold_alert_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_low'] : ''), $thold_alert_text);
$thold_alert_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_fail_trigger'] : ''), $thold_alert_text);
$thold_alert_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_fail_length'], 'time') : '', $thold_alert_text);
$thold_alert_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_alert_text);
$thold_alert_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_alert_text);
// Do some replacement of variables
$thold_warning_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_warning_text);
$thold_warning_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_warning_text);
$thold_warning_text = str_replace('<TIME>', time(), $thold_warning_text);
$thold_warning_text = str_replace('<GRAPHID>', $graph_id, $thold_warning_text);
$thold_warning_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_warning_text);
$thold_warning_text = str_replace('<CURRENTVALUE>', $currentval, $thold_warning_text);
$thold_warning_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_warning_text);
$thold_warning_text = str_replace('<DSNAME>', $name, $thold_warning_text);
$thold_warning_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_warning_text);
$thold_warning_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_warning_hi'] : ''), $thold_warning_text);
$thold_warning_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_warning_low'] : ''), $thold_warning_text);
$thold_warning_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_warning_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_warning_fail_trigger'] : ''), $thold_warning_text);
$thold_warning_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_warning_fail_length'], 'time') : '', $thold_warning_text);
$thold_warning_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_warning_text);
$thold_warning_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_warning_text);
$msg = $thold_alert_text;
$warn_msg = $thold_warning_text;
if ($thold_send_text_only == 'on') {
$file_array = '';
} else {
$file_array = array(0 => array('local_graph_id' => $graph_id, 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id={$graph_id}&rra_id=0&view_type=tree", 'mimetype' => 'image/png', 'filename' => $graph_id));
}
$url = $httpurl . "/graph.php?local_graph_id=" . $graph_id . "&rra_id=all";
switch ($item['thold_type']) {
case 0:
/* hi/low */
if ($currentval != '') {
$breach_up = $item['thold_hi'] != '' && $currentval > $item['thold_hi'];
$breach_down = $item['thold_low'] != '' && $currentval < $item['thold_low'];
$warning_breach_up = $item['thold_warning_hi'] != '' && $currentval > $item['thold_warning_hi'];
$warning_breach_down = $item['thold_warning_low'] != '' && $currentval < $item['thold_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
/* is in alert status */
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold HI / Low check breached HI:' . $item['thold_hi'] . ' LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
$item['thold_fail_count']++;
$item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
/* Re-Alert? */
$ra = $item['thold_fail_count'] > $trigger && $item['repeat_alert'] != 0 && $item['thold_fail_count'] % $item['repeat_alert'] == 0;
if ($item['thold_fail_count'] == $trigger || $ra) {
$notify = true;
}
$subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still ' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $trigger, $item['thold_fail_count'], $url);
}
if ($alert_command != '') {
exec_script($alert_command);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count=" . $item['thold_fail_count'] . ",\r\n\t\t\t\tthold_warning_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
thold_debug('Threshold HI / Low Warning check breached HI:' . $item['thold_warning_hi'] . ' LOW:' . $item['thold_warning_low'] . ' VALUE:' . $currentval);
$item['thold_warning_fail_count']++;
$item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
/* re-alert? */
$ra = $item['thold_warning_fail_count'] > $warning_trigger && $item['repeat_alert'] != 0 && $item['thold_warning_fail_count'] % $item['repeat_alert'] == 0;
if ($item['thold_warning_fail_count'] == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low']) . " with {$currentval}";
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($item['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
}
if ($warning_command != '') {
exec_script($warning_command);
}
if (trim($warning_phones) != '') {
thold_sms($warning_phones, $subject);
}
if (trim($warning_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
} elseif ($item['thold_warning_fail_count'] >= $warning_trigger && $item['thold_fail_count'] >= $trigger) {
$subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Changed to Warning Threshold with Value {$currentval}";
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count=" . $item['thold_warning_fail_count'] . ",\r\n\t\t\t\tthold_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} else {
thold_debug('Threshold HI / Low check is normal HI:' . $item['thold_hi'] . ' LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
/* if we were at an alert status before */
if ($alertstat != 0) {
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Restored to Normal Threshold with Value {$currentval}";
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_fail_count=0, thold_warning_fail_count=0\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
if ($item['thold_warning_fail_count'] >= $warning_trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
}
if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($warning_phones, $subject);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
} elseif ($item['thold_fail_count'] >= $trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
}
if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
}
}
break;
case 1:
/* baseline */
$bl_alert_prev = $item['bl_alert'];
$bl_count_prev = $item['bl_fail_count'];
$bl_fail_trigger = $item['bl_fail_trigger'] == '' ? $alert_bl_trigger : $item['bl_fail_trigger'];
$item['bl_alert'] = thold_check_baseline($rra_id, $name, $currentval, $item);
switch ($item['bl_alert']) {
case -2:
/* exception is active, Future Release 'todo' */
break;
case -1:
/* reference value not available, Future Release 'todo' */
break;
case 0:
/* all clear */
/* if we were at an alert status before */
if ($alertstat != 0) {
thold_debug('Threshold Baseline check is normal');
if ($item['bl_fail_count'] >= $bl_fail_trigger && $item['restored_alert'] != 'on') {
thold_debug('Threshold Baseline check returned to normal');
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
}
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
}
$item['bl_fail_count'] = 0;
break;
case 1:
/* value is below calculated threshold */
/* value is below calculated threshold */
case 2:
/* value is above calculated threshold */
$item['bl_fail_count']++;
$breach_up = $item['bl_alert'] == STAT_HI;
$breach_down = $item['bl_alert'] == STAT_LO;
thold_debug('Threshold Baseline check breached');
/* re-alert? */
$ra = $item['bl_fail_count'] > $bl_fail_trigger && $item['bl_fail_count'] % ($item['repeat_alert'] == '' ? $realert : $item['repeat_alert']) == 0;
if ($item['bl_fail_count'] == $bl_fail_trigger || $ra) {
thold_debug('Alerting is necessary');
$subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . " calculated baseline threshold " . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
if ($logset == 1) {
logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
}
if ($alert_command != '') {
exec_script($alert_command);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
} else {
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
break;
}
db_execute("UPDATE thold_data SET thold_alert=0, thold_fail_count=0,\r\n\t\t\tbl_alert='" . $item['bl_alert'] . "',\r\n\t\t\tbl_fail_count='" . $item['bl_fail_count'] . "',\r\n\t\t\tthold_low='" . $item['thold_low'] . "',\r\n\t\t\tthold_hi='" . $item['thold_hi'] . "',\r\n\t\t\tbl_thold_valid='" . $item['bl_thold_valid'] . "'\r\n\t\t\tWHERE rra_id='{$rra_id}' AND data_id=" . $item['data_id']);
break;
case 2:
/* time based */
if ($currentval != '') {
$breach_up = $item['time_hi'] != '' && $currentval > $item['time_hi'];
$breach_down = $item['time_low'] != '' && $currentval < $item['time_low'];
$warning_breach_up = $item['time_warning_hi'] != '' && $currentval > $item['time_warning_hi'];
$warning_breach_down = $item['time_warning_low'] != '' && $currentval < $item['time_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
$step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $rra_id, FALSE);
/* alerts */
$trigger = $item['time_fail_trigger'];
$time = time() - $item['time_fail_length'] * $step;
$failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_TRIGGERA . "," . ST_NOTIFYRA . "," . ST_NOTIFYAL . ") AND time>" . $time);
/* warnings */
$warning_trigger = $item['time_warning_fail_trigger'];
$warning_time = time() - $item['time_warning_fail_length'] * $step;
$warning_failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_NOTIFYWA . "," . ST_TRIGGERW . ") AND time>" . $warning_time) + $failures;
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold Time Based check breached HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
$item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
$item['thold_fail_count'] = $failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($item['repeat_alert'] - 1) * $step;
$lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYAL . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
$ra = $failures > $trigger && $item['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$failures++;
thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($failures == $trigger || $ra) {
$notify = true;
}
$subject = ($notify ? "ALERT: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['time_hi'] : $item['time_low']) . " with {$currentval}";
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($item['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $item['time_hi'] : $item['time_low'], $currentval, $trigger, $failures, $url);
}
if ($alert_command != '') {
exec_script($alert_command);
}
if (trim($alert_phones) != '') {
thold_sms($alert_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
$item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
$item['thold_warning_fail_count'] = $warning_failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($item['time_warning_fail_length'] - 1) * $step;
$lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYWA . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
$ra = $warning_failures > $warning_trigger && $item['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$warning_failures++;
thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($warning_failures == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low']) . " with {$currentval}";
if ($notify) {
if ($logset == 1) {
logger($item['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low'], $currentval, $warning_trigger, $warning_failures, $url);
}
if ($warning_command != '') {
exec_script($warning_command);
}
if (trim($warning_phones) != '') {
thold_sms($warning_phones, $subject);
}
if (trim($alert_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
} elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
$subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
}
db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count={$warning_failures},\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} else {
thold_debug('Threshold Time Based check is normal HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
if ($alertstat != 0 && $warning_failures < $warning_trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
}
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $msg, $file_array);
}
if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($warning_phones, $subject);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} elseif ($alertstat != 0 && $failures < $trigger && $item['restored_alert'] != 'on') {
if ($logset == 1) {
logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
}
$subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $msg, $file_array);
}
if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
thold_sms($alert_phones, $subject);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
} else {
db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_fail_count={$failures},\r\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
}
}
break;
}
}
function thold_check_threshold(&$thold_data)
{
global $config, $plugins, $debug, $thold_types;
$name = db_fetch_cell_prepared('SELECT data_source_name FROM data_template_rrd WHERE id = ?', array($thold_data['data_template_rrd_id']));
thold_debug('Checking Threshold:' . ' Name: ' . $name . ', local_data_id: ' . $thold_data['local_data_id'] . ', data_template_rrd_id: ' . $thold_data['data_template_rrd_id'] . ', value: ' . $thold_data['lastread']);
$debug = false;
// Do not proceed if we have chosen to globally disable all alerts
if (read_config_option('thold_disable_all') == 'on') {
thold_debug('Threshold checking is disabled globally');
return;
}
$alert_exempt = read_config_option('alert_exempt');
/* check for exemptions */
$weekday = date('l');
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* check for the weekend exemption on the threshold level */
if (($weekday == 'Saturday' || $weekday == 'Sunday') && $thold_data['exempt'] == 'on') {
thold_debug('Threshold checking is disabled by global weekend exemption');
return;
}
/* don't alert for this host if it's selected for maintenance */
if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
include_once $config['base_path'] . '/plugins/maint/functions.php';
if (plugin_maint_check_cacti_host($thold_data['host_id'])) {
thold_debug('Threshold checking is disabled by maintenance schedule');
return;
}
}
$local_graph_id = $thold_data['local_graph_id'];
/* only alert if Device is in UP mode (not down, unknown, or recovering) */
$h = db_fetch_row('SELECT * FROM host WHERE id=' . $thold_data['host_id']);
if (sizeof($h) && $h['status'] != 3) {
thold_debug('Threshold checking halted by Device Status (' . $h['status'] . ')');
return;
}
/* pull the cached name, if not present, it means that the graph hasn't polled yet */
$t = db_fetch_assoc('SELECT id, name, name_cache
FROM data_template_data
WHERE local_data_id = ' . $thold_data['local_data_id'] . '
ORDER BY id
LIMIT 1');
/* pull a few default settings */
$global_alert_address = read_config_option('alert_email');
$global_notify_enabled = read_config_option('alert_notify_default') == 'on';
$logset = read_config_option('alert_syslog') == 'on';
$deadnotify = read_config_option('alert_deadnotify') == 'on';
$realert = read_config_option('alert_repeat');
$alert_trigger = read_config_option('alert_trigger');
$alert_bl_trigger = read_config_option('alert_bl_trigger');
$httpurl = read_config_option('base_url');
$thold_send_text_only = read_config_option('thold_send_text_only');
$thold_snmp_traps = read_config_option('thold_alert_snmp') == 'on';
$thold_snmp_warning_traps = read_config_option('thold_alert_snmp_warning') != 'on';
$thold_snmp_normal_traps = read_config_option('thold_alert_snmp_normal') != 'on';
$cacti_polling_interval = read_config_option('poller_interval');
/* remove this after adding an option for it */
$thold_show_datasource = thold_datasource_required($thold_data['name'], $name);
$trigger = $thold_data['thold_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_fail_trigger'];
$warning_trigger = $thold_data['thold_warning_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_warning_fail_trigger'];
$alertstat = $thold_data['thold_alert'];
$alert_emails = get_thold_alert_emails($thold_data);
$warning_emails = get_thold_warning_emails($thold_data);
$alert_msg = get_thold_alert_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
$warn_msg = get_thold_warning_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
$thold_snmp_data = get_thold_snmp_data($name, $thold_data, $h, $thold_data['lastread']);
$file_array = '';
if ($thold_send_text_only != 'on') {
if (!empty($thold_data['local_graph_id'])) {
$file_array[] = array('local_graph_id' => $thold_data['local_graph_id'], 'local_data_id' => $thold_data['local_data_id'], 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id=" . $thold_data['local_graph_id'] . '&rra_id=0&view_type=tree', 'mimetype' => 'image/png', 'filename' => clean_up_name($thold_data['name']));
}
}
$url = $httpurl . '/graph.php?local_graph_id=' . $thold_data['local_graph_id'] . '&rra_id=all';
switch ($thold_data['thold_type']) {
case 0:
/* hi/low */
if ($thold_data['lastread'] != '') {
$breach_up = $thold_data['thold_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_hi'];
$breach_down = $thold_data['thold_low'] != '' && $thold_data['lastread'] < $thold_data['thold_low'];
$warning_breach_up = $thold_data['thold_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_warning_hi'];
$warning_breach_down = $thold_data['thold_warning_low'] != '' && $thold_data['lastread'] < $thold_data['thold_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
/* is in alert status */
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold HI / Low check breached HI:' . $thold_data['thold_hi'] . ' LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
$thold_data['thold_fail_count']++;
$thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
/* Re-Alert? */
$ra = $thold_data['thold_fail_count'] > $trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_fail_count'] % $thold_data['repeat_alert'] == 0;
if ($thold_data['thold_fail_count'] == $trigger || $ra) {
$notify = true;
}
$subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps) {
$thold_snmp_data['eventClass'] = 3;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['thold_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['thold_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ',
thold_fail_count=' . $thold_data['thold_fail_count'] . ",\n\t\t\t\tthold_warning_fail_count=0\n\t\t\t\tWHERE id=" . $thold_data['id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
thold_debug('Threshold HI / Low Warning check breached HI:' . $thold_data['thold_warning_hi'] . ' LOW:' . $thold_data['thold_warning_low'] . ' VALUE:' . $thold_data['lastread']);
$thold_data['thold_warning_fail_count']++;
$thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
/* re-alert? */
$ra = $thold_data['thold_warning_fail_count'] > $warning_trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_warning_fail_count'] % $thold_data['repeat_alert'] == 0;
if ($thold_data['thold_warning_fail_count'] == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($thold_data['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
}
if (trim($warning_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_warning_traps) {
$thold_snmp_data['eventClass'] = 2;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailDurationTrigger'] = $warning_trigger * $cacti_polling_interval;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
} elseif ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['thold_fail_count'] >= $trigger) {
$subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Changed to Warning Threshold with Value ' . $thold_data['lastread'];
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_warning_traps) {
$thold_snmp_data['eventClass'] = 2;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYAW + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ',
thold_warning_fail_count=' . $thold_data['thold_warning_fail_count'] . ',
thold_fail_count=0
WHERE id=' . $thold_data['id']);
} else {
thold_debug('Threshold HI / Low check is normal HI:' . $thold_data['thold_hi'] . ' LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
/* if we were at an alert status before */
if ($alertstat != 0) {
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Restored to Normal Threshold with Value ' . $thold_data['lastread'];
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_fail_count=0, \n\t\t\t\t\tthold_warning_fail_count=0\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
if ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
}
if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
} elseif ($thold_data['thold_fail_count'] >= $trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
}
if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
}
}
}
break;
case 1:
/* baseline */
$bl_alert_prev = $thold_data['bl_alert'];
$bl_count_prev = $thold_data['bl_fail_count'];
$bl_fail_trigger = $thold_data['bl_fail_trigger'] == '' ? $alert_bl_trigger : $thold_data['bl_fail_trigger'];
$thold_data['bl_alert'] = thold_check_baseline($local_data_id, $name, $thold_data['lastread'], $thold_data);
switch ($thold_data['bl_alert']) {
case -2:
/* exception is active, Future Release 'todo' */
break;
case -1:
/* reference value not available, Future Release 'todo' */
break;
case 0:
/* all clear */
/* if we were at an alert status before */
if ($alertstat != 0) {
thold_debug('Threshold Baseline check is normal');
if ($thold_data['bl_fail_count'] >= $bl_fail_trigger && $thold_data['restored_alert'] != 'on') {
thold_debug('Threshold Baseline check returned to normal');
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
}
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$hold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
}
}
$thold_data['bl_fail_count'] = 0;
break;
case 1:
/* value is below calculated threshold */
/* value is below calculated threshold */
case 2:
/* value is above calculated threshold */
$thold_data['bl_fail_count']++;
$breach_up = $thold_data['bl_alert'] == STAT_HI;
$breach_down = $thold_data['bl_alert'] == STAT_LO;
thold_debug('Threshold Baseline check breached');
/* re-alert? */
$ra = $thold_data['bl_fail_count'] > $bl_fail_trigger && $thold_data['bl_fail_count'] % ($thold_data['repeat_alert'] == '' ? $realert : $thold_data['repeat_alert']) == 0;
if ($thold_data['bl_fail_count'] == $bl_fail_trigger || $ra) {
thold_debug('Alerting is necessary');
$subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' calculated baseline threshold ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
if ($logset == 1) {
logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps) {
$thold_snmp_data['eventClass'] = 3;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['bl_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
$thold_snmp_data['eventFailCount'] = $thold_data['bl_fail_count'];
$thold_snmp_data['eventFailDuration'] = $thold_data['bl_fail_count'] * $cacti_polling_interval;
$thold_snmp_data['eventFailCountTrigger'] = $bl_fail_trigger;
$thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
} else {
thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
}
break;
}
db_execute("UPDATE thold_data \n\t\t\tSET thold_alert=0, \n\t\t\tthold_fail_count=0,\n\t\t\tbl_alert='" . $thold_data['bl_alert'] . "',\n\t\t\tbl_fail_count='" . $thold_data['bl_fail_count'] . "',\n\t\t\tthold_low='" . $thold_data['thold_low'] . "',\n\t\t\tthold_hi='" . $thold_data['thold_hi'] . "',\n\t\t\tbl_thold_valid='" . $thold_data['bl_thold_valid'] . "'\n\t\t\tWHERE id=" . $thold_data['id']);
break;
case 2:
/* time based */
if ($thold_data['lastread'] != '') {
$breach_up = $thold_data['time_hi'] != '' && $thold_data['lastread'] > $thold_data['time_hi'];
$breach_down = $thold_data['time_low'] != '' && $thold_data['lastread'] < $thold_data['time_low'];
$warning_breach_up = $thold_data['time_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['time_warning_hi'];
$warning_breach_down = $thold_data['time_warning_low'] != '' && $thold_data['lastread'] < $thold_data['time_warning_low'];
} else {
$breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
}
$step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $local_data_id, FALSE);
/* alerts */
$trigger = $thold_data['time_fail_trigger'];
$time = time() - $thold_data['time_fail_length'] * $step;
$failures = db_fetch_cell('SELECT count(id)
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_TRIGGERA . ',' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ')
AND time>' . $time);
/* warnings */
$warning_trigger = $thold_data['time_warning_fail_trigger'];
$warning_time = time() - $thold_data['time_warning_fail_length'] * $step;
$warning_failures = db_fetch_cell('SELECT count(id)
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_NOTIFYWA . ',' . ST_TRIGGERW . ')
AND time>' . $warning_time) + $failures;
if ($breach_up || $breach_down) {
$notify = false;
thold_debug('Threshold Time Based check breached HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
$thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
$thold_data['thold_fail_count'] = $failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($thold_data['repeat_alert'] - 1) * $step;
$lastemailtime = db_fetch_cell('SELECT time
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ')
ORDER BY time DESC
LIMIT 1', FALSE);
$ra = $failures > $trigger && $thold_data['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$failures++;
thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($failures == $trigger || $ra) {
$notify = true;
}
$subject = ($notify ? 'ALERT: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['time_hi'] : $thold_data['time_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
thold_debug('Alerting is necessary');
if ($logset == 1) {
logger($thold_data['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], $thold_data['lastread'], $trigger, $failures, $url);
}
if (trim($alert_emails) != '') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps) {
$thold_snmp_data['eventClass'] = 3;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA) + 1;
$thold_snmp_data['eventFailCount'] = $failures;
$thold_snmp_data['eventFailCountTrigger'] = $trigger;
$thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
} elseif ($warning_breach_up || $warning_breach_down) {
$notify = false;
$thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
$thold_data['thold_warning_fail_count'] = $warning_failures;
/* we should only re-alert X minutes after last email, not every 5 pollings, etc...
re-alert? */
$realerttime = ($thold_data['time_warning_fail_length'] - 1) * $step;
$lastemailtime = db_fetch_cell('SELECT time
FROM plugin_thold_log
WHERE threshold_id=' . $thold_data['id'] . '
AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYWA . ')
ORDER BY time DESC
LIMIT 1', FALSE);
$ra = $warning_failures > $warning_trigger && $thold_data['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
$warning_failures++;
thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
if ($warning_failures == $warning_trigger || $ra) {
$notify = true;
}
$subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low']) . ' with ' . $thold_data['lastread'];
if ($notify) {
if ($logset == 1) {
logger($thold_data['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low'], $thold_data['lastread'], $warning_trigger, $warning_failures, $url);
}
if (trim($alert_emails) != '') {
thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_warning_traps) {
$thold_snmp_data['eventClass'] = 2;
$thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
$thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
$thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
$thold_snmp_data['eventNotificationType'] = ($warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
$thold_snmp_data['eventFailCount'] = $warning_failures;
$thold_snmp_data['eventFailCountTrigger'] = $warning_trigger;
$thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
} elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
$subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
} else {
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails));
}
db_execute('UPDATE thold_data
SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_warning_fail_count={$warning_failures},\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
} else {
thold_debug('Threshold Time Based check is normal HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
if ($alertstat != 0 && $warning_failures < $warning_trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
}
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($warning_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
} elseif ($alertstat != 0 && $failures < $trigger && $thold_data['restored_alert'] != 'on') {
if ($logset == 1) {
logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
}
$subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
}
if ($thold_snmp_traps && $thold_snmp_normal_traps) {
$thold_snmp_data['eventClass'] = 1;
$thold_snmp_data['eventSeverity'] = 1;
$thold_snmp_data['eventStatus'] = 1;
$thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
thold_snmptrap($thold_snmp_data);
}
thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
} else {
db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_fail_count={$failures},\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
}
}
break;
}
}