예제 #1
0
function thold_check_threshold($rra_id, $data_id, $name, $currentval, $cdef)
{
    global $config, $plugins, $debug;
    thold_debug("Checking Threshold:  DS:{$name} RRA_ID:{$rra_id} DATA_ID:{$data_id} VALUE:{$currentval}");
    $debug = false;
    // Do not proceed if we have chosen to globally disable all alerts
    if (read_config_option('thold_disable_all') == 'on') {
        thold_debug('Threshold checking is disabled globally');
        return;
    }
    $alert_exempt = read_config_option('alert_exempt');
    /* check for exemptions */
    $weekday = date('l');
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* Get all the info about the item from the database */
    $item = db_fetch_assoc("SELECT * FROM thold_data WHERE thold_enabled='on' AND data_id=" . $data_id);
    /* return if the item doesn't exist, which means its disabled */
    if (!isset($item[0])) {
        thold_debug('Threshold is disabled');
        return;
    }
    $item = $item[0];
    /* check for the weekend exemption on the threshold level */
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $item['exempt'] == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* don't alert for this host if it's selected for maintenance */
    if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
        include_once $config["base_path"] . '/plugins/maint/functions.php';
        if (plugin_maint_check_cacti_host($item['host_id'])) {
            thold_debug('Threshold checking is disabled by maintenance schedule');
            return;
        }
    }
    $graph_id = $item['graph_id'];
    /* only alert if Host is in UP mode (not down, unknown, or recovering) */
    $h = db_fetch_row('SELECT * FROM host WHERE id=' . $item['host_id']);
    if ($h['status'] != 3) {
        thold_debug('Threshold checking halted by Host Status (' . $h['status'] . ')');
        return;
    }
    /* pull the cached name, if not present, it means that the graph hasn't polled yet */
    $t = db_fetch_assoc('SELECT id, name, name_cache
		FROM data_template_data
		WHERE local_data_id = ' . $rra_id . '
		ORDER BY id
		LIMIT 1');
    /* pull a few default settings */
    $global_alert_address = read_config_option('alert_email');
    $global_notify_enabled = read_config_option('alert_notify_default') == 'on';
    $logset = read_config_option('alert_syslog') == 'on';
    $deadnotify = read_config_option('alert_deadnotify') == 'on';
    $realert = read_config_option('alert_repeat');
    $alert_trigger = read_config_option('alert_trigger');
    $alert_bl_trigger = read_config_option('alert_bl_trigger');
    $httpurl = read_config_option('alert_base_url');
    $thold_show_datasource = read_config_option('thold_show_datasource');
    $thold_send_text_only = read_config_option('thold_send_text_only');
    $thold_alert_text = read_config_option('thold_alert_text');
    $thold_warning_text = read_config_option('thold_warning_text');
    /* remove this after adding an option for it */
    $thold_show_datasource = true;
    $trigger = $item['thold_fail_trigger'] == '' ? $alert_trigger : $item['thold_fail_trigger'];
    $warning_trigger = $item['thold_warning_fail_trigger'] == '' ? $alert_trigger : $item['thold_warning_fail_trigger'];
    $alertstat = $item['thold_alert'];
    /* make sure the alert text has been set */
    if (!isset($thold_alert_text) || $thold_alert_text == '') {
        $thold_alert_text = "<html><body>An alert has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
    }
    /* make sure the warning text has been set */
    if (!isset($thold_warning_text) || $thold_warning_text == '') {
        $thold_warning_text = "<html><body>A warning has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
    }
    $hostname = db_fetch_row('SELECT description, hostname from host WHERE id = ' . $item['host_id']);
    $rows = db_fetch_assoc('SELECT plugin_thold_contacts.data
		FROM plugin_thold_contacts, plugin_thold_threshold_contact
		WHERE plugin_thold_contacts.id=plugin_thold_threshold_contact.contact_id
		AND plugin_thold_threshold_contact.thold_id = ' . $item['id']);
    $alert_emails = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        $alert_emails = array();
        if (count($rows)) {
            foreach ($rows as $row) {
                $alert_emails[] = $row['data'];
            }
        }
        $alert_emails = implode(',', $alert_emails);
        if ($alert_emails != '') {
            $alert_emails .= ',' . $item['notify_extra'];
        } else {
            $alert_emails = $item['notify_extra'];
        }
    }
    $alert_emails .= (strlen($alert_emails) ? "," : "") . get_thold_notification_emails($item['notify_alert']);
    $alert_phones = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        //	$alert_phones = array();
        //	if (count($rows)) {
        //		foreach ($rows as $row) {
        //		$alert_phones[] = $row['data'];
        //		}
        //	}
        //	$alert_phones = implode(',', $alert_phones);
        if ($alert_phones != '') {
            $alert_phones .= ',' . $thold['alert_phones_extra'];
        } else {
            $alert_phones = $item['alert_phones_extra'];
        }
    }
    $alert_phones .= (strlen($alert_phones) ? "," : "") . get_thold_notification_phones($item['notify_alert']);
    $warning_emails = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        $warning_emails = $item['notify_warning_extra'];
    }
    $warning_emails .= (strlen($warning_emails) ? "," : "") . get_thold_notification_emails($item['notify_warning']);
    $warning_phones = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        $warning_phones = $item['warning_phones_extra'];
    }
    $alert_command = '';
    $alert_command = $item['alert_command'];
    thold_debug('Alert Command: ' . $alert_command);
    $warning_command = $item['warning_command'];
    $types = array('High/Low', 'Baseline Deviation', 'Time Based');
    // Do some replacement of variables
    $thold_alert_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_alert_text);
    $thold_alert_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_alert_text);
    $thold_alert_text = str_replace('<TIME>', time(), $thold_alert_text);
    $thold_alert_text = str_replace('<GRAPHID>', $graph_id, $thold_alert_text);
    $thold_alert_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_alert_text);
    $thold_alert_text = str_replace('<CURRENTVALUE>', $currentval, $thold_alert_text);
    $thold_alert_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_alert_text);
    $thold_alert_text = str_replace('<DSNAME>', $name, $thold_alert_text);
    $thold_alert_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_alert_text);
    $thold_alert_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_hi'] : ''), $thold_alert_text);
    $thold_alert_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_low'] : ''), $thold_alert_text);
    $thold_alert_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_fail_trigger'] : ''), $thold_alert_text);
    $thold_alert_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_fail_length'], 'time') : '', $thold_alert_text);
    $thold_alert_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_alert_text);
    $thold_alert_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_alert_text);
    // Do some replacement of variables
    $thold_warning_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_warning_text);
    $thold_warning_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_warning_text);
    $thold_warning_text = str_replace('<TIME>', time(), $thold_warning_text);
    $thold_warning_text = str_replace('<GRAPHID>', $graph_id, $thold_warning_text);
    $thold_warning_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_warning_text);
    $thold_warning_text = str_replace('<CURRENTVALUE>', $currentval, $thold_warning_text);
    $thold_warning_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_warning_text);
    $thold_warning_text = str_replace('<DSNAME>', $name, $thold_warning_text);
    $thold_warning_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_warning_text);
    $thold_warning_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_warning_hi'] : ''), $thold_warning_text);
    $thold_warning_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_warning_low'] : ''), $thold_warning_text);
    $thold_warning_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_warning_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_warning_fail_trigger'] : ''), $thold_warning_text);
    $thold_warning_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_warning_fail_length'], 'time') : '', $thold_warning_text);
    $thold_warning_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_warning_text);
    $thold_warning_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_warning_text);
    $msg = $thold_alert_text;
    $warn_msg = $thold_warning_text;
    if ($thold_send_text_only == 'on') {
        $file_array = '';
    } else {
        $file_array = array(0 => array('local_graph_id' => $graph_id, 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id={$graph_id}&rra_id=0&view_type=tree", 'mimetype' => 'image/png', 'filename' => $graph_id));
    }
    $url = $httpurl . "/graph.php?local_graph_id=" . $graph_id . "&rra_id=all";
    switch ($item['thold_type']) {
        case 0:
            /* hi/low */
            if ($currentval != '') {
                $breach_up = $item['thold_hi'] != '' && $currentval > $item['thold_hi'];
                $breach_down = $item['thold_low'] != '' && $currentval < $item['thold_low'];
                $warning_breach_up = $item['thold_warning_hi'] != '' && $currentval > $item['thold_warning_hi'];
                $warning_breach_down = $item['thold_warning_low'] != '' && $currentval < $item['thold_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            /* is in alert status */
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low check breached HI:' . $item['thold_hi'] . '  LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
                $item['thold_fail_count']++;
                $item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                /* Re-Alert? */
                $ra = $item['thold_fail_count'] > $trigger && $item['repeat_alert'] != 0 && $item['thold_fail_count'] % $item['repeat_alert'] == 0;
                if ($item['thold_fail_count'] == $trigger || $ra) {
                    $notify = true;
                }
                $subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still ' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $trigger, $item['thold_fail_count'], $url);
                    }
                    if ($alert_command != '') {
                        exec_script($alert_command);
                    }
                    if (trim($alert_phones) != '') {
                        thold_sms($alert_phones, $subject);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $msg, $file_array);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count=" . $item['thold_fail_count'] . ",\r\n\t\t\t\tthold_warning_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low Warning check breached HI:' . $item['thold_warning_hi'] . '  LOW:' . $item['thold_warning_low'] . ' VALUE:' . $currentval);
                $item['thold_warning_fail_count']++;
                $item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                /* re-alert? */
                $ra = $item['thold_warning_fail_count'] > $warning_trigger && $item['repeat_alert'] != 0 && $item['thold_warning_fail_count'] % $item['repeat_alert'] == 0;
                if ($item['thold_warning_fail_count'] == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low']) . " with {$currentval}";
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($item['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
                    }
                    if ($warning_command != '') {
                        exec_script($warning_command);
                    }
                    if (trim($warning_phones) != '') {
                        thold_sms($warning_phones, $subject);
                    }
                    if (trim($warning_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
                } elseif ($item['thold_warning_fail_count'] >= $warning_trigger && $item['thold_fail_count'] >= $trigger) {
                    $subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Changed to Warning Threshold with Value {$currentval}";
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if (trim($alert_phones) != '') {
                        thold_sms($alert_phones, $subject);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count=" . $item['thold_warning_fail_count'] . ",\r\n\t\t\t\tthold_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } else {
                thold_debug('Threshold HI / Low check is normal HI:' . $item['thold_hi'] . '  LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
                /* if we were at an alert status before */
                if ($alertstat != 0) {
                    $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Restored to Normal Threshold with Value {$currentval}";
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_fail_count=0, thold_warning_fail_count=0\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                    if ($item['thold_warning_fail_count'] >= $warning_trigger && $item['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
                        }
                        if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
                            thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                        }
                        if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
                            thold_sms($warning_phones, $subject);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
                    } elseif ($item['thold_fail_count'] >= $trigger && $item['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
                        }
                        if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
                            thold_mail($alert_emails, '', $subject, $msg, $file_array);
                        }
                        if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
                            thold_sms($alert_phones, $subject);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                    }
                }
            }
            break;
        case 1:
            /* baseline */
            $bl_alert_prev = $item['bl_alert'];
            $bl_count_prev = $item['bl_fail_count'];
            $bl_fail_trigger = $item['bl_fail_trigger'] == '' ? $alert_bl_trigger : $item['bl_fail_trigger'];
            $item['bl_alert'] = thold_check_baseline($rra_id, $name, $currentval, $item);
            switch ($item['bl_alert']) {
                case -2:
                    /* exception is active, Future Release 'todo' */
                    break;
                case -1:
                    /* reference value not available, Future Release 'todo' */
                    break;
                case 0:
                    /* all clear */
                    /* if we were at an alert status before */
                    if ($alertstat != 0) {
                        thold_debug('Threshold Baseline check is normal');
                        if ($item['bl_fail_count'] >= $bl_fail_trigger && $item['restored_alert'] != 'on') {
                            thold_debug('Threshold Baseline check returned to normal');
                            if ($logset == 1) {
                                logger($item['name'], 'ok', 0, $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
                            }
                            $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
                            if (trim($alert_emails) != '') {
                                thold_mail($alert_emails, '', $subject, $msg, $file_array);
                            }
                            if (trim($alert_phones) != '') {
                                thold_sms($alert_phones, $subject);
                            }
                            thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                        }
                    }
                    $item['bl_fail_count'] = 0;
                    break;
                case 1:
                    /* value is below calculated threshold */
                /* value is below calculated threshold */
                case 2:
                    /* value is above calculated threshold */
                    $item['bl_fail_count']++;
                    $breach_up = $item['bl_alert'] == STAT_HI;
                    $breach_down = $item['bl_alert'] == STAT_LO;
                    thold_debug('Threshold Baseline check breached');
                    /* re-alert? */
                    $ra = $item['bl_fail_count'] > $bl_fail_trigger && $item['bl_fail_count'] % ($item['repeat_alert'] == '' ? $realert : $item['repeat_alert']) == 0;
                    if ($item['bl_fail_count'] == $bl_fail_trigger || $ra) {
                        thold_debug('Alerting is necessary');
                        $subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . " calculated baseline threshold " . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
                        if ($logset == 1) {
                            logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
                        }
                        if ($alert_command != '') {
                            exec_script($alert_command);
                        }
                        if (trim($alert_phones) != '') {
                            thold_sms($alert_phones, $subject);
                        }
                        if (trim($alert_emails) != '') {
                            thold_mail($alert_emails, '', $subject, $msg, $file_array);
                        }
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
                    } else {
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                    }
                    break;
            }
            db_execute("UPDATE thold_data SET thold_alert=0, thold_fail_count=0,\r\n\t\t\tbl_alert='" . $item['bl_alert'] . "',\r\n\t\t\tbl_fail_count='" . $item['bl_fail_count'] . "',\r\n\t\t\tthold_low='" . $item['thold_low'] . "',\r\n\t\t\tthold_hi='" . $item['thold_hi'] . "',\r\n\t\t\tbl_thold_valid='" . $item['bl_thold_valid'] . "'\r\n\t\t\tWHERE rra_id='{$rra_id}' AND data_id=" . $item['data_id']);
            break;
        case 2:
            /* time based */
            if ($currentval != '') {
                $breach_up = $item['time_hi'] != '' && $currentval > $item['time_hi'];
                $breach_down = $item['time_low'] != '' && $currentval < $item['time_low'];
                $warning_breach_up = $item['time_warning_hi'] != '' && $currentval > $item['time_warning_hi'];
                $warning_breach_down = $item['time_warning_low'] != '' && $currentval < $item['time_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            $step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $rra_id, FALSE);
            /* alerts */
            $trigger = $item['time_fail_trigger'];
            $time = time() - $item['time_fail_length'] * $step;
            $failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_TRIGGERA . "," . ST_NOTIFYRA . "," . ST_NOTIFYAL . ") AND time>" . $time);
            /* warnings */
            $warning_trigger = $item['time_warning_fail_trigger'];
            $warning_time = time() - $item['time_warning_fail_length'] * $step;
            $warning_failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_NOTIFYWA . "," . ST_TRIGGERW . ") AND time>" . $warning_time) + $failures;
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold Time Based check breached HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
                $item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                $item['thold_fail_count'] = $failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($item['repeat_alert'] - 1) * $step;
                $lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYAL . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
                $ra = $failures > $trigger && $item['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $failures++;
                thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($failures == $trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? "ALERT: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['time_hi'] : $item['time_low']) . " with {$currentval}";
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($item['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $item['time_hi'] : $item['time_low'], $currentval, $trigger, $failures, $url);
                    }
                    if ($alert_command != '') {
                        exec_script($alert_command);
                    }
                    if (trim($alert_phones) != '') {
                        thold_sms($alert_phones, $subject);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $msg, $file_array);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                $item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                $item['thold_warning_fail_count'] = $warning_failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($item['time_warning_fail_length'] - 1) * $step;
                $lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYWA . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
                $ra = $warning_failures > $warning_trigger && $item['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $warning_failures++;
                thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($warning_failures == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low']) . " with {$currentval}";
                if ($notify) {
                    if ($logset == 1) {
                        logger($item['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low'], $currentval, $warning_trigger, $warning_failures, $url);
                    }
                    if ($warning_command != '') {
                        exec_script($warning_command);
                    }
                    if (trim($warning_phones) != '') {
                        thold_sms($warning_phones, $subject);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
                } elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
                    $subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count={$warning_failures},\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } else {
                thold_debug('Threshold Time Based check is normal HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
                if ($alertstat != 0 && $warning_failures < $warning_trigger && $item['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
                    }
                    $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
                    if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
                        thold_mail($warning_emails, '', $subject, $msg, $file_array);
                    }
                    if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
                        thold_sms($warning_phones, $subject);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                } elseif ($alertstat != 0 && $failures < $trigger && $item['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
                    }
                    $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
                    if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
                        thold_mail($alert_emails, '', $subject, $msg, $file_array);
                    }
                    if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
                        thold_sms($alert_phones, $subject);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                } else {
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_fail_count={$failures},\r\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                }
            }
            break;
    }
}
예제 #2
0
function thold_check_threshold(&$thold_data)
{
    global $config, $plugins, $debug, $thold_types;
    $name = db_fetch_cell_prepared('SELECT data_source_name FROM data_template_rrd WHERE id = ?', array($thold_data['data_template_rrd_id']));
    thold_debug('Checking Threshold:' . ' Name: ' . $name . ', local_data_id: ' . $thold_data['local_data_id'] . ', data_template_rrd_id: ' . $thold_data['data_template_rrd_id'] . ', value: ' . $thold_data['lastread']);
    $debug = false;
    // Do not proceed if we have chosen to globally disable all alerts
    if (read_config_option('thold_disable_all') == 'on') {
        thold_debug('Threshold checking is disabled globally');
        return;
    }
    $alert_exempt = read_config_option('alert_exempt');
    /* check for exemptions */
    $weekday = date('l');
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* check for the weekend exemption on the threshold level */
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $thold_data['exempt'] == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* don't alert for this host if it's selected for maintenance */
    if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
        include_once $config['base_path'] . '/plugins/maint/functions.php';
        if (plugin_maint_check_cacti_host($thold_data['host_id'])) {
            thold_debug('Threshold checking is disabled by maintenance schedule');
            return;
        }
    }
    $local_graph_id = $thold_data['local_graph_id'];
    /* only alert if Device is in UP mode (not down, unknown, or recovering) */
    $h = db_fetch_row('SELECT * FROM host WHERE id=' . $thold_data['host_id']);
    if (sizeof($h) && $h['status'] != 3) {
        thold_debug('Threshold checking halted by Device Status (' . $h['status'] . ')');
        return;
    }
    /* pull the cached name, if not present, it means that the graph hasn't polled yet */
    $t = db_fetch_assoc('SELECT id, name, name_cache
		FROM data_template_data
		WHERE local_data_id = ' . $thold_data['local_data_id'] . '
		ORDER BY id
		LIMIT 1');
    /* pull a few default settings */
    $global_alert_address = read_config_option('alert_email');
    $global_notify_enabled = read_config_option('alert_notify_default') == 'on';
    $logset = read_config_option('alert_syslog') == 'on';
    $deadnotify = read_config_option('alert_deadnotify') == 'on';
    $realert = read_config_option('alert_repeat');
    $alert_trigger = read_config_option('alert_trigger');
    $alert_bl_trigger = read_config_option('alert_bl_trigger');
    $httpurl = read_config_option('base_url');
    $thold_send_text_only = read_config_option('thold_send_text_only');
    $thold_snmp_traps = read_config_option('thold_alert_snmp') == 'on';
    $thold_snmp_warning_traps = read_config_option('thold_alert_snmp_warning') != 'on';
    $thold_snmp_normal_traps = read_config_option('thold_alert_snmp_normal') != 'on';
    $cacti_polling_interval = read_config_option('poller_interval');
    /* remove this after adding an option for it */
    $thold_show_datasource = thold_datasource_required($thold_data['name'], $name);
    $trigger = $thold_data['thold_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_fail_trigger'];
    $warning_trigger = $thold_data['thold_warning_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_warning_fail_trigger'];
    $alertstat = $thold_data['thold_alert'];
    $alert_emails = get_thold_alert_emails($thold_data);
    $warning_emails = get_thold_warning_emails($thold_data);
    $alert_msg = get_thold_alert_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
    $warn_msg = get_thold_warning_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
    $thold_snmp_data = get_thold_snmp_data($name, $thold_data, $h, $thold_data['lastread']);
    $file_array = '';
    if ($thold_send_text_only != 'on') {
        if (!empty($thold_data['local_graph_id'])) {
            $file_array[] = array('local_graph_id' => $thold_data['local_graph_id'], 'local_data_id' => $thold_data['local_data_id'], 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id=" . $thold_data['local_graph_id'] . '&rra_id=0&view_type=tree', 'mimetype' => 'image/png', 'filename' => clean_up_name($thold_data['name']));
        }
    }
    $url = $httpurl . '/graph.php?local_graph_id=' . $thold_data['local_graph_id'] . '&rra_id=all';
    switch ($thold_data['thold_type']) {
        case 0:
            /* hi/low */
            if ($thold_data['lastread'] != '') {
                $breach_up = $thold_data['thold_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_hi'];
                $breach_down = $thold_data['thold_low'] != '' && $thold_data['lastread'] < $thold_data['thold_low'];
                $warning_breach_up = $thold_data['thold_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_warning_hi'];
                $warning_breach_down = $thold_data['thold_warning_low'] != '' && $thold_data['lastread'] < $thold_data['thold_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            /* is in alert status */
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low check breached HI:' . $thold_data['thold_hi'] . '  LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
                $thold_data['thold_fail_count']++;
                $thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                /* Re-Alert? */
                $ra = $thold_data['thold_fail_count'] > $trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_fail_count'] % $thold_data['repeat_alert'] == 0;
                if ($thold_data['thold_fail_count'] == $trigger || $ra) {
                    $notify = true;
                }
                $subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps) {
                        $thold_snmp_data['eventClass'] = 3;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
                        $thold_snmp_data['eventFailCount'] = $thold_data['thold_fail_count'];
                        $thold_snmp_data['eventFailDuration'] = $thold_data['thold_fail_count'] * $cacti_polling_interval;
                        $thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
                        $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ',
				thold_fail_count=' . $thold_data['thold_fail_count'] . ",\n\t\t\t\tthold_warning_fail_count=0\n\t\t\t\tWHERE id=" . $thold_data['id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low Warning check breached HI:' . $thold_data['thold_warning_hi'] . '  LOW:' . $thold_data['thold_warning_low'] . ' VALUE:' . $thold_data['lastread']);
                $thold_data['thold_warning_fail_count']++;
                $thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                /* re-alert? */
                $ra = $thold_data['thold_warning_fail_count'] > $warning_trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_warning_fail_count'] % $thold_data['repeat_alert'] == 0;
                if ($thold_data['thold_warning_fail_count'] == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($thold_data['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
                    }
                    if (trim($warning_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_warning_traps) {
                        $thold_snmp_data['eventClass'] = 2;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
                        $thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
                        $thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
                        $thold_snmp_data['eventFailDurationTrigger'] = $warning_trigger * $cacti_polling_interval;
                        $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
                } elseif ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['thold_fail_count'] >= $trigger) {
                    $subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Changed to Warning Threshold with Value ' . $thold_data['lastread'];
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_warning_traps) {
                        $thold_snmp_data['eventClass'] = 2;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventNotificationType'] = ST_NOTIFYAW + 1;
                        $thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
                        $thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
                        $thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
                        $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ',
				thold_warning_fail_count=' . $thold_data['thold_warning_fail_count'] . ',
				thold_fail_count=0
				WHERE id=' . $thold_data['id']);
            } else {
                thold_debug('Threshold HI / Low check is normal HI:' . $thold_data['thold_hi'] . '  LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
                /* if we were at an alert status before */
                if ($alertstat != 0) {
                    $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Restored to Normal Threshold with Value ' . $thold_data['lastread'];
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_fail_count=0, \n\t\t\t\t\tthold_warning_fail_count=0\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                    if ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
                        }
                        if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
                            thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                        }
                        if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                            $thold_snmp_data['eventClass'] = 1;
                            $thold_snmp_data['eventSeverity'] = 1;
                            $thold_snmp_data['eventStatus'] = 1;
                            $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                            thold_snmptrap($thold_snmp_data);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
                    } elseif ($thold_data['thold_fail_count'] >= $trigger && $thold_data['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
                        }
                        if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
                            thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                        }
                        if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                            $thold_snmp_data['eventClass'] = 1;
                            $thold_snmp_data['eventSeverity'] = 1;
                            $thold_snmp_data['eventStatus'] = 1;
                            $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                            thold_snmptrap($thold_snmp_data);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
                    }
                }
            }
            break;
        case 1:
            /* baseline */
            $bl_alert_prev = $thold_data['bl_alert'];
            $bl_count_prev = $thold_data['bl_fail_count'];
            $bl_fail_trigger = $thold_data['bl_fail_trigger'] == '' ? $alert_bl_trigger : $thold_data['bl_fail_trigger'];
            $thold_data['bl_alert'] = thold_check_baseline($local_data_id, $name, $thold_data['lastread'], $thold_data);
            switch ($thold_data['bl_alert']) {
                case -2:
                    /* exception is active, Future Release 'todo' */
                    break;
                case -1:
                    /* reference value not available, Future Release 'todo' */
                    break;
                case 0:
                    /* all clear */
                    /* if we were at an alert status before */
                    if ($alertstat != 0) {
                        thold_debug('Threshold Baseline check is normal');
                        if ($thold_data['bl_fail_count'] >= $bl_fail_trigger && $thold_data['restored_alert'] != 'on') {
                            thold_debug('Threshold Baseline check returned to normal');
                            if ($logset == 1) {
                                logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
                            }
                            $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
                            if (trim($alert_emails) != '') {
                                thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                            }
                            if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                                $thold_snmp_data['eventClass'] = 1;
                                $hold_snmp_data['eventSeverity'] = 1;
                                $thold_snmp_data['eventStatus'] = 1;
                                $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                                thold_snmptrap($thold_snmp_data);
                            }
                            thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
                        }
                    }
                    $thold_data['bl_fail_count'] = 0;
                    break;
                case 1:
                    /* value is below calculated threshold */
                /* value is below calculated threshold */
                case 2:
                    /* value is above calculated threshold */
                    $thold_data['bl_fail_count']++;
                    $breach_up = $thold_data['bl_alert'] == STAT_HI;
                    $breach_down = $thold_data['bl_alert'] == STAT_LO;
                    thold_debug('Threshold Baseline check breached');
                    /* re-alert? */
                    $ra = $thold_data['bl_fail_count'] > $bl_fail_trigger && $thold_data['bl_fail_count'] % ($thold_data['repeat_alert'] == '' ? $realert : $thold_data['repeat_alert']) == 0;
                    if ($thold_data['bl_fail_count'] == $bl_fail_trigger || $ra) {
                        thold_debug('Alerting is necessary');
                        $subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' calculated baseline threshold ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
                        if ($logset == 1) {
                            logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
                        }
                        if (trim($alert_emails) != '') {
                            thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                        }
                        if ($thold_snmp_traps) {
                            $thold_snmp_data['eventClass'] = 3;
                            $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
                            $thold_snmp_data['eventStatus'] = $thold_data['bl_alert'] + 1;
                            $thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
                            $thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
                            $thold_snmp_data['eventFailCount'] = $thold_data['bl_fail_count'];
                            $thold_snmp_data['eventFailDuration'] = $thold_data['bl_fail_count'] * $cacti_polling_interval;
                            $thold_snmp_data['eventFailCountTrigger'] = $bl_fail_trigger;
                            $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                            thold_snmptrap($thold_snmp_data);
                        }
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
                    } else {
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
                    }
                    break;
            }
            db_execute("UPDATE thold_data \n\t\t\tSET thold_alert=0, \n\t\t\tthold_fail_count=0,\n\t\t\tbl_alert='" . $thold_data['bl_alert'] . "',\n\t\t\tbl_fail_count='" . $thold_data['bl_fail_count'] . "',\n\t\t\tthold_low='" . $thold_data['thold_low'] . "',\n\t\t\tthold_hi='" . $thold_data['thold_hi'] . "',\n\t\t\tbl_thold_valid='" . $thold_data['bl_thold_valid'] . "'\n\t\t\tWHERE id=" . $thold_data['id']);
            break;
        case 2:
            /* time based */
            if ($thold_data['lastread'] != '') {
                $breach_up = $thold_data['time_hi'] != '' && $thold_data['lastread'] > $thold_data['time_hi'];
                $breach_down = $thold_data['time_low'] != '' && $thold_data['lastread'] < $thold_data['time_low'];
                $warning_breach_up = $thold_data['time_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['time_warning_hi'];
                $warning_breach_down = $thold_data['time_warning_low'] != '' && $thold_data['lastread'] < $thold_data['time_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            $step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $local_data_id, FALSE);
            /* alerts */
            $trigger = $thold_data['time_fail_trigger'];
            $time = time() - $thold_data['time_fail_length'] * $step;
            $failures = db_fetch_cell('SELECT count(id) 
			FROM plugin_thold_log 
			WHERE threshold_id=' . $thold_data['id'] . ' 
			AND status IN (' . ST_TRIGGERA . ',' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ') 
			AND time>' . $time);
            /* warnings */
            $warning_trigger = $thold_data['time_warning_fail_trigger'];
            $warning_time = time() - $thold_data['time_warning_fail_length'] * $step;
            $warning_failures = db_fetch_cell('SELECT count(id) 
			FROM plugin_thold_log 
			WHERE threshold_id=' . $thold_data['id'] . ' 
			AND status IN (' . ST_NOTIFYWA . ',' . ST_TRIGGERW . ') 
			AND time>' . $warning_time) + $failures;
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold Time Based check breached HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
                $thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                $thold_data['thold_fail_count'] = $failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($thold_data['repeat_alert'] - 1) * $step;
                $lastemailtime = db_fetch_cell('SELECT time
				FROM plugin_thold_log
				WHERE threshold_id=' . $thold_data['id'] . '
				AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ')
				ORDER BY time DESC
				LIMIT 1', FALSE);
                $ra = $failures > $trigger && $thold_data['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $failures++;
                thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($failures == $trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? 'ALERT: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['time_hi'] : $thold_data['time_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($thold_data['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], $thold_data['lastread'], $trigger, $failures, $url);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps) {
                        $thold_snmp_data['eventClass'] = 3;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA) + 1;
                        $thold_snmp_data['eventFailCount'] = $failures;
                        $thold_snmp_data['eventFailCountTrigger'] = $trigger;
                        $thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                $thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                $thold_data['thold_warning_fail_count'] = $warning_failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($thold_data['time_warning_fail_length'] - 1) * $step;
                $lastemailtime = db_fetch_cell('SELECT time
				FROM plugin_thold_log
				WHERE threshold_id=' . $thold_data['id'] . '
				AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYWA . ')
				ORDER BY time DESC
				LIMIT 1', FALSE);
                $ra = $warning_failures > $warning_trigger && $thold_data['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $warning_failures++;
                thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($warning_failures == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    if ($logset == 1) {
                        logger($thold_data['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low'], $thold_data['lastread'], $warning_trigger, $warning_failures, $url);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_warning_traps) {
                        $thold_snmp_data['eventClass'] = 2;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
                        $thold_snmp_data['eventFailCount'] = $warning_failures;
                        $thold_snmp_data['eventFailCountTrigger'] = $warning_trigger;
                        $thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
                } elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
                    $subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_warning_fail_count={$warning_failures},\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
            } else {
                thold_debug('Threshold Time Based check is normal HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
                if ($alertstat != 0 && $warning_failures < $warning_trigger && $thold_data['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
                    }
                    $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
                    if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
                        thold_mail($warning_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                        $thold_snmp_data['eventClass'] = 1;
                        $thold_snmp_data['eventSeverity'] = 1;
                        $thold_snmp_data['eventStatus'] = 1;
                        $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                } elseif ($alertstat != 0 && $failures < $trigger && $thold_data['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
                    }
                    $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
                    if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
                        thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                        $thold_snmp_data['eventClass'] = 1;
                        $thold_snmp_data['eventSeverity'] = 1;
                        $thold_snmp_data['eventStatus'] = 1;
                        $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                } else {
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_fail_count={$failures},\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                }
            }
            break;
    }
}