예제 #1
0
function thold_check_threshold($rra_id, $data_id, $name, $currentval, $cdef)
{
    global $config, $plugins, $debug;
    thold_debug("Checking Threshold:  DS:{$name} RRA_ID:{$rra_id} DATA_ID:{$data_id} VALUE:{$currentval}");
    $debug = false;
    // Do not proceed if we have chosen to globally disable all alerts
    if (read_config_option('thold_disable_all') == 'on') {
        thold_debug('Threshold checking is disabled globally');
        return;
    }
    $alert_exempt = read_config_option('alert_exempt');
    /* check for exemptions */
    $weekday = date('l');
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* Get all the info about the item from the database */
    $item = db_fetch_assoc("SELECT * FROM thold_data WHERE thold_enabled='on' AND data_id=" . $data_id);
    /* return if the item doesn't exist, which means its disabled */
    if (!isset($item[0])) {
        thold_debug('Threshold is disabled');
        return;
    }
    $item = $item[0];
    /* check for the weekend exemption on the threshold level */
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $item['exempt'] == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* don't alert for this host if it's selected for maintenance */
    if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
        include_once $config["base_path"] . '/plugins/maint/functions.php';
        if (plugin_maint_check_cacti_host($item['host_id'])) {
            thold_debug('Threshold checking is disabled by maintenance schedule');
            return;
        }
    }
    $graph_id = $item['graph_id'];
    /* only alert if Host is in UP mode (not down, unknown, or recovering) */
    $h = db_fetch_row('SELECT * FROM host WHERE id=' . $item['host_id']);
    if ($h['status'] != 3) {
        thold_debug('Threshold checking halted by Host Status (' . $h['status'] . ')');
        return;
    }
    /* pull the cached name, if not present, it means that the graph hasn't polled yet */
    $t = db_fetch_assoc('SELECT id, name, name_cache
		FROM data_template_data
		WHERE local_data_id = ' . $rra_id . '
		ORDER BY id
		LIMIT 1');
    /* pull a few default settings */
    $global_alert_address = read_config_option('alert_email');
    $global_notify_enabled = read_config_option('alert_notify_default') == 'on';
    $logset = read_config_option('alert_syslog') == 'on';
    $deadnotify = read_config_option('alert_deadnotify') == 'on';
    $realert = read_config_option('alert_repeat');
    $alert_trigger = read_config_option('alert_trigger');
    $alert_bl_trigger = read_config_option('alert_bl_trigger');
    $httpurl = read_config_option('alert_base_url');
    $thold_show_datasource = read_config_option('thold_show_datasource');
    $thold_send_text_only = read_config_option('thold_send_text_only');
    $thold_alert_text = read_config_option('thold_alert_text');
    $thold_warning_text = read_config_option('thold_warning_text');
    /* remove this after adding an option for it */
    $thold_show_datasource = true;
    $trigger = $item['thold_fail_trigger'] == '' ? $alert_trigger : $item['thold_fail_trigger'];
    $warning_trigger = $item['thold_warning_fail_trigger'] == '' ? $alert_trigger : $item['thold_warning_fail_trigger'];
    $alertstat = $item['thold_alert'];
    /* make sure the alert text has been set */
    if (!isset($thold_alert_text) || $thold_alert_text == '') {
        $thold_alert_text = "<html><body>An alert has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
    }
    /* make sure the warning text has been set */
    if (!isset($thold_warning_text) || $thold_warning_text == '') {
        $thold_warning_text = "<html><body>A warning has been issued that requires your attention.<br><br><strong>Host</strong>: <DESCRIPTION> (<HOSTNAME>)<br><strong>URL</strong>: <URL><br><strong>Message</strong>: <SUBJECT><br><br><GRAPH></body></html>";
    }
    $hostname = db_fetch_row('SELECT description, hostname from host WHERE id = ' . $item['host_id']);
    $rows = db_fetch_assoc('SELECT plugin_thold_contacts.data
		FROM plugin_thold_contacts, plugin_thold_threshold_contact
		WHERE plugin_thold_contacts.id=plugin_thold_threshold_contact.contact_id
		AND plugin_thold_threshold_contact.thold_id = ' . $item['id']);
    $alert_emails = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        $alert_emails = array();
        if (count($rows)) {
            foreach ($rows as $row) {
                $alert_emails[] = $row['data'];
            }
        }
        $alert_emails = implode(',', $alert_emails);
        if ($alert_emails != '') {
            $alert_emails .= ',' . $item['notify_extra'];
        } else {
            $alert_emails = $item['notify_extra'];
        }
    }
    $alert_emails .= (strlen($alert_emails) ? "," : "") . get_thold_notification_emails($item['notify_alert']);
    $alert_phones = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        //	$alert_phones = array();
        //	if (count($rows)) {
        //		foreach ($rows as $row) {
        //		$alert_phones[] = $row['data'];
        //		}
        //	}
        //	$alert_phones = implode(',', $alert_phones);
        if ($alert_phones != '') {
            $alert_phones .= ',' . $thold['alert_phones_extra'];
        } else {
            $alert_phones = $item['alert_phones_extra'];
        }
    }
    $alert_phones .= (strlen($alert_phones) ? "," : "") . get_thold_notification_phones($item['notify_alert']);
    $warning_emails = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        $warning_emails = $item['notify_warning_extra'];
    }
    $warning_emails .= (strlen($warning_emails) ? "," : "") . get_thold_notification_emails($item['notify_warning']);
    $warning_phones = '';
    if (read_config_option('thold_disable_legacy') != 'on') {
        $warning_phones = $item['warning_phones_extra'];
    }
    $alert_command = '';
    $alert_command = $item['alert_command'];
    thold_debug('Alert Command: ' . $alert_command);
    $warning_command = $item['warning_command'];
    $types = array('High/Low', 'Baseline Deviation', 'Time Based');
    // Do some replacement of variables
    $thold_alert_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_alert_text);
    $thold_alert_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_alert_text);
    $thold_alert_text = str_replace('<TIME>', time(), $thold_alert_text);
    $thold_alert_text = str_replace('<GRAPHID>', $graph_id, $thold_alert_text);
    $thold_alert_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_alert_text);
    $thold_alert_text = str_replace('<CURRENTVALUE>', $currentval, $thold_alert_text);
    $thold_alert_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_alert_text);
    $thold_alert_text = str_replace('<DSNAME>', $name, $thold_alert_text);
    $thold_alert_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_alert_text);
    $thold_alert_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_hi'] : ''), $thold_alert_text);
    $thold_alert_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_low'] : ''), $thold_alert_text);
    $thold_alert_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_fail_trigger'] : ''), $thold_alert_text);
    $thold_alert_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_fail_length'], 'time') : '', $thold_alert_text);
    $thold_alert_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_alert_text);
    $thold_alert_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_alert_text);
    // Do some replacement of variables
    $thold_warning_text = str_replace('<DESCRIPTION>', $hostname['description'], $thold_warning_text);
    $thold_warning_text = str_replace('<HOSTNAME>', $hostname['hostname'], $thold_warning_text);
    $thold_warning_text = str_replace('<TIME>', time(), $thold_warning_text);
    $thold_warning_text = str_replace('<GRAPHID>', $graph_id, $thold_warning_text);
    $thold_warning_text = str_replace('<URL>', "<a href='{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1'>{$httpurl}/graph.php?local_graph_id={$graph_id}&rra_id=1</a>", $thold_warning_text);
    $thold_warning_text = str_replace('<CURRENTVALUE>', $currentval, $thold_warning_text);
    $thold_warning_text = str_replace('<THRESHOLDNAME>', $item['name'], $thold_warning_text);
    $thold_warning_text = str_replace('<DSNAME>', $name, $thold_warning_text);
    $thold_warning_text = str_replace('<THOLDTYPE>', $types[$item['thold_type']], $thold_warning_text);
    $thold_warning_text = str_replace('<HI>', $item['thold_type'] == 0 ? $item['thold_hi'] : ($item['thold_type'] == 2 ? $item['time_warning_hi'] : ''), $thold_warning_text);
    $thold_warning_text = str_replace('<LOW>', $item['thold_type'] == 0 ? $item['thold_low'] : ($item['thold_type'] == 2 ? $item['time_warning_low'] : ''), $thold_warning_text);
    $thold_warning_text = str_replace('<TRIGGER>', $item['thold_type'] == 0 ? $item['thold_warning_fail_trigger'] : ($item['thold_type'] == 2 ? $item['time_warning_fail_trigger'] : ''), $thold_warning_text);
    $thold_warning_text = str_replace('<DURATION>', $item['thold_type'] == 2 ? plugin_thold_duration_convert($item['rra_id'], $item['time_warning_fail_length'], 'time') : '', $thold_warning_text);
    $thold_warning_text = str_replace('<DATE_RFC822>', date(DATE_RFC822), $thold_warning_text);
    $thold_warning_text = str_replace('<DEVICENOTE>', $h['notes'], $thold_warning_text);
    $msg = $thold_alert_text;
    $warn_msg = $thold_warning_text;
    if ($thold_send_text_only == 'on') {
        $file_array = '';
    } else {
        $file_array = array(0 => array('local_graph_id' => $graph_id, 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id={$graph_id}&rra_id=0&view_type=tree", 'mimetype' => 'image/png', 'filename' => $graph_id));
    }
    $url = $httpurl . "/graph.php?local_graph_id=" . $graph_id . "&rra_id=all";
    switch ($item['thold_type']) {
        case 0:
            /* hi/low */
            if ($currentval != '') {
                $breach_up = $item['thold_hi'] != '' && $currentval > $item['thold_hi'];
                $breach_down = $item['thold_low'] != '' && $currentval < $item['thold_low'];
                $warning_breach_up = $item['thold_warning_hi'] != '' && $currentval > $item['thold_warning_hi'];
                $warning_breach_down = $item['thold_warning_low'] != '' && $currentval < $item['thold_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            /* is in alert status */
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low check breached HI:' . $item['thold_hi'] . '  LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
                $item['thold_fail_count']++;
                $item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                /* Re-Alert? */
                $ra = $item['thold_fail_count'] > $trigger && $item['repeat_alert'] != 0 && $item['thold_fail_count'] % $item['repeat_alert'] == 0;
                if ($item['thold_fail_count'] == $trigger || $ra) {
                    $notify = true;
                }
                $subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still ' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $trigger, $item['thold_fail_count'], $url);
                    }
                    if ($alert_command != '') {
                        exec_script($alert_command);
                    }
                    if (trim($alert_phones) != '') {
                        thold_sms($alert_phones, $subject);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $msg, $file_array);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count=" . $item['thold_fail_count'] . ",\r\n\t\t\t\tthold_warning_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low Warning check breached HI:' . $item['thold_warning_hi'] . '  LOW:' . $item['thold_warning_low'] . ' VALUE:' . $currentval);
                $item['thold_warning_fail_count']++;
                $item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                /* re-alert? */
                $ra = $item['thold_warning_fail_count'] > $warning_trigger && $item['repeat_alert'] != 0 && $item['thold_warning_fail_count'] % $item['repeat_alert'] == 0;
                if ($item['thold_warning_fail_count'] == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low']) . " with {$currentval}";
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($item['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
                    }
                    if ($warning_command != '') {
                        exec_script($warning_command);
                    }
                    if (trim($warning_phones) != '') {
                        thold_sms($warning_phones, $subject);
                    }
                    if (trim($warning_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
                } elseif ($item['thold_warning_fail_count'] >= $warning_trigger && $item['thold_fail_count'] >= $trigger) {
                    $subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Changed to Warning Threshold with Value {$currentval}";
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if (trim($alert_phones) != '') {
                        thold_sms($alert_phones, $subject);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['thold_warning_hi'] : $item['thold_warning_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count=" . $item['thold_warning_fail_count'] . ",\r\n\t\t\t\tthold_fail_count=0\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } else {
                thold_debug('Threshold HI / Low check is normal HI:' . $item['thold_hi'] . '  LOW:' . $item['thold_low'] . ' VALUE:' . $currentval);
                /* if we were at an alert status before */
                if ($alertstat != 0) {
                    $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " Restored to Normal Threshold with Value {$currentval}";
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_fail_count=0, thold_warning_fail_count=0\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                    if ($item['thold_warning_fail_count'] >= $warning_trigger && $item['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
                        }
                        if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
                            thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                        }
                        if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
                            thold_sms($warning_phones, $subject);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
                    } elseif ($item['thold_fail_count'] >= $trigger && $item['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
                        }
                        if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
                            thold_mail($alert_emails, '', $subject, $msg, $file_array);
                        }
                        if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
                            thold_sms($alert_phones, $subject);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                    }
                }
            }
            break;
        case 1:
            /* baseline */
            $bl_alert_prev = $item['bl_alert'];
            $bl_count_prev = $item['bl_fail_count'];
            $bl_fail_trigger = $item['bl_fail_trigger'] == '' ? $alert_bl_trigger : $item['bl_fail_trigger'];
            $item['bl_alert'] = thold_check_baseline($rra_id, $name, $currentval, $item);
            switch ($item['bl_alert']) {
                case -2:
                    /* exception is active, Future Release 'todo' */
                    break;
                case -1:
                    /* reference value not available, Future Release 'todo' */
                    break;
                case 0:
                    /* all clear */
                    /* if we were at an alert status before */
                    if ($alertstat != 0) {
                        thold_debug('Threshold Baseline check is normal');
                        if ($item['bl_fail_count'] >= $bl_fail_trigger && $item['restored_alert'] != 'on') {
                            thold_debug('Threshold Baseline check returned to normal');
                            if ($logset == 1) {
                                logger($item['name'], 'ok', 0, $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
                            }
                            $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
                            if (trim($alert_emails) != '') {
                                thold_mail($alert_emails, '', $subject, $msg, $file_array);
                            }
                            if (trim($alert_phones) != '') {
                                thold_sms($alert_phones, $subject);
                            }
                            thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                        }
                    }
                    $item['bl_fail_count'] = 0;
                    break;
                case 1:
                    /* value is below calculated threshold */
                /* value is below calculated threshold */
                case 2:
                    /* value is above calculated threshold */
                    $item['bl_fail_count']++;
                    $breach_up = $item['bl_alert'] == STAT_HI;
                    $breach_down = $item['bl_alert'] == STAT_LO;
                    thold_debug('Threshold Baseline check breached');
                    /* re-alert? */
                    $ra = $item['bl_fail_count'] > $bl_fail_trigger && $item['bl_fail_count'] % ($item['repeat_alert'] == '' ? $realert : $item['repeat_alert']) == 0;
                    if ($item['bl_fail_count'] == $bl_fail_trigger || $ra) {
                        thold_debug('Alerting is necessary');
                        $subject = "ALERT: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . " calculated baseline threshold " . ($breach_up ? $item['thold_hi'] : $item['thold_low']) . " with {$currentval}";
                        if ($logset == 1) {
                            logger($item['name'], $ra ? 'realert' : 'alert', $breach_up ? $item['thold_hi'] : $item['thold_low'], $currentval, $item['bl_fail_trigger'], $item['bl_fail_count'], $url);
                        }
                        if ($alert_command != '') {
                            exec_script($alert_command);
                        }
                        if (trim($alert_phones) != '') {
                            thold_sms($alert_phones, $subject);
                        }
                        if (trim($alert_emails) != '') {
                            thold_mail($alert_emails, '', $subject, $msg, $file_array);
                        }
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
                    } else {
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['thold_hi'] : $item['thold_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                    }
                    break;
            }
            db_execute("UPDATE thold_data SET thold_alert=0, thold_fail_count=0,\r\n\t\t\tbl_alert='" . $item['bl_alert'] . "',\r\n\t\t\tbl_fail_count='" . $item['bl_fail_count'] . "',\r\n\t\t\tthold_low='" . $item['thold_low'] . "',\r\n\t\t\tthold_hi='" . $item['thold_hi'] . "',\r\n\t\t\tbl_thold_valid='" . $item['bl_thold_valid'] . "'\r\n\t\t\tWHERE rra_id='{$rra_id}' AND data_id=" . $item['data_id']);
            break;
        case 2:
            /* time based */
            if ($currentval != '') {
                $breach_up = $item['time_hi'] != '' && $currentval > $item['time_hi'];
                $breach_down = $item['time_low'] != '' && $currentval < $item['time_low'];
                $warning_breach_up = $item['time_warning_hi'] != '' && $currentval > $item['time_warning_hi'];
                $warning_breach_down = $item['time_warning_low'] != '' && $currentval < $item['time_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            $step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $rra_id, FALSE);
            /* alerts */
            $trigger = $item['time_fail_trigger'];
            $time = time() - $item['time_fail_length'] * $step;
            $failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_TRIGGERA . "," . ST_NOTIFYRA . "," . ST_NOTIFYAL . ") AND time>" . $time);
            /* warnings */
            $warning_trigger = $item['time_warning_fail_trigger'];
            $warning_time = time() - $item['time_warning_fail_length'] * $step;
            $warning_failures = db_fetch_cell("SELECT count(id) FROM plugin_thold_log WHERE threshold_id=" . $item['id'] . " AND status IN (" . ST_NOTIFYWA . "," . ST_TRIGGERW . ") AND time>" . $warning_time) + $failures;
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold Time Based check breached HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
                $item['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                $item['thold_fail_count'] = $failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($item['repeat_alert'] - 1) * $step;
                $lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYAL . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
                $ra = $failures > $trigger && $item['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $failures++;
                thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($failures == $trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? "ALERT: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $item['time_hi'] : $item['time_low']) . " with {$currentval}";
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($item['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $item['time_hi'] : $item['time_low'], $currentval, $trigger, $failures, $url);
                    }
                    if ($alert_command != '') {
                        exec_script($alert_command);
                    }
                    if (trim($alert_phones) != '') {
                        thold_sms($alert_phones, $subject);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $msg, $file_array);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones, 'command' => $alert_command));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                $item['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                $item['thold_warning_fail_count'] = $warning_failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($item['time_warning_fail_length'] - 1) * $step;
                $lastemailtime = db_fetch_cell("SELECT time\r\n\t\t\t\tFROM plugin_thold_log\r\n\t\t\t\tWHERE threshold_id=" . $item['id'] . "\r\n\t\t\t\tAND status IN (" . ST_NOTIFYRA . "," . ST_NOTIFYWA . ")\r\n\t\t\t\tORDER BY time DESC\r\n\t\t\t\tLIMIT 1", FALSE);
                $ra = $warning_failures > $warning_trigger && $item['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $warning_failures++;
                thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($warning_failures == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? "WARNING: " : "TRIGGER: ") . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low']) . " with {$currentval}";
                if ($notify) {
                    if ($logset == 1) {
                        logger($item['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $item['time_warning_hi'] : $item['time_warning_low'], $currentval, $warning_trigger, $warning_failures, $url);
                    }
                    if ($warning_command != '') {
                        exec_script($warning_command);
                    }
                    if (trim($warning_phones) != '') {
                        thold_sms($warning_phones, $subject);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones, 'command' => $warning_command));
                } elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
                    $subject = "ALERT -> WARNING: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $warning_phones));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => $warning_breach_up ? $item['time_hi'] : $item['time_low'], 'current' => $currentval, 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
                }
                db_execute("UPDATE thold_data\r\n\t\t\t\tSET thold_alert=" . $item['thold_alert'] . ",\r\n\t\t\t\tthold_warning_fail_count={$warning_failures},\r\n\t\t\t\tthold_fail_count={$failures}\r\n\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
            } else {
                thold_debug('Threshold Time Based check is normal HI:' . $item['time_hi'] . ' LOW:' . $item['time_low'] . ' VALUE:' . $currentval);
                if ($alertstat != 0 && $warning_failures < $warning_trigger && $item['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($item['name'], 'ok', 0, $currentval, $warning_trigger, $item['thold_warning_fail_count'], $url);
                    }
                    $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to normal threshold with value {$currentval}";
                    if (trim($warning_emails) != '' && $item['restored_alert'] != 'on') {
                        thold_mail($warning_emails, '', $subject, $msg, $file_array);
                    }
                    if (trim($warning_phones) != '' && $item['restored_alert'] != 'on') {
                        thold_sms($warning_phones, $subject);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails, 'phones' => $warning_phones));
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                } elseif ($alertstat != 0 && $failures < $trigger && $item['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($item['name'], 'ok', 0, $currentval, $trigger, $item['thold_fail_count'], $url);
                    }
                    $subject = "NORMAL: " . $item['name'] . ($thold_show_datasource ? " [{$name}]" : '') . " restored to warning threshold with value {$currentval}";
                    if (trim($alert_emails) != '' && $item['restored_alert'] != 'on') {
                        thold_mail($alert_emails, '', $subject, $msg, $file_array);
                    }
                    if (trim($alert_phones) != '' && $item['restored_alert'] != 'on') {
                        thold_sms($alert_phones, $subject);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $item['host_id'], 'graph_id' => $graph_id, 'threshold_id' => $item['id'], 'threshold_value' => '', 'current' => $currentval, 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails, 'phones' => $alert_phones));
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_alert=0, thold_warning_fail_count={$warning_failures}, thold_fail_count={$failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                } else {
                    db_execute("UPDATE thold_data\r\n\t\t\t\t\tSET thold_fail_count={$failures},\r\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\r\n\t\t\t\t\tWHERE rra_id={$rra_id} AND data_id=" . $item['data_id']);
                }
            }
            break;
    }
}
예제 #2
0
function thold_update_host_status () {
	global $config;
	// Return if we aren't set to notify
	$deadnotify = (read_config_option('alert_deadnotify') == 'on');
	if (!$deadnotify) return 0;
	include_once($config['base_path'] . '/plugins/thold/thold_functions.php');

	if (api_plugin_is_enabled('maint')) {
		include_once($config["base_path"] . '/plugins/maint/functions.php');
	}

	$alert_email = read_config_option('alert_email');
	$ping_failure_count = read_config_option('ping_failure_count');

	// Lets find hosts that were down, but are now back up
	$failed = db_fetch_assoc("SELECT * FROM plugin_thold_host_failed");
	if (sizeof($failed)) {
		foreach($failed as $fh) {
			if (!empty($fh['host_id'])) {
				if (api_plugin_is_enabled('maint')) {
					if (plugin_maint_check_cacti_host ($fh['host_id'])) {
						continue;
					}
				}
				$host = db_fetch_row('SELECT * FROM host WHERE id = ' . $fh['host_id']);
				if ($host['status'] == HOST_UP) {
					$snmp_system = '';
					$snmp_hostname = '';
					$snmp_location = '';
					$snmp_contact = '';
					$snmp_uptime = '';
					$uptimelong = '';
					$downtimemsg = '';

					if (($host["snmp_community"] == "" && $host["snmp_username"] == "") || $host["snmp_version"] == 0) {
						// SNMP not in use
						$snmp_system = 'SNMP not in use';
					} else {
						$snmp_system = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.1.0", $host["snmp_version"],
										$host["snmp_username"], $host["snmp_password"],
										$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
										$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);

						if (substr_count($snmp_system, "00:")) {
							$snmp_system = str_replace("00:", "", $snmp_system);
							$snmp_system = str_replace(":", " ", $snmp_system);
						}
						if ($snmp_system != "") {
							$snmp_uptime   = cacti_snmp_get($host['hostname'], $host['snmp_community'], ".1.3.6.1.2.1.1.3.0", $host['snmp_version'],
											$host['snmp_username'], $host['snmp_password'],
											$host['snmp_auth_protocol'], $host['snmp_priv_passphrase'], $host['snmp_priv_protocol'],
											$host['snmp_context'], $host['snmp_port'], $host['snmp_timeout'], read_config_option("snmp_retries"), SNMP_WEBUI);
							 $snmp_hostname = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.5.0", $host["snmp_version"],
											$host["snmp_username"], $host["snmp_password"],
											$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
											$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);
							 $snmp_location = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.6.0", $host["snmp_version"],
											$host["snmp_username"], $host["snmp_password"],
											$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
											$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);
							 $snmp_contact  = cacti_snmp_get($host["hostname"], $host["snmp_community"], ".1.3.6.1.2.1.1.4.0", $host["snmp_version"],
											$host["snmp_username"], $host["snmp_password"],
											$host["snmp_auth_protocol"], $host["snmp_priv_passphrase"], $host["snmp_priv_protocol"],
											$host["snmp_context"], $host["snmp_port"], $host["snmp_timeout"], read_config_option("snmp_retries"), SNMP_WEBUI);
							$days      = intval($snmp_uptime / (60*60*24*100));
							$remainder = $snmp_uptime % (60*60*24*100);
							$hours     = intval($remainder / (60*60*100));
							$remainder = $remainder % (60*60*100);
							$minutes   = intval($remainder / (60*100));
							$uptimelong = $days . "d " . $hours . "h " . $minutes . "m";
						}
						$downtime = time() - strtotime($host['status_fail_date']);
						$downtime_days = floor($downtime/86400);
						$downtime_hours = floor(($downtime - ($downtime_days * 86400))/3600);
						$downtime_minutes = floor(($downtime - ($downtime_days * 86400) - ($downtime_hours * 3600))/60);
						$downtime_seconds = $downtime - ($downtime_days * 86400) - ($downtime_hours * 3600) - ($downtime_minutes * 60);
						if ($downtime_days > 0 ) {
							$downtimemsg = $downtime_days . "d " . $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s ";
						} elseif ($downtime_hours > 0 ) {
							$downtimemsg = $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s";
						} elseif ($downtime_minutes > 0 ) {
							$downtimemsg = $downtime_minutes . "m " . $downtime_seconds . "s";
						} else {
							$downtimemsg = $downtime_seconds . "s ";
						}
					}


					$subject = read_config_option('thold_up_subject');
					if ($subject == '') {
						$subject = 'Host Notice: <DESCRIPTION> (<HOSTNAME>) returned from DOWN state';
					}
					$subject = str_replace('<HOSTNAME>', !$snmp_hostname ? $host['hostname'] : $snmp_hostname, $subject);
					$subject = str_replace('<DESCRIPTION>', $host['description'], $subject);
					$subject = str_replace('<DOWN/UP>', 'UP', $subject);
					$subject = strip_tags($subject);

					$msg = read_config_option('thold_up_text');
					if ($msg == '') {
						$msg = '<br>System <DESCRIPTION> (<HOSTNAME>) status: <DOWN/UP><br><br>Current ping response: <CUR_TIME> ms<br>Average system response : <AVG_TIME> ms<br>System availability: <AVAILABILITY><br>Total Checks Since Clear: <TOT_POLL><br>Total Failed Checks: <FAIL_POLL><br>Last Date Checked UP: <LAST_FAIL><br>Host Previously DOWN for: <DOWNTIME><br><br>Snmp Info:<br>Name - <SNMP_HOSTNAME><br>Location - <SNMP_LOCATION><br>Uptime - <UPTIMETEXT> (<UPTIME> ms)<br>System - <SNMP_SYSTEM><br><br>NOTE: <NOTES>';
					}
					$msg = str_replace('<SUBJECT>', $subject, $msg);
					$msg = str_replace('<HOSTNAME>', $host['hostname'], $msg);
					$msg = str_replace('<DESCRIPTION>', $host['description'], $msg);
					$msg = str_replace('<UPTIME>', $snmp_uptime, $msg);
					$msg = str_replace('<UPTIMETEXT>', $uptimelong, $msg);

					$msg = str_replace('<DOWNTIME>', $downtimemsg, $msg);
					$msg = str_replace('<MESSAGE>', '', $msg);
					$msg = str_replace('<DOWN/UP>', 'UP', $msg);

					$msg = str_replace('<SNMP_HOSTNAME>', $snmp_hostname, $msg);
					$msg = str_replace('<SNMP_LOCATION>', $snmp_location, $msg);
					$msg = str_replace('<SNMP_CONTACT>', $snmp_contact, $msg);
					$msg = str_replace('<SNMP_SYSTEM>', html_split_string($snmp_system), $msg);
					$msg = str_replace('<LAST_FAIL>', $host["status_fail_date"], $msg);
					$msg = str_replace('<AVAILABILITY>', round(($host["availability"]), 2) . ' %', $msg);
					$msg = str_replace('<TOT_POLL>', $host["total_polls"], $msg);
					$msg = str_replace('<FAIL_POLL>', $host["failed_polls"], $msg);
					$msg = str_replace('<CUR_TIME>', round(($host["cur_time"]), 2), $msg);
					$msg = str_replace('<AVG_TIME>', round(($host["avg_time"]), 2), $msg);
					$msg = str_replace('<NOTES>', $host["notes"], $msg);
					$msg = str_replace("\n", '<br>', $msg);
					switch($host['thold_send_email']) {
						case '0': // Disabled
							$alert_email = '';
							break;
						case '1': // Global List
							break;
						case '2': // Host List Only
							$alert_email = get_thold_notification_emails($host['thold_host_email']);
							break;
						case '3': // Global and Host List
							$alert_email = $alert_email . ',' . get_thold_notification_emails($host['thold_host_email']);
							break;
					}

					if ($alert_email == '' && $host['thold_send_email'] > 0) {
						cacti_log('THOLD: Can not send a Host Recovering Email since the \'Alert Email\' setting is not set !', true, 'POLLER');
					} elseif ($host['thold_send_email'] == '0') {
						cacti_log('THOLD: Did not send a Host Recovering Email, disabled per host setting : ' . $host['description'] . ' !', true, 'POLLER');
					} elseif ($alert_email != '') {
						thold_mail($alert_email, '', $subject, $msg, '');
					}
					thold_log(array(
						"type" => 3,
						"time" => time(),
						"host_id" => $host['id'],
						"graph_id" => 0,
						"threshold_id" => 0,
						"threshold_value" => 'Up/Down',
						"current" => 'Up',
						"status" => 5,
						"description" => 'Host Notice: '.$host['hostname'].' returned from DOWN state',
						"emails" => $alert_email
						)	
					);
				}
			}
		}
	}

	// Lets find hosts that are down
	$hosts = db_fetch_assoc('SELECT *
		FROM host
		WHERE disabled=""
		AND status=' . HOST_DOWN . '
		AND status_event_count=' . $ping_failure_count);

	$total_hosts = sizeof($hosts);
	if (count($hosts)) {
		foreach($hosts as $host) {
			if (api_plugin_is_enabled('maint')) {
				if (plugin_maint_check_cacti_host ($host['id'])) {
					continue;
				}
			}

			$downtime = time() - strtotime($host['status_rec_date']);
			$downtime_days = floor($downtime/86400);
			$downtime_hours = floor(($downtime - ($downtime_days * 86400))/3600);
			$downtime_minutes = floor(($downtime - ($downtime_days * 86400) - ($downtime_hours * 3600))/60);
			$downtime_seconds = $downtime - ($downtime_days * 86400) - ($downtime_hours * 3600) - ($downtime_minutes * 60);
			if ($downtime_days > 0 ) {
				$downtimemsg = $downtime_days . "d " . $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s ";
			} elseif ($downtime_hours > 0 ) {
				$downtimemsg = $downtime_hours . "h " . $downtime_minutes . "m " . $downtime_seconds . "s";
			} elseif ($downtime_minutes > 0 ) {
				$downtimemsg = $downtime_minutes . "m " . $downtime_seconds . "s";
			} else {
				$downtimemsg = $downtime_seconds . "s ";
			}
			
			/*$subject = read_config_option('thold_down_subject');
			if ($subject == '') {
				$subject = 'Host Error: <DESCRIPTION> (<HOSTNAME>) is DOWN';
			}
			$subject = str_replace('<HOSTNAME>', $host['hostname'], $subject); */

			$subject = read_config_option('thold_down_subject');
			if ($subject == '') {
				$subject = 'Host Error: <DESCRIPTION> (<HOSTNAME>) is DOWN';
			}
			$snmpsys = db_fetch_assoc('SELECT snmp_sysName, snmp_sysLocation, snmp_sysDescr FROM mac_track_devices WHERE hostname = \''.$host['hostname'].'\'');
			$subject = str_replace('<HOSTNAME>', !$snmpsys ? $host['hostname'] : $snmpsys[0]['snmp_sysName'], $subject);
			$subject = str_replace('<DESCRIPTION>', $host['description'], $subject);
			$subject = str_replace('<DOWN/UP>', 'DOWN', $subject);
			$subject = strip_tags($subject);

			$msg = read_config_option('thold_down_text');
			if ($msg == '') {
				$msg = 'System Error : <DESCRIPTION> (<HOSTNAME>) is <DOWN/UP><br>Reason: <MESSAGE><SNMP_HOSTNAME><SNMP_LOCATION><SNMP_SYSTEM><br><br>Average system response : <AVG_TIME> ms<br>System availability: <AVAILABILITY><br>Total Checks Since Clear: <TOT_POLL><br>Total Failed Checks: <FAIL_POLL><br>Last Date Checked DOWN : <LAST_FAIL><br>Host Previously UP for: <DOWNTIME><br>NOTE: <NOTES>';
			}
			$msg = str_replace('<SUBJECT>', $subject, $msg);
			$msg = str_replace('<HOSTNAME>', $host['hostname'], $msg);
			$msg = str_replace('<DESCRIPTION>', $host['description'], $msg);
			$msg = str_replace('<UPTIME>', '', $msg);
			$msg = str_replace('<DOWNTIME>', $downtimemsg, $msg);
			$msg = str_replace('<MESSAGE>', $host['status_last_error'], $msg);
			$msg = str_replace('<DOWN/UP>', 'DOWN', $msg);
			$msg = str_replace('<SNMP_HOSTNAME>', !$snmpsys ? '' : '<br><br>Device name: '.$snmpsys[0]['snmp_sysName'], $msg);
			$msg = str_replace('<SNMP_LOCATION>', !$snmpsys ? '' : '<br>Location: '.$snmpsys[0]['snmp_sysLocation'], $msg);
			$msg = str_replace('<SNMP_CONTACT>', '', $msg);
			$msg = str_replace('<SNMP_SYSTEM>', !$snmpsys ? '' : '<br>Description: '.$snmpsys[0]['snmp_sysDescr'], $msg);
			$msg = str_replace('<LAST_FAIL>', $host["status_fail_date"], $msg);
			$msg = str_replace('<AVAILABILITY>', round(($host["availability"]), 2) . ' %', $msg);
			$msg = str_replace('<CUR_TIME>', round(($host["cur_time"]), 2), $msg);
			$msg = str_replace('<TOT_POLL>', $host["total_polls"], $msg);
			$msg = str_replace('<FAIL_POLL>', $host["failed_polls"], $msg);
			$msg = str_replace('<AVG_TIME>', round(($host["avg_time"]), 2), $msg);
			$msg = str_replace('<NOTES>', $host["notes"], $msg);
			$msg = str_replace("\n", '<br>', $msg);
			switch($host['thold_send_email']) {
				case '0': // Disabled
					$alert_email = '';
					break;
				case '1': // Global List
					break;
				case '2': // Host List Only
					$alert_email = get_thold_notification_emails($host['thold_host_email']);
					break;
				case '3': // Global and Host List
					$alert_email = $alert_email . ',' . get_thold_notification_emails($host['thold_host_email']);
					break;
			}

			if ($alert_email == '' && $host['thold_send_email'] > 0) {
				cacti_log('THOLD: Can not send a Host Down Email since the \'Alert Email\' setting is not set !', true, 'POLLER');
			} elseif ($host['thold_send_email'] == '0') {
				cacti_log('THOLD: Did not send a Host Down Email, disabled per host setting : ' . $host['description'] . ' !', true, 'POLLER');
			} elseif ($alert_email != '') {
				thold_mail($alert_email, '', $subject, $msg, '');
			}
			thold_log(array(
				"type" => 3,
				"time" => time(),
				"host_id" => $host['id'],
				"graph_id" => 0,
				"threshold_id" => 0,
				"threshold_value" => 'Up/Down',
				"current" => 'Down',
				"status" => 4,
				"description" => 'Host Error: '.$host['hostname'].' is DOWN',
				"emails" => $alert_email
				)	
			);
		}
	}

	// Now lets record all failed hosts
	db_execute("TRUNCATE TABLE plugin_thold_host_failed");
	$hosts = db_fetch_assoc('SELECT id
		FROM host
		WHERE disabled=""
		AND status!=' . HOST_UP);

	$failed = '';
	if (sizeof($hosts)) {
		foreach ($hosts as $host) {
			if (api_plugin_is_enabled('maint')) {
				if (plugin_maint_check_cacti_host ($host['id'])) {
					continue;
				}
			}
			$failed .= (strlen($failed) ? "), (":"(") . $host['id'];
		}
		$failed .= ")";

		db_execute("INSERT INTO plugin_thold_host_failed (host_id) VALUES $failed");
	}

	return $total_hosts;
}
예제 #3
0
function thold_check_threshold(&$thold_data)
{
    global $config, $plugins, $debug, $thold_types;
    $name = db_fetch_cell_prepared('SELECT data_source_name FROM data_template_rrd WHERE id = ?', array($thold_data['data_template_rrd_id']));
    thold_debug('Checking Threshold:' . ' Name: ' . $name . ', local_data_id: ' . $thold_data['local_data_id'] . ', data_template_rrd_id: ' . $thold_data['data_template_rrd_id'] . ', value: ' . $thold_data['lastread']);
    $debug = false;
    // Do not proceed if we have chosen to globally disable all alerts
    if (read_config_option('thold_disable_all') == 'on') {
        thold_debug('Threshold checking is disabled globally');
        return;
    }
    $alert_exempt = read_config_option('alert_exempt');
    /* check for exemptions */
    $weekday = date('l');
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $alert_exempt == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* check for the weekend exemption on the threshold level */
    if (($weekday == 'Saturday' || $weekday == 'Sunday') && $thold_data['exempt'] == 'on') {
        thold_debug('Threshold checking is disabled by global weekend exemption');
        return;
    }
    /* don't alert for this host if it's selected for maintenance */
    if (api_plugin_is_enabled('maint') || in_array('maint', $plugins)) {
        include_once $config['base_path'] . '/plugins/maint/functions.php';
        if (plugin_maint_check_cacti_host($thold_data['host_id'])) {
            thold_debug('Threshold checking is disabled by maintenance schedule');
            return;
        }
    }
    $local_graph_id = $thold_data['local_graph_id'];
    /* only alert if Device is in UP mode (not down, unknown, or recovering) */
    $h = db_fetch_row('SELECT * FROM host WHERE id=' . $thold_data['host_id']);
    if (sizeof($h) && $h['status'] != 3) {
        thold_debug('Threshold checking halted by Device Status (' . $h['status'] . ')');
        return;
    }
    /* pull the cached name, if not present, it means that the graph hasn't polled yet */
    $t = db_fetch_assoc('SELECT id, name, name_cache
		FROM data_template_data
		WHERE local_data_id = ' . $thold_data['local_data_id'] . '
		ORDER BY id
		LIMIT 1');
    /* pull a few default settings */
    $global_alert_address = read_config_option('alert_email');
    $global_notify_enabled = read_config_option('alert_notify_default') == 'on';
    $logset = read_config_option('alert_syslog') == 'on';
    $deadnotify = read_config_option('alert_deadnotify') == 'on';
    $realert = read_config_option('alert_repeat');
    $alert_trigger = read_config_option('alert_trigger');
    $alert_bl_trigger = read_config_option('alert_bl_trigger');
    $httpurl = read_config_option('base_url');
    $thold_send_text_only = read_config_option('thold_send_text_only');
    $thold_snmp_traps = read_config_option('thold_alert_snmp') == 'on';
    $thold_snmp_warning_traps = read_config_option('thold_alert_snmp_warning') != 'on';
    $thold_snmp_normal_traps = read_config_option('thold_alert_snmp_normal') != 'on';
    $cacti_polling_interval = read_config_option('poller_interval');
    /* remove this after adding an option for it */
    $thold_show_datasource = thold_datasource_required($thold_data['name'], $name);
    $trigger = $thold_data['thold_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_fail_trigger'];
    $warning_trigger = $thold_data['thold_warning_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_warning_fail_trigger'];
    $alertstat = $thold_data['thold_alert'];
    $alert_emails = get_thold_alert_emails($thold_data);
    $warning_emails = get_thold_warning_emails($thold_data);
    $alert_msg = get_thold_alert_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
    $warn_msg = get_thold_warning_text($name, $thold_data, $h, $thold_data['lastread'], $thold_data['local_graph_id']);
    $thold_snmp_data = get_thold_snmp_data($name, $thold_data, $h, $thold_data['lastread']);
    $file_array = '';
    if ($thold_send_text_only != 'on') {
        if (!empty($thold_data['local_graph_id'])) {
            $file_array[] = array('local_graph_id' => $thold_data['local_graph_id'], 'local_data_id' => $thold_data['local_data_id'], 'rra_id' => 0, 'file' => "{$httpurl}/graph_image.php?local_graph_id=" . $thold_data['local_graph_id'] . '&rra_id=0&view_type=tree', 'mimetype' => 'image/png', 'filename' => clean_up_name($thold_data['name']));
        }
    }
    $url = $httpurl . '/graph.php?local_graph_id=' . $thold_data['local_graph_id'] . '&rra_id=all';
    switch ($thold_data['thold_type']) {
        case 0:
            /* hi/low */
            if ($thold_data['lastread'] != '') {
                $breach_up = $thold_data['thold_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_hi'];
                $breach_down = $thold_data['thold_low'] != '' && $thold_data['lastread'] < $thold_data['thold_low'];
                $warning_breach_up = $thold_data['thold_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['thold_warning_hi'];
                $warning_breach_down = $thold_data['thold_warning_low'] != '' && $thold_data['lastread'] < $thold_data['thold_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            /* is in alert status */
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low check breached HI:' . $thold_data['thold_hi'] . '  LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
                $thold_data['thold_fail_count']++;
                $thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                /* Re-Alert? */
                $ra = $thold_data['thold_fail_count'] > $trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_fail_count'] % $thold_data['repeat_alert'] == 0;
                if ($thold_data['thold_fail_count'] == $trigger || $ra) {
                    $notify = true;
                }
                $subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps) {
                        $thold_snmp_data['eventClass'] = 3;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
                        $thold_snmp_data['eventFailCount'] = $thold_data['thold_fail_count'];
                        $thold_snmp_data['eventFailDuration'] = $thold_data['thold_fail_count'] * $cacti_polling_interval;
                        $thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
                        $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ',
				thold_fail_count=' . $thold_data['thold_fail_count'] . ",\n\t\t\t\tthold_warning_fail_count=0\n\t\t\t\tWHERE id=" . $thold_data['id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                thold_debug('Threshold HI / Low Warning check breached HI:' . $thold_data['thold_warning_hi'] . '  LOW:' . $thold_data['thold_warning_low'] . ' VALUE:' . $thold_data['lastread']);
                $thold_data['thold_warning_fail_count']++;
                $thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                /* re-alert? */
                $ra = $thold_data['thold_warning_fail_count'] > $warning_trigger && $thold_data['repeat_alert'] != 0 && $thold_data['thold_warning_fail_count'] % $thold_data['repeat_alert'] == 0;
                if ($thold_data['thold_warning_fail_count'] == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($thold_data['name'], $ra ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
                    }
                    if (trim($warning_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_warning_traps) {
                        $thold_snmp_data['eventClass'] = 2;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
                        $thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
                        $thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
                        $thold_snmp_data['eventFailDurationTrigger'] = $warning_trigger * $cacti_polling_interval;
                        $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
                } elseif ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['thold_fail_count'] >= $trigger) {
                    $subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Changed to Warning Threshold with Value ' . $thold_data['lastread'];
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_warning_traps) {
                        $thold_snmp_data['eventClass'] = 2;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventNotificationType'] = ST_NOTIFYAW + 1;
                        $thold_snmp_data['eventFailCount'] = $thold_data['thold_warning_fail_count'];
                        $thold_snmp_data['eventFailDuration'] = $thold_data['thold_warning_fail_count'] * $cacti_polling_interval;
                        $thold_snmp_data['eventFailDurationTrigger'] = $trigger * $cacti_polling_interval;
                        $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['thold_warning_hi'] : $thold_data['thold_warning_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ',
				thold_warning_fail_count=' . $thold_data['thold_warning_fail_count'] . ',
				thold_fail_count=0
				WHERE id=' . $thold_data['id']);
            } else {
                thold_debug('Threshold HI / Low check is normal HI:' . $thold_data['thold_hi'] . '  LOW:' . $thold_data['thold_low'] . ' VALUE:' . $thold_data['lastread']);
                /* if we were at an alert status before */
                if ($alertstat != 0) {
                    $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' Restored to Normal Threshold with Value ' . $thold_data['lastread'];
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_fail_count=0, \n\t\t\t\t\tthold_warning_fail_count=0\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                    if ($thold_data['thold_warning_fail_count'] >= $warning_trigger && $thold_data['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
                        }
                        if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
                            thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                        }
                        if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                            $thold_snmp_data['eventClass'] = 1;
                            $thold_snmp_data['eventSeverity'] = 1;
                            $thold_snmp_data['eventStatus'] = 1;
                            $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                            thold_snmptrap($thold_snmp_data);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
                    } elseif ($thold_data['thold_fail_count'] >= $trigger && $thold_data['restored_alert'] != 'on') {
                        if ($logset == 1) {
                            logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
                        }
                        if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
                            thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                        }
                        if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                            $thold_snmp_data['eventClass'] = 1;
                            $thold_snmp_data['eventSeverity'] = 1;
                            $thold_snmp_data['eventStatus'] = 1;
                            $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                            thold_snmptrap($thold_snmp_data);
                        }
                        thold_log(array('type' => 0, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
                    }
                }
            }
            break;
        case 1:
            /* baseline */
            $bl_alert_prev = $thold_data['bl_alert'];
            $bl_count_prev = $thold_data['bl_fail_count'];
            $bl_fail_trigger = $thold_data['bl_fail_trigger'] == '' ? $alert_bl_trigger : $thold_data['bl_fail_trigger'];
            $thold_data['bl_alert'] = thold_check_baseline($local_data_id, $name, $thold_data['lastread'], $thold_data);
            switch ($thold_data['bl_alert']) {
                case -2:
                    /* exception is active, Future Release 'todo' */
                    break;
                case -1:
                    /* reference value not available, Future Release 'todo' */
                    break;
                case 0:
                    /* all clear */
                    /* if we were at an alert status before */
                    if ($alertstat != 0) {
                        thold_debug('Threshold Baseline check is normal');
                        if ($thold_data['bl_fail_count'] >= $bl_fail_trigger && $thold_data['restored_alert'] != 'on') {
                            thold_debug('Threshold Baseline check returned to normal');
                            if ($logset == 1) {
                                logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
                            }
                            $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
                            if (trim($alert_emails) != '') {
                                thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                            }
                            if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                                $thold_snmp_data['eventClass'] = 1;
                                $hold_snmp_data['eventSeverity'] = 1;
                                $thold_snmp_data['eventStatus'] = 1;
                                $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                                thold_snmptrap($thold_snmp_data);
                            }
                            thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
                        }
                    }
                    $thold_data['bl_fail_count'] = 0;
                    break;
                case 1:
                    /* value is below calculated threshold */
                /* value is below calculated threshold */
                case 2:
                    /* value is above calculated threshold */
                    $thold_data['bl_fail_count']++;
                    $breach_up = $thold_data['bl_alert'] == STAT_HI;
                    $breach_down = $thold_data['bl_alert'] == STAT_LO;
                    thold_debug('Threshold Baseline check breached');
                    /* re-alert? */
                    $ra = $thold_data['bl_fail_count'] > $bl_fail_trigger && $thold_data['bl_fail_count'] % ($thold_data['repeat_alert'] == '' ? $realert : $thold_data['repeat_alert']) == 0;
                    if ($thold_data['bl_fail_count'] == $bl_fail_trigger || $ra) {
                        thold_debug('Alerting is necessary');
                        $subject = 'ALERT: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($ra ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' calculated baseline threshold ' . ($breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low']) . ' with ' . $thold_data['lastread'];
                        if ($logset == 1) {
                            logger($thold_data['name'], $ra ? 'realert' : 'alert', $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], $thold_data['lastread'], $thold_data['bl_fail_trigger'], $thold_data['bl_fail_count'], $url);
                        }
                        if (trim($alert_emails) != '') {
                            thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                        }
                        if ($thold_snmp_traps) {
                            $thold_snmp_data['eventClass'] = 3;
                            $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
                            $thold_snmp_data['eventStatus'] = $thold_data['bl_alert'] + 1;
                            $thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
                            $thold_snmp_data['eventNotificationType'] = ($ra ? ST_NOTIFYRA : ST_NOTIFYAL) + 1;
                            $thold_snmp_data['eventFailCount'] = $thold_data['bl_fail_count'];
                            $thold_snmp_data['eventFailDuration'] = $thold_data['bl_fail_count'] * $cacti_polling_interval;
                            $thold_snmp_data['eventFailCountTrigger'] = $bl_fail_trigger;
                            $thold_snmp_data['eventDescription'] = str_replace(array('<FAIL_COUNT>', '<FAIL_DURATION>'), array($thold_snmp_data['eventFailCount'], $thold_snmp_data['eventFailDuration']), $thold_snmp_data['eventDescription']);
                            thold_snmptrap($thold_snmp_data);
                        }
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => $ra ? ST_NOTIFYRA : ST_NOTIFYAL, 'description' => $subject, 'emails' => $alert_emails));
                    } else {
                        thold_log(array('type' => 1, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['thold_hi'] : $thold_data['thold_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
                    }
                    break;
            }
            db_execute("UPDATE thold_data \n\t\t\tSET thold_alert=0, \n\t\t\tthold_fail_count=0,\n\t\t\tbl_alert='" . $thold_data['bl_alert'] . "',\n\t\t\tbl_fail_count='" . $thold_data['bl_fail_count'] . "',\n\t\t\tthold_low='" . $thold_data['thold_low'] . "',\n\t\t\tthold_hi='" . $thold_data['thold_hi'] . "',\n\t\t\tbl_thold_valid='" . $thold_data['bl_thold_valid'] . "'\n\t\t\tWHERE id=" . $thold_data['id']);
            break;
        case 2:
            /* time based */
            if ($thold_data['lastread'] != '') {
                $breach_up = $thold_data['time_hi'] != '' && $thold_data['lastread'] > $thold_data['time_hi'];
                $breach_down = $thold_data['time_low'] != '' && $thold_data['lastread'] < $thold_data['time_low'];
                $warning_breach_up = $thold_data['time_warning_hi'] != '' && $thold_data['lastread'] > $thold_data['time_warning_hi'];
                $warning_breach_down = $thold_data['time_warning_low'] != '' && $thold_data['lastread'] < $thold_data['time_warning_low'];
            } else {
                $breach_up = $breach_down = $warning_breach_up = $warning_breach_down = false;
            }
            $step = db_fetch_cell('SELECT rrd_step FROM data_template_data WHERE local_data_id = ' . $local_data_id, FALSE);
            /* alerts */
            $trigger = $thold_data['time_fail_trigger'];
            $time = time() - $thold_data['time_fail_length'] * $step;
            $failures = db_fetch_cell('SELECT count(id) 
			FROM plugin_thold_log 
			WHERE threshold_id=' . $thold_data['id'] . ' 
			AND status IN (' . ST_TRIGGERA . ',' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ') 
			AND time>' . $time);
            /* warnings */
            $warning_trigger = $thold_data['time_warning_fail_trigger'];
            $warning_time = time() - $thold_data['time_warning_fail_length'] * $step;
            $warning_failures = db_fetch_cell('SELECT count(id) 
			FROM plugin_thold_log 
			WHERE threshold_id=' . $thold_data['id'] . ' 
			AND status IN (' . ST_NOTIFYWA . ',' . ST_TRIGGERW . ') 
			AND time>' . $warning_time) + $failures;
            if ($breach_up || $breach_down) {
                $notify = false;
                thold_debug('Threshold Time Based check breached HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
                $thold_data['thold_alert'] = $breach_up ? STAT_HI : STAT_LO;
                $thold_data['thold_fail_count'] = $failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($thold_data['repeat_alert'] - 1) * $step;
                $lastemailtime = db_fetch_cell('SELECT time
				FROM plugin_thold_log
				WHERE threshold_id=' . $thold_data['id'] . '
				AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYAL . ')
				ORDER BY time DESC
				LIMIT 1', FALSE);
                $ra = $failures > $trigger && $thold_data['repeat_alert'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $failures++;
                thold_debug("Alert Time:'{$time}', Alert Trigger:'{$trigger}', Alert Failures:'{$failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($failures == $trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? 'ALERT: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($failures > $trigger ? 'is still' : 'went') . ' ' . ($breach_up ? 'above' : 'below') . ' threshold of ' . ($breach_up ? $thold_data['time_hi'] : $thold_data['time_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    thold_debug('Alerting is necessary');
                    if ($logset == 1) {
                        logger($thold_data['name'], $failures > $trigger ? 'realert' : 'alert', $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], $thold_data['lastread'], $trigger, $failures, $url);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps) {
                        $thold_snmp_data['eventClass'] = 3;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA) + 1;
                        $thold_snmp_data['eventFailCount'] = $failures;
                        $thold_snmp_data['eventFailCountTrigger'] = $trigger;
                        $thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $failures > $trigger ? ST_NOTIFYAL : ST_NOTIFYRA, 'description' => $subject, 'emails' => $alert_emails));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERA, 'description' => $subject, 'emails' => $alert_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
            } elseif ($warning_breach_up || $warning_breach_down) {
                $notify = false;
                $thold_data['thold_alert'] = $warning_breach_up ? STAT_HI : STAT_LO;
                $thold_data['thold_warning_fail_count'] = $warning_failures;
                /* we should only re-alert X minutes after last email, not every 5 pollings, etc...
                   re-alert? */
                $realerttime = ($thold_data['time_warning_fail_length'] - 1) * $step;
                $lastemailtime = db_fetch_cell('SELECT time
				FROM plugin_thold_log
				WHERE threshold_id=' . $thold_data['id'] . '
				AND status IN (' . ST_NOTIFYRA . ',' . ST_NOTIFYWA . ')
				ORDER BY time DESC
				LIMIT 1', FALSE);
                $ra = $warning_failures > $warning_trigger && $thold_data['time_warning_fail_length'] && !empty($lastemailtime) && $lastemailtime + $realerttime <= time();
                $warning_failures++;
                thold_debug("Warn Time:'{$warning_time}', Warn Trigger:'{$warning_trigger}', Warn Failures:'{$warning_failures}', RealertTime:'{$realerttime}', LastTime:'{$lastemailtime}', RA:'{$ra}', Diff:'" . ($realerttime + $lastemailtime) . "'<'" . time() . "'");
                if ($warning_failures == $warning_trigger || $ra) {
                    $notify = true;
                }
                $subject = ($notify ? 'WARNING: ' : 'TRIGGER: ') . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' ' . ($warning_failures > $warning_trigger ? 'is still' : 'went') . ' ' . ($warning_breach_up ? 'above' : 'below') . ' threshold of ' . ($warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low']) . ' with ' . $thold_data['lastread'];
                if ($notify) {
                    if ($logset == 1) {
                        logger($thold_data['name'], $warning_failures > $warning_trigger ? 'rewarning' : 'warning', $warning_breach_up ? $thold_data['time_warning_hi'] : $thold_data['time_warning_low'], $thold_data['lastread'], $warning_trigger, $warning_failures, $url);
                    }
                    if (trim($alert_emails) != '') {
                        thold_mail($warning_emails, '', $subject, $warn_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_warning_traps) {
                        $thold_snmp_data['eventClass'] = 2;
                        $thold_snmp_data['eventSeverity'] = $thold_data['snmp_event_warning_severity'];
                        $thold_snmp_data['eventStatus'] = $thold_data['thold_alert'] + 1;
                        $thold_snmp_data['eventRealertStatus'] = $ra ? $warning_breach_up ? 3 : 2 : 1;
                        $thold_snmp_data['eventNotificationType'] = ($warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA) + 1;
                        $thold_snmp_data['eventFailCount'] = $warning_failures;
                        $thold_snmp_data['eventFailCountTrigger'] = $warning_trigger;
                        $thold_snmp_data['eventDescription'] = str_replace('<FAIL_COUNT>', $thold_snmp_data['eventFailCount'], $thold_snmp_data['eventDescription']);
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => $warning_failures > $warning_trigger ? ST_NOTIFYRA : ST_NOTIFYWA, 'description' => $subject, 'emails' => $alert_emails));
                } elseif ($alertstat != 0 && $warning_failures < $warning_trigger && $failures < $trigger) {
                    $subject = 'ALERT -> WARNING: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYAW, 'description' => $subject, 'emails' => $alert_emails));
                } else {
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => $warning_breach_up ? $thold_data['time_hi'] : $thold_data['time_low'], 'current' => $thold_data['lastread'], 'status' => ST_TRIGGERW, 'description' => $subject, 'emails' => $warning_emails));
                }
                db_execute('UPDATE thold_data
				SET thold_alert=' . $thold_data['thold_alert'] . ",\n\t\t\t\tthold_warning_fail_count={$warning_failures},\n\t\t\t\tthold_fail_count={$failures}\n\t\t\t\tWHERE id=" . $thold_data['id']);
            } else {
                thold_debug('Threshold Time Based check is normal HI:' . $thold_data['time_hi'] . ' LOW:' . $thold_data['time_low'] . ' VALUE:' . $thold_data['lastread']);
                if ($alertstat != 0 && $warning_failures < $warning_trigger && $thold_data['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $warning_trigger, $thold_data['thold_warning_fail_count'], $url);
                    }
                    $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to normal threshold with value ' . $thold_data['lastread'];
                    if (trim($warning_emails) != '' && $thold_data['restored_alert'] != 'on') {
                        thold_mail($warning_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                        $thold_snmp_data['eventClass'] = 1;
                        $thold_snmp_data['eventSeverity'] = 1;
                        $thold_snmp_data['eventStatus'] = 1;
                        $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $warning_emails));
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                } elseif ($alertstat != 0 && $failures < $trigger && $thold_data['restored_alert'] != 'on') {
                    if ($logset == 1) {
                        logger($thold_data['name'], 'ok', 0, $thold_data['lastread'], $trigger, $thold_data['thold_fail_count'], $url);
                    }
                    $subject = 'NORMAL: ' . $thold_data['name'] . ($thold_show_datasource ? " [{$name}]" : '') . ' restored to warning threshold with value ' . $thold_data['lastread'];
                    if (trim($alert_emails) != '' && $thold_data['restored_alert'] != 'on') {
                        thold_mail($alert_emails, '', $subject, $alert_msg, $file_array);
                    }
                    if ($thold_snmp_traps && $thold_snmp_normal_traps) {
                        $thold_snmp_data['eventClass'] = 1;
                        $thold_snmp_data['eventSeverity'] = 1;
                        $thold_snmp_data['eventStatus'] = 1;
                        $thold_snmp_data['eventNotificationType'] = ST_NOTIFYRS + 1;
                        thold_snmptrap($thold_snmp_data);
                    }
                    thold_log(array('type' => 2, 'time' => time(), 'host_id' => $thold_data['host_id'], 'local_graph_id' => $thold_data['local_graph_id'], 'threshold_id' => $thold_data['id'], 'threshold_value' => '', 'current' => $thold_data['lastread'], 'status' => ST_NOTIFYRS, 'description' => $subject, 'emails' => $alert_emails));
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_alert=0, \n\t\t\t\t\tthold_warning_fail_count={$warning_failures}, \n\t\t\t\t\tthold_fail_count={$failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                } else {
                    db_execute("UPDATE thold_data\n\t\t\t\t\tSET thold_fail_count={$failures},\n\t\t\t\t\tthold_warning_fail_count={$warning_failures}\n\t\t\t\t\tWHERE id=" . $thold_data['id']);
                }
            }
            break;
    }
}