public function log($level, &$message) {
switch ($level) {
case LogHelper::LEVEL_DEBUG:
log_debug($message);
break;
case LogHelper::LEVEL_INFO:
log_info($message);
break;
case LogHelper::LEVEL_NOTICE:
log_notice($message);
break;
case LogHelper::LEVEL_WARNING:
log_warn($message);
break;
case LogHelper::LEVEL_ERROR:
log_error($message);
break;
case LogHelper::LEVEL_CRITICAL:
log_critical($message);
break;
case LogHelper::LEVEL_ALERT:
log_alert($message);
break;
case LogHelper::LEVEL_EMERGENCY:
log_emergency($message);
break;
}
}
function process_alerts($device)
{
global $config, $alert_rules, $alert_assoc;
echo "处理警报 " . $device['hostname'] . PHP_EOL;
$alert_table = cache_device_alert_table($device['device_id']);
$sql = "SELECT * FROM `alert_table`";
$sql .= " LEFT JOIN `alert_table-state` ON `alert_table`.`alert_table_id` = `alert_table-state`.`alert_table_id`";
$sql .= " WHERE `device_id` = ? AND `alert_status` IS NOT NULL;";
foreach (dbFetchRows($sql, array($device['device_id'])) as $entry) {
echo 'Alert: ' . $entry['alert_table_id'] . ' Status: ' . $entry['alert_status'] . ' ';
// If the alerter is now OK and has previously alerted, send an recovery notice.
if ($entry['alert_status'] == '1' && $entry['has_alerted'] == '1') {
$alert = $alert_rules[$entry['alert_test_id']];
if (!$alert['suppress_recovery']) {
$state = json_decode($entry['state'], TRUE);
$conditions = json_decode($alert['conditions'], TRUE);
$entity = get_entity_by_id_cache($entry['entity_type'], $entry['entity_id']);
$metric_array = array();
foreach ($state['metrics'] as $metric => $value) {
$metric_array[] = $metric . ' = ' . $value;
}
$message_tags = array('ALERT_STATE' => 'RECOVERY', 'ALERT_URL' => generate_url(array('page' => 'device', 'device' => $device['device_id'], 'tab' => 'alert', 'alert_entry' => $entry['alert_table_id'])), 'ALERT_MESSAGE' => $alert['alert_message'], 'METRICS' => implode(PHP_EOL . ' ', $metric_array), 'DURATION' => formatUptime(time() - $entry['last_failed']), 'ENTITY_LINK' => generate_entity_link($entry['entity_type'], $entry['entity_id'], $entity['entity_name']), 'ENTITY_NAME' => $entity['entity_name'], 'ENTITY_DESCRIPTION' => $entity['entity_descr'], 'DEVICE_HOSTNAME' => $device['hostname'], 'DEVICE_LINK' => generate_device_link($device), 'DEVICE_HARDWARE' => $device['hardware'], 'DEVICE_OS' => $device['os_text'] . ' ' . $device['version'] . ' ' . $device['features'], 'DEVICE_LOCATION' => $device['location'], 'DEVICE_UPTIME' => deviceUptime($device));
$message['text'] = simple_template('alert/email_text.tpl', $message_tags, array('is_file' => TRUE));
//$message_tags['CONDITIONS'] = nl2br($message_tags['CONDITIONS']);
$message_tags['METRICS'] = nl2br($message_tags['METRICS']);
$message['html'] = simple_template('alert/email_html.tpl', $message_tags, array('is_file' => TRUE));
//logfile('debug.log', var_export($message, TRUE));
alert_notify($device, alert_generate_subject('RECOVER', $device, $alert, $entity), $message, $entry['alert_test_id']);
log_alert('恢复发送的通知', $device, $entry, 'RECOVER_NOTIFY');
} else {
echo '已限制恢复.';
log_alert('限制恢复通知', $device, $entry, 'RECOVER_SUPPRESSED');
}
$update_array['last_recovered'] = time();
$update_array['has_alerted'] = 0;
dbUpdate($update_array, 'alert_table-state', '`alert_table_id` = ?', array($entry['alert_table_id']));
}
if ($entry['alert_status'] == '0') {
echo '警报已停止. ';
// Has this been alerted more frequently than the alert interval in the config?
/// FIXME -- this should be configurable per-entity or per-checker
if (time() - $entry['last_alerted'] < $config['alerts']['interval'] && !isset($GLOBALS['spam'])) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until set.
if (is_numeric($entry['ignore_until']) && $entry['ignore_until'] > time()) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until_ok set.
if (is_numeric($entry['ignore_until_ok']) && $entry['ignore_until_ok'] == '1') {
$entry['suppress_alert'] = TRUE;
}
if ($entry['suppress_alert'] != TRUE) {
echo 'Requires notification. ';
$alert = $alert_rules[$entry['alert_test_id']];
$state = json_decode($entry['state'], TRUE);
$conditions = json_decode($alert['conditions'], TRUE);
$entity = get_entity_by_id_cache($entry['entity_type'], $entry['entity_id']);
$condition_array = array();
foreach ($state['failed'] as $failed) {
$condition_array[] = $failed['metric'] . " " . $failed['condition'] . " " . $failed['value'] . " (" . $state['metrics'][$failed['metric']] . ")";
}
$graphs = "";
$metric_array = array();
foreach ($state['metrics'] as $metric => $value) {
$metric_array[] = $metric . ' = ' . $value;
}
if (is_array($config['entities'][$entry['entity_type']]['graph'])) {
// We can draw a graph for this type/metric pair!
$graph_array = $config['entities'][$entry['entity_type']]['graph'];
foreach ($graph_array as $key => $val) {
// Check to see if we need to do any substitution
if (substr($val, 0, 1) == "@") {
$nval = substr($val, 1);
echo " replaced " . $val . " with " . $entity[$nval] . " from entity. " . PHP_EOL . "<br />";
$graph_array[$key] = $entity[$nval];
}
}
//print_r($graph_array);
//logfile('debug.log', var_export($graph_array, TRUE));
$image_data_uri = generate_alert_graph($graph_array);
//print_r($image_data_uri);
//logfile('debug.log', var_export($image_data_uri, TRUE));
$graphs .= '<img src="' . $image_data_uri . '"><br />';
unset($graph_array);
}
$message_tags = array('ALERT_STATE' => 'ALERT', 'ALERT_URL' => generate_url(array('page' => 'device', 'device' => $device['device_id'], 'tab' => 'alert', 'alert_entry' => $entry['alert_table_id'])), 'ALERT_MESSAGE' => $alert['alert_message'], 'CONDITIONS' => implode(PHP_EOL . ' ', $condition_array), 'METRICS' => implode(PHP_EOL . ' ', $metric_array), 'DURATION' => formatUptime(time() - $entry['last_failed']), 'ENTITY_LINK' => generate_entity_link($entry['entity_type'], $entry['entity_id'], $entity['entity_name']), 'ENTITY_NAME' => $entity['entity_name'], 'ENTITY_DESCRIPTION' => $entity['entity_descr'], 'ENTITY_GRAPHS' => $graphs, 'DEVICE_HOSTNAME' => $device['hostname'], 'DEVICE_LINK' => generate_device_link($device), 'DEVICE_HARDWARE' => $device['hardware'], 'DEVICE_OS' => $device['os_text'] . ' ' . $device['version'] . ' ' . $device['features'], 'DEVICE_LOCATION' => $device['location'], 'DEVICE_UPTIME' => deviceUptime($device));
$message['text'] = simple_template('alert/email_text.tpl', $message_tags, array('is_file' => TRUE));
$message_tags['CONDITIONS'] = nl2br($message_tags['CONDITIONS']);
$message_tags['METRICS'] = nl2br($message_tags['METRICS']);
$message['html'] = simple_template('alert/email_html.tpl', $message_tags, array('is_file' => TRUE));
//logfile('debug.log', var_export($message, TRUE));
alert_notify($device, alert_generate_subject('ALERT', $device, $alert, $entity), $message, $entry['alert_test_id']);
log_alert('Alert notification sent', $device, $entry, 'ALERT_NOTIFY');
$update_array['last_alerted'] = time();
$update_array['has_alerted'] = 1;
dbUpdate($update_array, 'alert_table-state', '`alert_table_id` = ?', array($entry['alert_table_id']));
} else {
echo "没有通知要求. " . (time() - $entry['last_alerted']);
}
} else {
if ($entry['alert_status'] == '1') {
echo "状态: OK. ";
} else {
if ($entry['alert_status'] == '2') {
echo "Status: Notification Delayed. ";
} else {
if ($entry['alert_status'] == '3') {
echo "Status: Notification Suppressed. ";
} else {
echo "未知的状态.";
}
}
}
}
echo PHP_EOL;
}
}
function process_alerts($device)
{
global $config, $alert_rules, $alert_assoc;
$pid_info = check_process_run($device);
// This just clear stalled DB entries
add_process_info($device);
// Store process info
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "]", 1);
$alert_table = cache_device_alert_table($device['device_id']);
$sql = "SELECT * FROM `alert_table`";
//$sql .= " LEFT JOIN `alert_table-state` ON `alert_table`.`alert_table_id` = `alert_table-state`.`alert_table_id`";
$sql .= " WHERE `device_id` = ? AND `alert_status` IS NOT NULL;";
foreach (dbFetchRows($sql, array($device['device_id'])) as $entry) {
print_cli_data_field('Alert: ' . $entry['alert_table_id']);
print_cli('Status: [' . $entry['alert_status'] . '] ', 'color');
// If the alerter is now OK and has previously alerted, send an recovery notice.
if ($entry['alert_status'] == '1' && $entry['has_alerted'] == '1') {
$alert = $alert_rules[$entry['alert_test_id']];
if (!$alert['suppress_recovery']) {
alert_notifier($entry, "recovery");
$log_id = log_alert('Recovery notification sent', $device, $entry, 'RECOVER_NOTIFY');
} else {
echo 'Recovery suppressed.';
$log_id = log_alert('Recovery notification suppressed', $device, $entry, 'RECOVER_SUPPRESSED');
}
$update_array['last_recovered'] = time();
$update_array['has_alerted'] = 0;
dbUpdate($update_array, 'alert_table', '`alert_table_id` = ?', array($entry['alert_table_id']));
}
if ($entry['alert_status'] == '0') {
echo 'Alert tripped. ';
// Has this been alerted more frequently than the alert interval in the config?
/// FIXME -- this should be configurable per-entity or per-checker
if (time() - $entry['last_alerted'] < $config['alerts']['interval'] && !isset($GLOBALS['spam'])) {
$entry['suppress_alert'] = TRUE;
}
// Don't re-alert if interval set to 0
if ($config['alerts']['interval'] == 0 && $entry['last_alerted'] != 0) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until set.
if (is_numeric($entry['ignore_until']) && $entry['ignore_until'] > time()) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until_ok set.
if (is_numeric($entry['ignore_until_ok']) && $entry['ignore_until_ok'] == '1') {
$entry['suppress_alert'] = TRUE;
}
if ($entry['suppress_alert'] != TRUE) {
echo 'Requires notification. ';
alert_notifier($entry, "alert");
$log_id = log_alert('Alert notification sent', $device, $entry, 'ALERT_NOTIFY');
$update_array['last_alerted'] = time();
$update_array['has_alerted'] = 1;
dbUpdate($update_array, 'alert_table', '`alert_table_id` = ?', array($entry['alert_table_id']));
} else {
echo "No notification required. " . (time() - $entry['last_alerted']);
}
} else {
if ($entry['alert_status'] == '1') {
echo "Status: OK. ";
} else {
if ($entry['alert_status'] == '2') {
echo "Status: Notification Delayed. ";
} else {
if ($entry['alert_status'] == '3') {
echo "Status: Notification Suppressed. ";
} else {
echo "Unknown status.";
}
}
}
}
echo PHP_EOL;
}
echo PHP_EOL;
print_cli_heading($device['hostname'] . " [" . $device['device_id'] . "] completed notifications at " . date("Y-m-d H:i:s"), 1);
// Clean
del_process_info($device);
// Remove process info
}