function process_alerts($device)
{
global $config, $alert_rules, $alert_assoc;
echo "处理警报 " . $device['hostname'] . PHP_EOL;
$alert_table = cache_device_alert_table($device['device_id']);
$sql = "SELECT * FROM `alert_table`";
$sql .= " LEFT JOIN `alert_table-state` ON `alert_table`.`alert_table_id` = `alert_table-state`.`alert_table_id`";
$sql .= " WHERE `device_id` = ? AND `alert_status` IS NOT NULL;";
foreach (dbFetchRows($sql, array($device['device_id'])) as $entry) {
echo 'Alert: ' . $entry['alert_table_id'] . ' Status: ' . $entry['alert_status'] . ' ';
// If the alerter is now OK and has previously alerted, send an recovery notice.
if ($entry['alert_status'] == '1' && $entry['has_alerted'] == '1') {
$alert = $alert_rules[$entry['alert_test_id']];
if (!$alert['suppress_recovery']) {
$state = json_decode($entry['state'], TRUE);
$conditions = json_decode($alert['conditions'], TRUE);
$entity = get_entity_by_id_cache($entry['entity_type'], $entry['entity_id']);
$metric_array = array();
foreach ($state['metrics'] as $metric => $value) {
$metric_array[] = $metric . ' = ' . $value;
}
$message_tags = array('ALERT_STATE' => 'RECOVERY', 'ALERT_URL' => generate_url(array('page' => 'device', 'device' => $device['device_id'], 'tab' => 'alert', 'alert_entry' => $entry['alert_table_id'])), 'ALERT_MESSAGE' => $alert['alert_message'], 'METRICS' => implode(PHP_EOL . ' ', $metric_array), 'DURATION' => formatUptime(time() - $entry['last_failed']), 'ENTITY_LINK' => generate_entity_link($entry['entity_type'], $entry['entity_id'], $entity['entity_name']), 'ENTITY_NAME' => $entity['entity_name'], 'ENTITY_DESCRIPTION' => $entity['entity_descr'], 'DEVICE_HOSTNAME' => $device['hostname'], 'DEVICE_LINK' => generate_device_link($device), 'DEVICE_HARDWARE' => $device['hardware'], 'DEVICE_OS' => $device['os_text'] . ' ' . $device['version'] . ' ' . $device['features'], 'DEVICE_LOCATION' => $device['location'], 'DEVICE_UPTIME' => deviceUptime($device));
$message['text'] = simple_template('alert/email_text.tpl', $message_tags, array('is_file' => TRUE));
//$message_tags['CONDITIONS'] = nl2br($message_tags['CONDITIONS']);
$message_tags['METRICS'] = nl2br($message_tags['METRICS']);
$message['html'] = simple_template('alert/email_html.tpl', $message_tags, array('is_file' => TRUE));
//logfile('debug.log', var_export($message, TRUE));
alert_notify($device, alert_generate_subject('RECOVER', $device, $alert, $entity), $message, $entry['alert_test_id']);
log_alert('恢复发送的通知', $device, $entry, 'RECOVER_NOTIFY');
} else {
echo '已限制恢复.';
log_alert('限制恢复通知', $device, $entry, 'RECOVER_SUPPRESSED');
}
$update_array['last_recovered'] = time();
$update_array['has_alerted'] = 0;
dbUpdate($update_array, 'alert_table-state', '`alert_table_id` = ?', array($entry['alert_table_id']));
}
if ($entry['alert_status'] == '0') {
echo '警报已停止. ';
// Has this been alerted more frequently than the alert interval in the config?
/// FIXME -- this should be configurable per-entity or per-checker
if (time() - $entry['last_alerted'] < $config['alerts']['interval'] && !isset($GLOBALS['spam'])) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until set.
if (is_numeric($entry['ignore_until']) && $entry['ignore_until'] > time()) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until_ok set.
if (is_numeric($entry['ignore_until_ok']) && $entry['ignore_until_ok'] == '1') {
$entry['suppress_alert'] = TRUE;
}
if ($entry['suppress_alert'] != TRUE) {
echo 'Requires notification. ';
$alert = $alert_rules[$entry['alert_test_id']];
$state = json_decode($entry['state'], TRUE);
$conditions = json_decode($alert['conditions'], TRUE);
$entity = get_entity_by_id_cache($entry['entity_type'], $entry['entity_id']);
$condition_array = array();
foreach ($state['failed'] as $failed) {
$condition_array[] = $failed['metric'] . " " . $failed['condition'] . " " . $failed['value'] . " (" . $state['metrics'][$failed['metric']] . ")";
}
$graphs = "";
$metric_array = array();
foreach ($state['metrics'] as $metric => $value) {
$metric_array[] = $metric . ' = ' . $value;
}
if (is_array($config['entities'][$entry['entity_type']]['graph'])) {
// We can draw a graph for this type/metric pair!
$graph_array = $config['entities'][$entry['entity_type']]['graph'];
foreach ($graph_array as $key => $val) {
// Check to see if we need to do any substitution
if (substr($val, 0, 1) == "@") {
$nval = substr($val, 1);
echo " replaced " . $val . " with " . $entity[$nval] . " from entity. " . PHP_EOL . "<br />";
$graph_array[$key] = $entity[$nval];
}
}
//print_r($graph_array);
//logfile('debug.log', var_export($graph_array, TRUE));
$image_data_uri = generate_alert_graph($graph_array);
//print_r($image_data_uri);
//logfile('debug.log', var_export($image_data_uri, TRUE));
$graphs .= '<img src="' . $image_data_uri . '"><br />';
unset($graph_array);
}
$message_tags = array('ALERT_STATE' => 'ALERT', 'ALERT_URL' => generate_url(array('page' => 'device', 'device' => $device['device_id'], 'tab' => 'alert', 'alert_entry' => $entry['alert_table_id'])), 'ALERT_MESSAGE' => $alert['alert_message'], 'CONDITIONS' => implode(PHP_EOL . ' ', $condition_array), 'METRICS' => implode(PHP_EOL . ' ', $metric_array), 'DURATION' => formatUptime(time() - $entry['last_failed']), 'ENTITY_LINK' => generate_entity_link($entry['entity_type'], $entry['entity_id'], $entity['entity_name']), 'ENTITY_NAME' => $entity['entity_name'], 'ENTITY_DESCRIPTION' => $entity['entity_descr'], 'ENTITY_GRAPHS' => $graphs, 'DEVICE_HOSTNAME' => $device['hostname'], 'DEVICE_LINK' => generate_device_link($device), 'DEVICE_HARDWARE' => $device['hardware'], 'DEVICE_OS' => $device['os_text'] . ' ' . $device['version'] . ' ' . $device['features'], 'DEVICE_LOCATION' => $device['location'], 'DEVICE_UPTIME' => deviceUptime($device));
$message['text'] = simple_template('alert/email_text.tpl', $message_tags, array('is_file' => TRUE));
$message_tags['CONDITIONS'] = nl2br($message_tags['CONDITIONS']);
$message_tags['METRICS'] = nl2br($message_tags['METRICS']);
$message['html'] = simple_template('alert/email_html.tpl', $message_tags, array('is_file' => TRUE));
//logfile('debug.log', var_export($message, TRUE));
alert_notify($device, alert_generate_subject('ALERT', $device, $alert, $entity), $message, $entry['alert_test_id']);
log_alert('Alert notification sent', $device, $entry, 'ALERT_NOTIFY');
$update_array['last_alerted'] = time();
$update_array['has_alerted'] = 1;
dbUpdate($update_array, 'alert_table-state', '`alert_table_id` = ?', array($entry['alert_table_id']));
} else {
echo "没有通知要求. " . (time() - $entry['last_alerted']);
}
} else {
if ($entry['alert_status'] == '1') {
echo "状态: OK. ";
} else {
if ($entry['alert_status'] == '2') {
echo "Status: Notification Delayed. ";
} else {
if ($entry['alert_status'] == '3') {
echo "Status: Notification Suppressed. ";
} else {
echo "未知的状态.";
}
}
}
}
echo PHP_EOL;
}
}
/**
* Check all alerts for a device to see if they should be notified or not
*
* @param array device
* @return NULL
*/
function process_alerts($device)
{
global $config, $alert_rules, $alert_assoc;
echo "Processing alerts for " . $device['hostname'] . PHP_EOL;
$alert_table = cache_device_alert_table($device['device_id']);
$sql = "SELECT * FROM `alert_table`";
$sql .= " LEFT JOIN `alert_table-state` ON `alert_table`.`alert_table_id` = `alert_table-state`.`alert_table_id`";
$sql .= " WHERE `device_id` = ?";
foreach (dbFetchRows($sql, array($device['device_id'])) as $entry) {
echo 'Alert: ' . $entry['alert_table_id'] . ' Status: ' . $entry['alert_status'] . ' ';
// If the alerter is now OK and has previously alerted, send an recovery notice.
if ($entry['alert_status'] == '1' && $entry['has_alerted'] == '1') {
$alert = $alert_rules[$entry['alert_test_id']];
$state = json_decode($entry['state'], TRUE);
$conditions = json_decode($alert['conditions'], TRUE);
$entity = get_entity_by_id_cache($entry['entity_type'], $entry['entity_id']);
$graphs = "";
$metric_text = "";
foreach ($state['metrics'] as $metric => $value) {
$metric_text .= $metric . " = " . $value . PHP_EOL . "<br />";
}
// FIXME De-dup this shit soon.
// - adama
$message = '
<head>
<title>Observium Alert</title>
<style>
.observium{ width:100%; max-width: 500px; -webkit-border-radius: 5px; -moz-border-radius: 5px; border-radius: 5px; border:1px solid #DDDDDD; background-color:#FAFAFA;
font-size: 13px; color: #777777; }
.header{ font-weight: bold; font-size: 16px; padding: 5px; color: #555555; }
.red { color: #cc0000; }
#deviceinfo tr:nth-child(odd) { background: #ffffff; }
</style>
<style type="text/css"></style></head>
<body>
<table class="observium">
<tbody>
<tr>
<td>
<table class="observium" id="deviceinfo">
<tbody>
<tr><td class="header">RECOVERY</td><td><a style="float: right;" href="' . generate_url(array('page' => 'device', 'device' => $device['device_id'], 'tab' => 'alert', 'alert_entry' => $entry['alert_table_id'])) . '">Modify</a></td></tr>
<tr><td><b>Alert</b></font></td><td class="red">' . $alert['alert_message'] . '</font></td></tr>
<tr><td><b>Entity</b></font></td><td>' . generate_entity_link($entry['entity_type'], $entry['entity_id'], $entity['entity_name']) . '</font></td></tr>';
if (strlen($entity['entity_descr']) > 0) {
$message .= '<tr><td><b>Descr</b></font></td><td>' . $entity['entity_descr'] . '</font>';
}
$message .= '
<tr><td><b>Metrics</b></font></td><td>' . $metric_text . '</font></td></tr>
<tr><td><b>Duration</b></font></td><td>' . formatUptime(time() - $entry['last_failed']) . '</font></td></tr>
<tr><td colspan="2" class="header">Device</td></tr>
<tr><td><b>Device</b></font></td><td>' . generate_device_link($device) . '</font></td></tr>
<tr><td><b>Hardware</b></font></td><td>' . $device['hardware'] . '</font></td></tr>
<tr><td><b>Operating System</b></font></td><td>' . $device['os_text'] . ' ' . $device['version'] . ' ' . $device['features'] . '</font></td></tr>
<tr><td><b>Location</b></font></td><td>' . htmlspecialchars($device['location']) . '</font></td></tr>
<tr><td><b>Uptime</b></font></td><td>' . deviceUptime($device) . '</font></td></tr>
</tbody></table>
</td></tr>
<tr><td>
<center>' . $graphs . '</center></td></tr>
</tbody></table>
</body>
</html>';
alert_notify($device, "RECOVER: [" . $device['hostname'] . "] [" . $alert['entity_type'] . "] [" . $entity['entity_name'] . "] " . $alert['alert_message'], $message);
$update_array['last_recovered'] = time();
$update_array['has_alerted'] = 0;
dbUpdate($update_array, 'alert_table-state', '`alert_table_id` = ?', array($entry['alert_table_id']));
}
if ($entry['alert_status'] == '0') {
echo 'Alert tripped. ';
// Has this been alerted more frequently than the alert interval in the config?
/// FIXME -- this should be configurable per-entity or per-checker
if (time() - $entry['last_alerted'] < $config['alerts']['interval'] && !isset($GLOBALS['spam'])) {
$entry['suppress_alert'] = TRUE;
}
// Check if alert has ignore_until set.
if (is_numeric($entry['ignore_until']) && $entry['ignore_until'] > time()) {
$entry['suppress_alert'] = TRUE;
}
if ($entry['suppress_alert'] != TRUE) {
echo 'Requires notification. ';
$alert = $alert_rules[$entry['alert_test_id']];
$state = json_decode($entry['state'], TRUE);
$conditions = json_decode($alert['conditions'], TRUE);
$entity = get_entity_by_id_cache($entry['entity_type'], $entry['entity_id']);
$condition_text = "";
foreach ($state['failed'] as $failed) {
$condition_text .= $failed['metric'] . " " . $failed['condition'] . " " . $failed['value'] . " (" . $state['metrics'][$failed['metric']] . ")<br />";
}
$graphs = "";
$metric_text = "";
foreach ($state['metrics'] as $metric => $value) {
$metric_text .= $metric . " = " . $value . PHP_EOL . "<br />";
}
if (is_array($config['entities'][$entry['entity_type']]['graph'])) {
// We can draw a graph for this type/metric pair!
$graph_array = $config['entities'][$entry['entity_type']]['graph'];
foreach ($graph_array as $key => $val) {
// Check to see if we need to do any substitution
if (substr($val, 0, 1) == "@") {
$nval = substr($val, 1);
echo " replaced " . $val . " with " . $entity[$nval] . " from entity. " . PHP_EOL . "<br />";
$graph_array[$key] = $entity[$nval];
}
}
print_r($graph_array);
$image_data_uri = generate_alert_graph($graph_array);
print_r(strlen($image_data_uri));
$graphs .= '<img src="' . $image_data_uri . '">' . "<br />";
unset($graph_array);
}
#$css = data_uri($config['html_dir'].'/css/bootstrap-mini.css' ,'text/css');
$message = '
<head>
<title>Observium Alert</title>
<style>
.observium{ width:100%; max-width: 500px; -webkit-border-radius: 5px; -moz-border-radius: 5px; border-radius: 5px; border:1px solid #DDDDDD; background-color:#FAFAFA;
font-size: 13px; color: #777777; }
.header{ font-weight: bold; font-size: 16px; padding: 5px; color: #555555; }
.red { color: #cc0000; }
#deviceinfo tr:nth-child(odd) { background: #ffffff; }
</style>
<style type="text/css"></style></head>
<body>
<table class="observium">
<tbody>
<tr>
<td>
<table class="observium" id="deviceinfo">
<tbody>
<tr><td class="header">ALERT</td><td><a style="float: right;" href="' . generate_url(array('page' => 'device', 'device' => $device['device_id'], 'tab' => 'alert', 'alert_entry' => $entry['alert_table_id'])) . '">Modify</a></td></tr>
<tr><td><b>Alert</b></font></td><td class="red">' . $alert['alert_message'] . '</font></td></tr>
<tr><td><b>Entity</b></font></td><td>' . generate_entity_link($entry['entity_type'], $entry['entity_id'], $entity['entity_name']) . '</font></td></tr>';
if (strlen($entity['entity_descr']) > 0) {
$message .= '<tr><td><b>Descr</b></font></td><td>' . $entity['entity_descr'] . '</font>';
}
$message .= '
<tr><td><b>Conditions</b></font></td><td>' . $condition_text . '</font></td></tr>
<tr><td><b>Metrics</b></font></td><td>' . $metric_text . '</font></td></tr>
<tr><td><b>Duration</b></font></td><td>' . formatUptime(time() - $entry['last_failed']) . '</font></td></tr>
<tr><td colspan="2" class="header">Device</td></tr>
<tr><td><b>Device</b></font></td><td>' . generate_device_link($device) . '</font></td></tr>
<tr><td><b>Hardware</b></font></td><td>' . $device['hardware'] . '</font></td></tr>
<tr><td><b>Operating System</b></font></td><td>' . $device['os_text'] . ' ' . $device['version'] . ' ' . $device['features'] . '</font></td></tr>
<tr><td><b>Location</b></font></td><td>' . htmlspecialchars($device['location']) . '</font></td></tr>
<tr><td><b>Uptime</b></font></td><td>' . deviceUptime($device) . '</font></td></tr>
</tbody></table>
</td></tr>
<tr><td>
<center>' . $graphs . '</center></td></tr>
</tbody></table>
</body>
</html>';
alert_notify($device, "ALERT: [" . $device['hostname'] . "] [" . $alert['entity_type'] . "] [" . $entity['entity_name'] . "] " . $alert['alert_message'], $message);
$update_array['last_alerted'] = time();
$update_array['has_alerted'] = 1;
dbUpdate($update_array, 'alert_table-state', '`alert_table_id` = ?', array($entry['alert_table_id']));
} else {
echo "No notification required. " . (time() - $entry['last_alerted']);
}
} elseif ($entry['alert_status'] == '1') {
echo "Status: OK. ";
} else {
echo "Unknown status.";
}
echo PHP_EOL;
}
}
/**
* Generate notifications for an alert entry
*
* @param array entry
* @return NULL
*/
function alert_notifier($entry, $type = "alert")
{
global $config, $alert_rules;
$device = device_by_id_cache($entry['device_id']);
$alert = $alert_rules[$entry['alert_test_id']];
$state = json_decode($entry['state'], TRUE);
$conditions = json_decode($alert['conditions'], TRUE);
$entity = get_entity_by_id_cache($entry['entity_type'], $entry['entity_id']);
$condition_array = array();
foreach ($state['failed'] as $failed) {
$condition_array[] = $failed['metric'] . " " . $failed['condition'] . " " . $failed['value'] . " (" . $state['metrics'][$failed['metric']] . ")";
}
$metric_array = array();
foreach ($state['metrics'] as $metric => $value) {
$metric_array[] = $metric . ' = ' . $value;
}
$graphs = array();
$graph_done = array();
foreach ($state['metrics'] as $metric => $value) {
if ($config['email']['graphs'] !== FALSE && is_array($config['entities'][$entry['entity_type']]['metric_graphs'][$metric]) && !in_array($config['entities'][$entry['entity_type']]['metric_graphs'][$metric]['type'], $graph_done)) {
$graph_array = $config['entities'][$entry['entity_type']]['metric_graphs'][$metric];
foreach ($graph_array as $key => $val) {
// Check to see if we need to do any substitution
if (substr($val, 0, 1) == '@') {
$nval = substr($val, 1);
//echo(" replaced " . $val . " with " . $entity[$nval] . " from entity. " . PHP_EOL . "<br />");
$graph_array[$key] = $entity[$nval];
}
}
$image_data_uri = generate_alert_graph($graph_array);
$image_url = generate_graph_url($graph_array);
$graphs[] = array('label' => $graph_array['type'], 'type' => $graph_array['type'], 'url' => $image_url, 'data' => $image_data_uri);
$graph_done[] = $graph_array['type'];
}
unset($graph_array);
}
if ($config['email']['graphs'] !== FALSE && count($graph_done) == 0 && is_array($config['entities'][$entry['entity_type']]['graph'])) {
// We can draw a graph for this type/metric pair!
$graph_array = $config['entities'][$entry['entity_type']]['graph'];
foreach ($graph_array as $key => $val) {
// Check to see if we need to do any substitution
if (substr($val, 0, 1) == '@') {
$nval = substr($val, 1);
//echo(" replaced ".$val." with ". $entity[$nval] ." from entity. ".PHP_EOL."<br />");
$graph_array[$key] = $entity[$nval];
}
}
//print_vars($graph_array);
$image_data_uri = generate_alert_graph($graph_array);
$image_url = generate_graph_url($graph_array);
$graphs[] = array('label' => $graph_array['type'], 'type' => $graph_array['type'], 'url' => $image_url, 'data' => $image_data_uri);
unset($graph_array);
}
$graphs_html = "";
foreach ($graphs as $graph) {
$graphs_html .= '<h4>' . $graph['type'] . '</h4>';
$graphs_html .= '<a href="' . $graph['url'] . '"><img src="' . $graph['data'] . '"></a><br />';
}
//print_vars($graphs);
//print_vars($graphs_html);
//print_vars($entry);
$message_tags = array('ALERT_STATE' => $entry['alert_status'] == '1' ? "RECOVER" : "ALERT", 'ALERT_URL' => generate_url(array('page' => 'device', 'device' => $device['device_id'], 'tab' => 'alert', 'alert_entry' => $entry['alert_table_id'])), 'ALERT_ID' => $entry['alert_table_id'], 'ALERT_MESSAGE' => $alert['alert_message'], 'CONDITIONS' => implode(PHP_EOL . ' ', $condition_array), 'METRICS' => implode(PHP_EOL . ' ', $metric_array), 'DURATION' => $entry['alert_status'] == '1' ? $entry['last_ok'] > 0 ? formatUptime(time() - $entry['last_ok']) . " (" . format_unixtime($entry['last_ok']) . ")" : "Unknown" : ($entry['last_ok'] > 0 ? formatUptime(time() - $entry['last_ok']) . " (" . format_unixtime($entry['last_ok']) . ")" : "Unknown"), 'ENTITY_LINK' => generate_entity_link($entry['entity_type'], $entry['entity_id'], $entity['entity_name']), 'ENTITY_NAME' => $entity['entity_name'], 'ENTITY_TYPE' => $alert['entity_type'], 'ENTITY_DESCRIPTION' => $entity['entity_descr'], 'ENTITY_GRAPHS_ARRAY' => json_encode($graphs), 'DEVICE_HOSTNAME' => $device['hostname'], 'DEVICE_LINK' => generate_device_link($device), 'DEVICE_HARDWARE' => $device['hardware'], 'DEVICE_OS' => $device['os_text'] . ' ' . $device['version'] . ' ' . $device['features'], 'DEVICE_LOCATION' => $device['location'], 'DEVICE_UPTIME' => deviceUptime($device));
//logfile('debug.log', var_export($message, TRUE));
$title = alert_generate_subject($device, $message_tags['ALERT_STATE'], $message_tags);
$message_tags['TITLE'] = $title;
$alert_id = $entry['alert_test_id'];
$notify_status = FALSE;
// Set alert notify status to FALSE by default
$notification_type = 'alert';
$transports = get_alert_contacts($device, $alert_id, $notification_type);
if (!empty($transports)) {
// WARNING, alerts queue currently experimental
if (isset($config['alerts']['queue']) && isset($config['alerts']['queue'])) {
// Add notification to queue
$notification = array('device_id' => $device['device_id'], 'log_id' => $log_id, 'aca_type' => $notification_type, 'endpoints' => json_encode($transports), 'message_graphs' => $message_tags['ENTITY_GRAPHS_ARRAY'], 'notification_added' => time(), 'notification_lifetime' => 300, 'notification_entry' => json_encode($entry));
$notification_message_tags = $message_tags;
unset($notification_message_tags['ENTITY_GRAPHS_ARRAY']);
$notification['message_tags'] = json_encode($notification_message_tags);
$notification_id = dbInsert($notification, 'notifications_queue');
} else {
// Use classic instant notifications send
$notification_count = 0;
foreach ($transports as $method => $endpoints) {
if (isset($config['alerts']['disable'][$method]) && $config['alerts']['disable'][$method]) {
continue;
}
// Skip if method disabled globally
foreach ($endpoints as $endpoint) {
$method_include = $config['install_dir'] . "/includes/alerting/" . $method . ".inc.php";
if (is_file($method_include)) {
print_cli_data("Notifying", "[" . $method . "] " . $endpoint['contact_descr'] . ": " . $endpoint['contact_endpoint']);
// Split out endpoint data as stored JSON in the database into array for use in transport
// The original string also remains available as the contact_endpoint key
foreach (json_decode($endpoint['contact_endpoint']) as $field => $value) {
$endpoint[$field] = $value;
}
include $method_include;
// FIXME check success
// FIXME log notification + success/failure!
if ($notify_status['success']) {
$notification_count++;
}
} else {
print_cli_data("Missing include", $method_include);
}
}
}
if ($notification_count) {
dbUpdate(array('notified' => 1), 'alert_log', '`event_id` = ?', array($notification['log_id']));
}
}
}
}