mirror of
https://github.com/google/alertmanager-irc-relay.git
synced 2024-11-05 11:19:21 +01:00
Add basic monitoring instrumentation
Signed-off-by: Luca Bigliardi <shammash@google.com>
This commit is contained in:
parent
d51a33f4e7
commit
b62dde73d4
42
http.go
42
http.go
@ -21,12 +21,33 @@ import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
promtmpl "github.com/prometheus/alertmanager/template"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
promtmpl "github.com/prometheus/alertmanager/template"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
var (
|
||||
handledAlertGroups = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "webhook_handled_alert_groups",
|
||||
Help: "Number of alert groups received"},
|
||||
[]string{"ircchannel"},
|
||||
)
|
||||
handledAlerts = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "webhook_handled_alerts",
|
||||
Help: "Number of single alert messages relayed"},
|
||||
[]string{"ircchannel"},
|
||||
)
|
||||
alertHandlingErrors = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "webhook_alert_handling_errors",
|
||||
Help: "Errors while processing webhook requests"},
|
||||
[]string{"ircchannel", "error"},
|
||||
)
|
||||
)
|
||||
|
||||
type HTTPListener func(string, http.Handler) error
|
||||
@ -65,7 +86,7 @@ func NewHTTPServerForTesting(config *Config, alertMsgs chan AlertMsg,
|
||||
return server, nil
|
||||
}
|
||||
|
||||
func (server *HTTPServer) FormatMsg(data interface{}) string {
|
||||
func (server *HTTPServer) FormatMsg(ircChannel string, data interface{}) string {
|
||||
output := bytes.Buffer{}
|
||||
var msg string
|
||||
if err := server.MsgTemplate.Execute(&output, data); err != nil {
|
||||
@ -74,6 +95,7 @@ func (server *HTTPServer) FormatMsg(data interface{}) string {
|
||||
log.Printf("Could not apply msg template on alert (%s): %s",
|
||||
err, msg)
|
||||
log.Printf("Sending raw alert")
|
||||
alertHandlingErrors.WithLabelValues(ircChannel, "format_msg").Inc()
|
||||
} else {
|
||||
msg = output.String()
|
||||
}
|
||||
@ -84,12 +106,12 @@ func (server *HTTPServer) GetMsgsFromAlertMessage(ircChannel string,
|
||||
data *promtmpl.Data) []AlertMsg {
|
||||
msgs := []AlertMsg{}
|
||||
if server.MsgOnce {
|
||||
msg := server.FormatMsg(data)
|
||||
msg := server.FormatMsg(ircChannel, data)
|
||||
msgs = append(msgs,
|
||||
AlertMsg{Channel: ircChannel, Alert: msg})
|
||||
} else {
|
||||
for _, alert := range data.Alerts {
|
||||
msg := server.FormatMsg(alert)
|
||||
msg := server.FormatMsg(ircChannel, alert)
|
||||
msgs = append(msgs,
|
||||
AlertMsg{Channel: ircChannel, Alert: msg})
|
||||
}
|
||||
@ -104,13 +126,14 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1024*1024*1024))
|
||||
if err != nil {
|
||||
log.Printf("Could not get body: %s", err)
|
||||
alertHandlingErrors.WithLabelValues(ircChannel, "read_body").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var alertMessage = promtmpl.Data{}
|
||||
if err := json.Unmarshal(body, &alertMessage); err != nil {
|
||||
log.Printf("Could not decode request body (%s): %s", err, body)
|
||||
|
||||
alertHandlingErrors.WithLabelValues(ircChannel, "decode_body").Inc()
|
||||
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
|
||||
w.WriteHeader(422) // Unprocessable entity
|
||||
if err := json.NewEncoder(w).Encode(err); err != nil {
|
||||
@ -119,13 +142,16 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
return
|
||||
}
|
||||
handledAlertGroups.WithLabelValues(ircChannel).Inc()
|
||||
for _, alertMsg := range server.GetMsgsFromAlertMessage(
|
||||
ircChannel, &alertMessage) {
|
||||
select {
|
||||
case server.AlertMsgs <- alertMsg:
|
||||
handledAlerts.WithLabelValues(ircChannel).Inc()
|
||||
default:
|
||||
log.Printf("Could not send this alert to the IRC routine: %s",
|
||||
alertMsg)
|
||||
alertHandlingErrors.WithLabelValues(ircChannel, "internal_comm_channel_full").Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -133,6 +159,8 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
|
||||
func (server *HTTPServer) Run() {
|
||||
router := mux.NewRouter().StrictSlash(true)
|
||||
|
||||
router.Path("/metrics").Handler(promhttp.Handler())
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
server.RelayAlert(w, r)
|
||||
})
|
||||
|
23
irc.go
23
irc.go
@ -22,6 +22,8 @@ import (
|
||||
"time"
|
||||
|
||||
irc "github.com/fluffle/goirc/client"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -32,6 +34,23 @@ const (
|
||||
ircConnectBackoffResetSecs = 1800
|
||||
)
|
||||
|
||||
var (
|
||||
ircConnectedGauge = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "irc_connected",
|
||||
Help: "Wether the IRC connection is established",
|
||||
})
|
||||
ircSentMsgs = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "irc_sent_msgs",
|
||||
Help: "Number of IRC messages sent"},
|
||||
[]string{"ircchannel"},
|
||||
)
|
||||
ircSendMsgErrors = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "irc_send_msg_errors",
|
||||
Help: "Errors while sending IRC messages"},
|
||||
[]string{"ircchannel", "error"},
|
||||
)
|
||||
)
|
||||
|
||||
func loggerHandler(_ *irc.Conn, line *irc.Line) {
|
||||
log.Printf("Received: '%s'", line.Raw)
|
||||
}
|
||||
@ -200,6 +219,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) {
|
||||
if !notifier.sessionUp {
|
||||
log.Printf("Cannot send alert to %s : IRC not connected",
|
||||
alertMsg.Channel)
|
||||
ircSendMsgErrors.WithLabelValues(alertMsg.Channel, "not_connected").Inc()
|
||||
return
|
||||
}
|
||||
notifier.JoinChannel(&IRCChannel{Name: alertMsg.Channel})
|
||||
@ -209,6 +229,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) {
|
||||
} else {
|
||||
notifier.Client.Notice(alertMsg.Channel, alertMsg.Alert)
|
||||
}
|
||||
ircSentMsgs.WithLabelValues(alertMsg.Channel).Inc()
|
||||
}
|
||||
|
||||
func (notifier *IRCNotifier) Run() {
|
||||
@ -237,10 +258,12 @@ func (notifier *IRCNotifier) Run() {
|
||||
notifier.sessionUp = true
|
||||
notifier.MaybeIdentifyNick()
|
||||
notifier.JoinChannels()
|
||||
ircConnectedGauge.Set(1)
|
||||
case <-notifier.sessionDownSignal:
|
||||
notifier.sessionUp = false
|
||||
notifier.CleanupChannels()
|
||||
notifier.Client.Quit("see ya")
|
||||
ircConnectedGauge.Set(0)
|
||||
case <-notifier.StopRunning:
|
||||
log.Printf("IRC routine asked to terminate")
|
||||
keepGoing = false
|
||||
|
Loading…
Reference in New Issue
Block a user