From b62dde73d40923a54b32f6903292b6dec2ad7c99 Mon Sep 17 00:00:00 2001 From: Luca Bigliardi Date: Thu, 5 Mar 2020 11:58:39 +0000 Subject: [PATCH] Add basic monitoring instrumentation Signed-off-by: Luca Bigliardi --- http.go | 42 +++++++++++++++++++++++++++++++++++------- irc.go | 23 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/http.go b/http.go index 152dd48..bf64093 100644 --- a/http.go +++ b/http.go @@ -21,12 +21,33 @@ import ( "io/ioutil" "log" "net/http" - - "github.com/gorilla/mux" - promtmpl "github.com/prometheus/alertmanager/template" "strconv" "strings" "text/template" + + "github.com/gorilla/mux" + promtmpl "github.com/prometheus/alertmanager/template" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + handledAlertGroups = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "webhook_handled_alert_groups", + Help: "Number of alert groups received"}, + []string{"ircchannel"}, + ) + handledAlerts = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "webhook_handled_alerts", + Help: "Number of single alert messages relayed"}, + []string{"ircchannel"}, + ) + alertHandlingErrors = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "webhook_alert_handling_errors", + Help: "Errors while processing webhook requests"}, + []string{"ircchannel", "error"}, + ) ) type HTTPListener func(string, http.Handler) error @@ -65,7 +86,7 @@ func NewHTTPServerForTesting(config *Config, alertMsgs chan AlertMsg, return server, nil } -func (server *HTTPServer) FormatMsg(data interface{}) string { +func (server *HTTPServer) FormatMsg(ircChannel string, data interface{}) string { output := bytes.Buffer{} var msg string if err := server.MsgTemplate.Execute(&output, data); err != nil { @@ -74,6 +95,7 @@ func (server *HTTPServer) FormatMsg(data interface{}) string { log.Printf("Could not apply msg template on alert (%s): %s", err, msg) log.Printf("Sending raw alert") + alertHandlingErrors.WithLabelValues(ircChannel, "format_msg").Inc() } else { msg = output.String() } @@ -84,12 +106,12 @@ func (server *HTTPServer) GetMsgsFromAlertMessage(ircChannel string, data *promtmpl.Data) []AlertMsg { msgs := []AlertMsg{} if server.MsgOnce { - msg := server.FormatMsg(data) + msg := server.FormatMsg(ircChannel, data) msgs = append(msgs, AlertMsg{Channel: ircChannel, Alert: msg}) } else { for _, alert := range data.Alerts { - msg := server.FormatMsg(alert) + msg := server.FormatMsg(ircChannel, alert) msgs = append(msgs, AlertMsg{Channel: ircChannel, Alert: msg}) } @@ -104,13 +126,14 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1024*1024*1024)) if err != nil { log.Printf("Could not get body: %s", err) + alertHandlingErrors.WithLabelValues(ircChannel, "read_body").Inc() return } var alertMessage = promtmpl.Data{} if err := json.Unmarshal(body, &alertMessage); err != nil { log.Printf("Could not decode request body (%s): %s", err, body) - + alertHandlingErrors.WithLabelValues(ircChannel, "decode_body").Inc() w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.WriteHeader(422) // Unprocessable entity if err := json.NewEncoder(w).Encode(err); err != nil { @@ -119,13 +142,16 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) { } return } + handledAlertGroups.WithLabelValues(ircChannel).Inc() for _, alertMsg := range server.GetMsgsFromAlertMessage( ircChannel, &alertMessage) { select { case server.AlertMsgs <- alertMsg: + handledAlerts.WithLabelValues(ircChannel).Inc() default: log.Printf("Could not send this alert to the IRC routine: %s", alertMsg) + alertHandlingErrors.WithLabelValues(ircChannel, "internal_comm_channel_full").Inc() } } } @@ -133,6 +159,8 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) { func (server *HTTPServer) Run() { router := mux.NewRouter().StrictSlash(true) + router.Path("/metrics").Handler(promhttp.Handler()) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server.RelayAlert(w, r) }) diff --git a/irc.go b/irc.go index 902d8b7..d4c0b62 100644 --- a/irc.go +++ b/irc.go @@ -22,6 +22,8 @@ import ( "time" irc "github.com/fluffle/goirc/client" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) const ( @@ -32,6 +34,23 @@ const ( ircConnectBackoffResetSecs = 1800 ) +var ( + ircConnectedGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "irc_connected", + Help: "Wether the IRC connection is established", + }) + ircSentMsgs = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "irc_sent_msgs", + Help: "Number of IRC messages sent"}, + []string{"ircchannel"}, + ) + ircSendMsgErrors = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "irc_send_msg_errors", + Help: "Errors while sending IRC messages"}, + []string{"ircchannel", "error"}, + ) +) + func loggerHandler(_ *irc.Conn, line *irc.Line) { log.Printf("Received: '%s'", line.Raw) } @@ -200,6 +219,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) { if !notifier.sessionUp { log.Printf("Cannot send alert to %s : IRC not connected", alertMsg.Channel) + ircSendMsgErrors.WithLabelValues(alertMsg.Channel, "not_connected").Inc() return } notifier.JoinChannel(&IRCChannel{Name: alertMsg.Channel}) @@ -209,6 +229,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) { } else { notifier.Client.Notice(alertMsg.Channel, alertMsg.Alert) } + ircSentMsgs.WithLabelValues(alertMsg.Channel).Inc() } func (notifier *IRCNotifier) Run() { @@ -237,10 +258,12 @@ func (notifier *IRCNotifier) Run() { notifier.sessionUp = true notifier.MaybeIdentifyNick() notifier.JoinChannels() + ircConnectedGauge.Set(1) case <-notifier.sessionDownSignal: notifier.sessionUp = false notifier.CleanupChannels() notifier.Client.Quit("see ya") + ircConnectedGauge.Set(0) case <-notifier.StopRunning: log.Printf("IRC routine asked to terminate") keepGoing = false