Add basic monitoring instrumentation

Signed-off-by: Luca Bigliardi <shammash@google.com>

http.go

@@ -21,12 +21,33 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-
-	"github.com/gorilla/mux"
-	promtmpl "github.com/prometheus/alertmanager/template"
 	"strconv"
 	"strings"
 	"text/template"
+
+	"github.com/gorilla/mux"
+	promtmpl "github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var (
+	handledAlertGroups = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "webhook_handled_alert_groups",
+		Help: "Number of alert groups received"},
+		[]string{"ircchannel"},
+	)
+	handledAlerts = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "webhook_handled_alerts",
+		Help: "Number of single alert messages relayed"},
+		[]string{"ircchannel"},
+	)
+	alertHandlingErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "webhook_alert_handling_errors",
+		Help: "Errors while processing webhook requests"},
+		[]string{"ircchannel", "error"},
+	)
 )
 
 type HTTPListener func(string, http.Handler) error
@@ -65,7 +86,7 @@ func NewHTTPServerForTesting(config *Config, alertMsgs chan AlertMsg,
 	return server, nil
 }
 
-func (server *HTTPServer) FormatMsg(data interface{}) string {
+func (server *HTTPServer) FormatMsg(ircChannel string, data interface{}) string {
 	output := bytes.Buffer{}
 	var msg string
 	if err := server.MsgTemplate.Execute(&output, data); err != nil {
@@ -74,6 +95,7 @@ func (server *HTTPServer) FormatMsg(data interface{}) string {
 		log.Printf("Could not apply msg template on alert (%s): %s",
 			err, msg)
 		log.Printf("Sending raw alert")
+		alertHandlingErrors.WithLabelValues(ircChannel, "format_msg").Inc()
 	} else {
 		msg = output.String()
 	}
@@ -84,12 +106,12 @@ func (server *HTTPServer) GetMsgsFromAlertMessage(ircChannel string,
 	data *promtmpl.Data) []AlertMsg {
 	msgs := []AlertMsg{}
 	if server.MsgOnce {
-		msg := server.FormatMsg(data)
+		msg := server.FormatMsg(ircChannel, data)
 		msgs = append(msgs,
 			AlertMsg{Channel: ircChannel, Alert: msg})
 	} else {
 		for _, alert := range data.Alerts {
-			msg := server.FormatMsg(alert)
+			msg := server.FormatMsg(ircChannel, alert)
 			msgs = append(msgs,
 				AlertMsg{Channel: ircChannel, Alert: msg})
 		}
@@ -104,13 +126,14 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
 	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1024*1024*1024))
 	if err != nil {
 		log.Printf("Could not get body: %s", err)
+		alertHandlingErrors.WithLabelValues(ircChannel, "read_body").Inc()
 		return
 	}
 
 	var alertMessage = promtmpl.Data{}
 	if err := json.Unmarshal(body, &alertMessage); err != nil {
 		log.Printf("Could not decode request body (%s): %s", err, body)
-
+		alertHandlingErrors.WithLabelValues(ircChannel, "decode_body").Inc()
 		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
 		w.WriteHeader(422) // Unprocessable entity
 		if err := json.NewEncoder(w).Encode(err); err != nil {
@@ -119,13 +142,16 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
 		}
 		return
 	}
+	handledAlertGroups.WithLabelValues(ircChannel).Inc()
 	for _, alertMsg := range server.GetMsgsFromAlertMessage(
 		ircChannel, &alertMessage) {
 		select {
 		case server.AlertMsgs <- alertMsg:
+			handledAlerts.WithLabelValues(ircChannel).Inc()
 		default:
 			log.Printf("Could not send this alert to the IRC routine: %s",
 				alertMsg)
+			alertHandlingErrors.WithLabelValues(ircChannel, "internal_comm_channel_full").Inc()
 		}
 	}
 }
@@ -133,6 +159,8 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
 func (server *HTTPServer) Run() {
 	router := mux.NewRouter().StrictSlash(true)
 
+	router.Path("/metrics").Handler(promhttp.Handler())
+
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		server.RelayAlert(w, r)
 	})

irc.go

@@ -22,6 +22,8 @@ import (
 	"time"
 
 	irc "github.com/fluffle/goirc/client"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 const (
@@ -32,6 +34,23 @@ const (
 	ircConnectBackoffResetSecs = 1800
 )
 
+var (
+	ircConnectedGauge = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "irc_connected",
+		Help: "Wether the IRC connection is established",
+	})
+	ircSentMsgs = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "irc_sent_msgs",
+		Help: "Number of IRC messages sent"},
+		[]string{"ircchannel"},
+	)
+	ircSendMsgErrors = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "irc_send_msg_errors",
+		Help: "Errors while sending IRC messages"},
+		[]string{"ircchannel", "error"},
+	)
+)
+
 func loggerHandler(_ *irc.Conn, line *irc.Line) {
 	log.Printf("Received: '%s'", line.Raw)
 }
@@ -200,6 +219,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) {
 	if !notifier.sessionUp {
 		log.Printf("Cannot send alert to %s : IRC not connected",
 			alertMsg.Channel)
+		ircSendMsgErrors.WithLabelValues(alertMsg.Channel, "not_connected").Inc()
 		return
 	}
 	notifier.JoinChannel(&IRCChannel{Name: alertMsg.Channel})
@@ -209,6 +229,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) {
 	} else {
 		notifier.Client.Notice(alertMsg.Channel, alertMsg.Alert)
 	}
+	ircSentMsgs.WithLabelValues(alertMsg.Channel).Inc()
 }
 
 func (notifier *IRCNotifier) Run() {
@@ -237,10 +258,12 @@ func (notifier *IRCNotifier) Run() {
 			notifier.sessionUp = true
 			notifier.MaybeIdentifyNick()
 			notifier.JoinChannels()
+			ircConnectedGauge.Set(1)
 		case <-notifier.sessionDownSignal:
 			notifier.sessionUp = false
 			notifier.CleanupChannels()
 			notifier.Client.Quit("see ya")
+			ircConnectedGauge.Set(0)
 		case <-notifier.StopRunning:
 			log.Printf("IRC routine asked to terminate")
 			keepGoing = false