Add basic monitoring instrumentation

Signed-off-by: Luca Bigliardi <shammash@google.com>
This commit is contained in:
Luca Bigliardi 2020-03-05 11:58:39 +00:00
parent d51a33f4e7
commit b62dde73d4
2 changed files with 58 additions and 7 deletions

42
http.go
View File

@ -21,12 +21,33 @@ import (
"io/ioutil"
"log"
"net/http"
"github.com/gorilla/mux"
promtmpl "github.com/prometheus/alertmanager/template"
"strconv"
"strings"
"text/template"
"github.com/gorilla/mux"
promtmpl "github.com/prometheus/alertmanager/template"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
handledAlertGroups = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "webhook_handled_alert_groups",
Help: "Number of alert groups received"},
[]string{"ircchannel"},
)
handledAlerts = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "webhook_handled_alerts",
Help: "Number of single alert messages relayed"},
[]string{"ircchannel"},
)
alertHandlingErrors = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "webhook_alert_handling_errors",
Help: "Errors while processing webhook requests"},
[]string{"ircchannel", "error"},
)
)
type HTTPListener func(string, http.Handler) error
@ -65,7 +86,7 @@ func NewHTTPServerForTesting(config *Config, alertMsgs chan AlertMsg,
return server, nil
}
func (server *HTTPServer) FormatMsg(data interface{}) string {
func (server *HTTPServer) FormatMsg(ircChannel string, data interface{}) string {
output := bytes.Buffer{}
var msg string
if err := server.MsgTemplate.Execute(&output, data); err != nil {
@ -74,6 +95,7 @@ func (server *HTTPServer) FormatMsg(data interface{}) string {
log.Printf("Could not apply msg template on alert (%s): %s",
err, msg)
log.Printf("Sending raw alert")
alertHandlingErrors.WithLabelValues(ircChannel, "format_msg").Inc()
} else {
msg = output.String()
}
@ -84,12 +106,12 @@ func (server *HTTPServer) GetMsgsFromAlertMessage(ircChannel string,
data *promtmpl.Data) []AlertMsg {
msgs := []AlertMsg{}
if server.MsgOnce {
msg := server.FormatMsg(data)
msg := server.FormatMsg(ircChannel, data)
msgs = append(msgs,
AlertMsg{Channel: ircChannel, Alert: msg})
} else {
for _, alert := range data.Alerts {
msg := server.FormatMsg(alert)
msg := server.FormatMsg(ircChannel, alert)
msgs = append(msgs,
AlertMsg{Channel: ircChannel, Alert: msg})
}
@ -104,13 +126,14 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1024*1024*1024))
if err != nil {
log.Printf("Could not get body: %s", err)
alertHandlingErrors.WithLabelValues(ircChannel, "read_body").Inc()
return
}
var alertMessage = promtmpl.Data{}
if err := json.Unmarshal(body, &alertMessage); err != nil {
log.Printf("Could not decode request body (%s): %s", err, body)
alertHandlingErrors.WithLabelValues(ircChannel, "decode_body").Inc()
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(422) // Unprocessable entity
if err := json.NewEncoder(w).Encode(err); err != nil {
@ -119,13 +142,16 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
}
return
}
handledAlertGroups.WithLabelValues(ircChannel).Inc()
for _, alertMsg := range server.GetMsgsFromAlertMessage(
ircChannel, &alertMessage) {
select {
case server.AlertMsgs <- alertMsg:
handledAlerts.WithLabelValues(ircChannel).Inc()
default:
log.Printf("Could not send this alert to the IRC routine: %s",
alertMsg)
alertHandlingErrors.WithLabelValues(ircChannel, "internal_comm_channel_full").Inc()
}
}
}
@ -133,6 +159,8 @@ func (server *HTTPServer) RelayAlert(w http.ResponseWriter, r *http.Request) {
func (server *HTTPServer) Run() {
router := mux.NewRouter().StrictSlash(true)
router.Path("/metrics").Handler(promhttp.Handler())
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
server.RelayAlert(w, r)
})

23
irc.go
View File

@ -22,6 +22,8 @@ import (
"time"
irc "github.com/fluffle/goirc/client"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
const (
@ -32,6 +34,23 @@ const (
ircConnectBackoffResetSecs = 1800
)
var (
ircConnectedGauge = promauto.NewGauge(prometheus.GaugeOpts{
Name: "irc_connected",
Help: "Wether the IRC connection is established",
})
ircSentMsgs = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "irc_sent_msgs",
Help: "Number of IRC messages sent"},
[]string{"ircchannel"},
)
ircSendMsgErrors = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "irc_send_msg_errors",
Help: "Errors while sending IRC messages"},
[]string{"ircchannel", "error"},
)
)
func loggerHandler(_ *irc.Conn, line *irc.Line) {
log.Printf("Received: '%s'", line.Raw)
}
@ -200,6 +219,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) {
if !notifier.sessionUp {
log.Printf("Cannot send alert to %s : IRC not connected",
alertMsg.Channel)
ircSendMsgErrors.WithLabelValues(alertMsg.Channel, "not_connected").Inc()
return
}
notifier.JoinChannel(&IRCChannel{Name: alertMsg.Channel})
@ -209,6 +229,7 @@ func (notifier *IRCNotifier) MaybeSendAlertMsg(alertMsg *AlertMsg) {
} else {
notifier.Client.Notice(alertMsg.Channel, alertMsg.Alert)
}
ircSentMsgs.WithLabelValues(alertMsg.Channel).Inc()
}
func (notifier *IRCNotifier) Run() {
@ -237,10 +258,12 @@ func (notifier *IRCNotifier) Run() {
notifier.sessionUp = true
notifier.MaybeIdentifyNick()
notifier.JoinChannels()
ircConnectedGauge.Set(1)
case <-notifier.sessionDownSignal:
notifier.sessionUp = false
notifier.CleanupChannels()
notifier.Client.Quit("see ya")
ircConnectedGauge.Set(0)
case <-notifier.StopRunning:
log.Printf("IRC routine asked to terminate")
keepGoing = false