From 494cc38bf817d33f148e1b1d6bbaee135f63bb11 Mon Sep 17 00:00:00 2001 From: Jonathon Klobucar Date: Tue, 29 Mar 2016 18:55:14 -0700 Subject: [PATCH] Don't panic when negative number counter is submitted. Instead now we will send a warning message to the logs and continue on with our day. Also a guard has been added when we detect a channel has been closed and the `Exporter.Listen` function will return out of its loop vs always running. When a channel is closed there will be no more data to consume, so this seems correct. It also allows for a test to be written for the panic. --- exporter.go | 13 +++++++++++- exporter_test.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 exporter_test.go diff --git a/exporter.go b/exporter.go index 4283d75..3a46397 100644 --- a/exporter.go +++ b/exporter.go @@ -191,7 +191,11 @@ func escapeMetricName(metricName string) string { func (b *Exporter) Listen(e <-chan Events) { for { - events := <-e + events, ok := <-e + if !ok { + log.Debug("Channel is closed. Break out of Exporter.Listener.") + return + } for _, event := range events { metricName := "" prometheusLabels := prometheus.Labels{} @@ -214,6 +218,13 @@ func (b *Exporter) Listen(e <-chan Events) { metricName+"_counter", prometheusLabels, ) + // We don't accept negative values for counters. Incrementing the counter with a negative number + // will cause the exporter to panic. Instead we will warn and continue to the next event. + if event.Value() < 0.0 { + log.Warnf("Counter %q is: '%f' (counter must be non-negative value)", metricName, event.Value()) + continue + } + counter.Add(event.Value()) eventStats.WithLabelValues("counter").Inc() diff --git a/exporter_test.go b/exporter_test.go new file mode 100644 index 0000000..b42aea9 --- /dev/null +++ b/exporter_test.go @@ -0,0 +1,52 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + "time" +) + +// TestNegativeCounter validates when we send a negative +// number to a counter that we no longer panic the Exporter Listener. +func TestNegativeCounter(t *testing.T) { + defer func() { + if e := recover(); e != nil { + err := e.(error) + if err.Error() == "counter cannot decrease in value" { + t.Fatalf("Counter was negative and causes a panic.") + } else { + t.Fatalf("Unknown panic and error: %q", err.Error()) + } + } + }() + + events := make(chan Events, 1) + c := Events{ + &CounterEvent{ + metricName: "foo", + value: -1, + }, + } + events <- c + ex := NewExporter(&metricMapper{}) + + // Close channel to signify we are done with the listener after a short period. + go func() { + time.Sleep(time.Millisecond * 100) + close(events) + }() + + ex.Listen(events) +}