merge with upstream master

CHANGELOG.md

@@ -1,3 +1,11 @@
+## v0.4.0 / 2017-05-12
+
+* [IMPROVEMENT] Improve mapping configuration parser #61
+* [IMPROVEMENT] Add increment/decrement support to Gauges #65
+* [BUGFIX] Tolerate more forms of broken lines from StatsD #48
+* [BUGFIX] Skip metrics with invalid utf8 #50
+* [BUGFIX] ListenAndServe now fails on exit #58
+
 ## 0.3.0 / 2016-05-05
 
 * [CHANGE] Drop `_count` suffix for `loaded_mappings` metric (#41)

README.md

@@ -88,6 +88,7 @@ with `-statsd.add-suffix=false`.
 
 An example mapping configuration with `-statsd.add-suffix=false`:
 
+    # comments are allowed
     test.dispatcher.*.*.*
     name="dispatcher_events_total"
     processor="$1"

VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

bridge_test.go

@@ -198,6 +198,21 @@ func TestHandlePacket(t *testing.T) {
 					labels:     map[string]string{},
 				},
 			},
+		}, {
+			name: "timings with sampling factor",
+			in:   "foo.timing:0.5|ms|@0.1",
+			out: Events{
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+				&TimerEvent{metricName: "foo.timing", value: 0.5, labels: map[string]string{}},
+			},
 		}, {
 			name: "bad line",
 			in:   "foo",

circle.yml

@@ -48,6 +48,7 @@ deployment:
     owner: prometheus
     commands:
       - promu crossbuild tarballs
+      - promu checksum .tarballs
       - promu release .tarballs
       - mkdir $CIRCLE_ARTIFACTS/releases/ && cp -a .tarballs/* $CIRCLE_ARTIFACTS/releases/
       - docker login -e $DOCKER_EMAIL -u $DOCKER_LOGIN -p $DOCKER_PASSWORD

exporter.go

@@ -69,7 +69,7 @@ func NewCounterContainer() *CounterContainer {
 	}
 }
 
-func (c *CounterContainer) Get(metricName string, labels prometheus.Labels) prometheus.Counter {
+func (c *CounterContainer) Get(metricName string, labels prometheus.Labels) (prometheus.Counter, error) {
 	hash := hashNameAndLabels(metricName, labels)
 	counter, ok := c.Elements[hash]
 	if !ok {
@@ -78,12 +78,12 @@ func (c *CounterContainer) Get(metricName string, labels prometheus.Labels) prom
 			Help:        defaultHelp,
 			ConstLabels: labels,
 		})
-		c.Elements[hash] = counter
 		if err := prometheus.Register(counter); err != nil {
-			log.Fatalf(regErrF, metricName, err)
+			return nil, err
 		}
+		c.Elements[hash] = counter
 	}
-	return counter
+	return counter, nil
 }
 
 type GaugeContainer struct {
@@ -96,7 +96,7 @@ func NewGaugeContainer() *GaugeContainer {
 	}
 }
 
-func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels) prometheus.Gauge {
+func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels) (prometheus.Gauge, error) {
 	hash := hashNameAndLabels(metricName, labels)
 	gauge, ok := c.Elements[hash]
 	if !ok {
@@ -105,12 +105,12 @@ func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels) promet
 			Help:        defaultHelp,
 			ConstLabels: labels,
 		})
-		c.Elements[hash] = gauge
 		if err := prometheus.Register(gauge); err != nil {
-			log.Fatalf(regErrF, metricName, err)
+			return nil, err
 		}
+		c.Elements[hash] = gauge
 	}
-	return gauge
+	return gauge, nil
 }
 
 type SummaryContainer struct {
@@ -123,7 +123,7 @@ func NewSummaryContainer() *SummaryContainer {
 	}
 }
 
-func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels) prometheus.Summary {
+func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels) (prometheus.Summary, error) {
 	hash := hashNameAndLabels(metricName, labels)
 	summary, ok := c.Elements[hash]
 	if !ok {
@@ -133,12 +133,12 @@ func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels) prom
 				Help:        defaultHelp,
 				ConstLabels: labels,
 			})
-		c.Elements[hash] = summary
 		if err := prometheus.Register(summary); err != nil {
-			log.Fatalf(regErrF, metricName, err)
+			return nil, err
 		}
+		c.Elements[hash] = summary
 	}
-	return summary
+	return summary, nil
 }
 
 type HistogramContainer struct {
@@ -153,7 +153,7 @@ func NewHistogramContainer(mapper *metricMapper) *HistogramContainer {
 	}
 }
 
-func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, mapping *metricMapping) prometheus.Histogram {
+func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, mapping *metricMapping) (prometheus.Histogram, error) {
 	hash := hashNameAndLabels(metricName, labels)
 	histogram, ok := c.Elements[hash]
 	if !ok {
@@ -170,10 +170,10 @@ func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, ma
 			})
 		c.Elements[hash] = histogram
 		if err := prometheus.Register(histogram); err != nil {
-			log.Fatalf(regErrF, metricName, err)
+			return nil, err
 		}
 	}
-	return histogram
+	return histogram, nil
 }
 
 type Event interface {
@@ -269,10 +269,6 @@ func (b *Exporter) Listen(e <-chan Events) {
 
 			switch ev := event.(type) {
 			case *CounterEvent:
-				counter := b.Counters.Get(
-					b.suffix(metricName, "counter"),
-					prometheusLabels,
-				)
 				// We don't accept negative values for counters. Incrementing the counter with a negative number
 				// will cause the exporter to panic. Instead we will warn and continue to the next event.
 				if event.Value() < 0.0 {
@@ -280,16 +276,26 @@ func (b *Exporter) Listen(e <-chan Events) {
 					continue
 				}
 
+				counter, err := b.Counters.Get(
+					b.suffix(metricName, "counter"),
+					prometheusLabels,
+				)
+				if err == nil {
 					counter.Add(event.Value())
 
 					eventStats.WithLabelValues("counter").Inc()
+				} else {
+					log.Errorf(regErrF, metricName, err)
+					conflictingEventStats.WithLabelValues("counter").Inc()
+				}
 
 			case *GaugeEvent:
-				gauge := b.Gauges.Get(
+				gauge, err := b.Gauges.Get(
 					b.suffix(metricName, "gauge"),
 					prometheusLabels,
 				)
 
+				if err == nil {
 					if ev.relative {
 						gauge.Add(event.Value())
 					} else {
@@ -297,6 +303,10 @@ func (b *Exporter) Listen(e <-chan Events) {
 					}
 
 					eventStats.WithLabelValues("gauge").Inc()
+				} else {
+					log.Errorf(regErrF, metricName, err)
+					conflictingEventStats.WithLabelValues("gauge").Inc()
+				}
 
 			case *TimerEvent:
 				timerType := ""
@@ -308,21 +318,31 @@ func (b *Exporter) Listen(e <-chan Events) {
 				}
 
 				if timerType == "histogram" {
-					histogram := b.Histograms.Get(
+					histogram, err := b.Histograms.Get(
 						b.suffix(metricName, "timer"),
 						prometheusLabels,
 						mapping,
 					)
+					if err == nil {
 						histogram.Observe(event.Value())
+						eventStats.WithLabelValues("timer").Inc()
 					} else {
-					summary := b.Summaries.Get(
+						log.Errorf(regErrF, metricName, err)
+						conflictingEventStats.WithLabelValues("timer").Inc()
+					}
+				} else {
+					summary, err := b.Summaries.Get(
 						b.suffix(metricName, "timer"),
 						prometheusLabels,
 					)
+					if err == nil {
 						summary.Observe(event.Value())
-				}
-
 						eventStats.WithLabelValues("timer").Inc()
+					} else {
+						log.Errorf(regErrF, metricName, err)
+						conflictingEventStats.WithLabelValues("timer").Inc()
+					}
+				}
 
 			default:
 				log.Errorln("Unsupported event type")
@@ -450,6 +470,7 @@ func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
 				continue
 			}
 
+			multiplyEvents := 1
 			labels := map[string]string{}
 			if len(components) >= 3 {
 				for _, component := range components[2:] {
@@ -463,9 +484,10 @@ func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
 				for _, component := range components[2:] {
 					switch component[0] {
 					case '@':
-						if statType != "c" {
+						if statType != "c" && statType != "ms" {
 							log.Errorln("Illegal sampling factor for non-counter metric on line", line)
 							networkStats.WithLabelValues("illegal_sample_factor").Inc()
+							continue
 						}
 						samplingFactor, err = strconv.ParseFloat(component[1:], 64)
 						if err != nil {
@@ -475,7 +497,12 @@ func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
 						if samplingFactor == 0 {
 							samplingFactor = 1
 						}
+
+						if statType == "c" {
 							value /= samplingFactor
+						} else if statType == "ms" {
+							multiplyEvents = int(1 / samplingFactor)
+						}
 					case '#':
 						labels = parseDogStatsDTagsToLabels(component)
 					default:
@@ -486,6 +513,7 @@ func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
 				}
 			}
 
+			for i := 0; i < multiplyEvents; i++ {
 				event, err := buildEvent(statType, metric, value, relative, labels)
 				if err != nil {
 					log.Errorf("Error building event on line %s: %s", line, err)
@@ -493,6 +521,7 @@ func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
 					continue
 				}
 				events = append(events, event)
+			}
 			networkStats.WithLabelValues("legal").Inc()
 		}
 	}

mapper.go

@@ -70,6 +70,11 @@ func (m *metricMapper) initFromString(fileContents string) error {
 	for i, line := range lines {
 		line = strings.TrimSpace(line)
 
+		// skip comments
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+
 		switch state {
 		case SEARCHING:
 			if line == "" {

mapper_test.go

@@ -28,11 +28,14 @@ func TestMetricMapper(t *testing.T) {
 		// Config with several mapping definitions.
 		{
 			config: `
+				# this is a comment
+				# this is another
 				test.dispatcher.*.*.*
 				name="dispatch_events"
 				processor="$1"
 				action="$2"
 				result="$3"
+				# here is a third
 				job="test_dispatcher"
 
 				test.my-dispatch-host01.name.dispatcher.*.*.*

telemetry.go

@@ -47,6 +47,13 @@ var (
 		Name: "statsd_exporter_loaded_mappings",
 		Help: "The current number of configured metric mappings.",
 	})
+	conflictingEventStats = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "statsd_exporter_events_conflict_total",
+			Help: "The total number of StatsD events with conflicting names.",
+		},
+		[]string{"type"},
+	)
 )
 
 func init() {
@@ -54,4 +61,5 @@ func init() {
 	prometheus.MustRegister(networkStats)
 	prometheus.MustRegister(configLoads)
 	prometheus.MustRegister(mappingsCount)
+	prometheus.MustRegister(conflictingEventStats)
 }