Allow support for inconsistent label sets by marking metrics

registered as unchecked collectors

Signed-off-by: Vitaliy Sakhartchouk <vsakhart@github.com>

exporter.go

@@ -38,9 +38,7 @@ import (
 
 const (
 	defaultHelp = "Metric autogenerated by statsd_exporter."
-	regErrF     = "A change of configuration created inconsistent metrics for " +
+	regErrF     = "Failed to update metric %q. Error: %s"
-		"%q. You have to restart the statsd_exporter, and you should " +
-		"consider the effects on your monitoring setup. Error: %s"
 )
 
 var (
@@ -51,7 +49,18 @@ var (
 	intBuf = make([]byte, 8)
 )
 
-func labelNames(labels prometheus.Labels) []string {
+// uncheckedCollector wraps a Collector but its Describe method yields no Desc.
+// This allows incoming metrics to have inconsistent label sets
+type uncheckedCollector struct {
+	c prometheus.Collector
+}
+
+func (u uncheckedCollector) Describe(_ chan<- *prometheus.Desc) {}
+func (u uncheckedCollector) Collect(c chan<- prometheus.Metric) {
+	u.c.Collect(c)
+}
+
+func getLabelNames(labels prometheus.Labels) []string {
 	names := make([]string, 0, len(labels))
 	for labelName := range labels {
 		names = append(names, labelName)
@@ -87,23 +96,28 @@ func NewCounterContainer() *CounterContainer {
 }
 
 func (c *CounterContainer) Get(metricName string, labels prometheus.Labels, help string) (prometheus.Counter, error) {
-	counterVec, ok := c.Elements[metricName]
+	labelNames := getLabelNames(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+
+	counterVec, ok := c.Elements[mapKey]
 	if !ok {
 		counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
 			Name: metricName,
 			Help: help,
-		}, labelNames(labels))
+		}, labelNames)
-		if err := prometheus.Register(counterVec); err != nil {
+		if err := prometheus.Register(uncheckedCollector{counterVec}); err != nil {
 			return nil, err
 		}
-		c.Elements[metricName] = counterVec
+		c.Elements[mapKey] = counterVec
 	}
 	return counterVec.GetMetricWith(labels)
 }
 
 func (c *CounterContainer) Delete(metricName string, labels prometheus.Labels) {
-	if _, ok := c.Elements[metricName]; ok {
+	labelNames := getLabelNames(labels)
-		c.Elements[metricName].Delete(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+	if _, ok := c.Elements[mapKey]; ok {
+		c.Elements[mapKey].Delete(labels)
 	}
 }
 
@@ -118,23 +132,28 @@ func NewGaugeContainer() *GaugeContainer {
 }
 
 func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels, help string) (prometheus.Gauge, error) {
-	gaugeVec, ok := c.Elements[metricName]
+	labelNames := getLabelNames(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+
+	gaugeVec, ok := c.Elements[mapKey]
 	if !ok {
 		gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: metricName,
 			Help: help,
-		}, labelNames(labels))
+		}, labelNames)
-		if err := prometheus.Register(gaugeVec); err != nil {
+		if err := prometheus.Register(uncheckedCollector{gaugeVec}); err != nil {
 			return nil, err
 		}
-		c.Elements[metricName] = gaugeVec
+		c.Elements[mapKey] = gaugeVec
 	}
 	return gaugeVec.GetMetricWith(labels)
 }
 
 func (c *GaugeContainer) Delete(metricName string, labels prometheus.Labels) {
-	if _, ok := c.Elements[metricName]; ok {
+	labelNames := getLabelNames(labels)
-		c.Elements[metricName].Delete(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+	if _, ok := c.Elements[mapKey]; ok {
+		c.Elements[mapKey].Delete(labels)
 	}
 }
 
@@ -151,7 +170,10 @@ func NewSummaryContainer(mapper *mapper.MetricMapper) *SummaryContainer {
 }
 
 func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) {
-	summaryVec, ok := c.Elements[metricName]
+	labelNames := getLabelNames(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+
+	summaryVec, ok := c.Elements[mapKey]
 	if !ok {
 		quantiles := c.mapper.Defaults.Quantiles
 		if mapping != nil && mapping.Quantiles != nil && len(mapping.Quantiles) > 0 {
@@ -166,18 +188,20 @@ func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, help
 				Name:       metricName,
 				Help:       help,
 				Objectives: objectives,
-			}, labelNames(labels))
+			}, getLabelNames(labels))
-		if err := prometheus.Register(summaryVec); err != nil {
+		if err := prometheus.Register(uncheckedCollector{summaryVec}); err != nil {
 			return nil, err
 		}
-		c.Elements[metricName] = summaryVec
+		c.Elements[mapKey] = summaryVec
 	}
 	return summaryVec.GetMetricWith(labels)
 }
 
 func (c *SummaryContainer) Delete(metricName string, labels prometheus.Labels) {
-	if _, ok := c.Elements[metricName]; ok {
+	labelNames := getLabelNames(labels)
-		c.Elements[metricName].Delete(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+	if _, ok := c.Elements[mapKey]; ok {
+		c.Elements[mapKey].Delete(labels)
 	}
 }
 
@@ -194,7 +218,10 @@ func NewHistogramContainer(mapper *mapper.MetricMapper) *HistogramContainer {
 }
 
 func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) {
-	histogramVec, ok := c.Elements[metricName]
+	labelNames := getLabelNames(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+
+	histogramVec, ok := c.Elements[mapKey]
 	if !ok {
 		buckets := c.mapper.Defaults.Buckets
 		if mapping != nil && mapping.Buckets != nil && len(mapping.Buckets) > 0 {
@@ -205,18 +232,20 @@ func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, he
 				Name:    metricName,
 				Help:    help,
 				Buckets: buckets,
-			}, labelNames(labels))
+			}, labelNames)
-		if err := prometheus.Register(histogramVec); err != nil {
+		if err := prometheus.Register(uncheckedCollector{histogramVec}); err != nil {
 			return nil, err
 		}
-		c.Elements[metricName] = histogramVec
+		c.Elements[mapKey] = histogramVec
 	}
 	return histogramVec.GetMetricWith(labels)
 }
 
 func (c *HistogramContainer) Delete(metricName string, labels prometheus.Labels) {
-	if _, ok := c.Elements[metricName]; ok {
+	labelNames := getLabelNames(labels)
-		c.Elements[metricName].Delete(labels)
+	mapKey := metricName + "," + strings.Join(labelNames, ",")
+	if _, ok := c.Elements[mapKey]; ok {
+		c.Elements[mapKey].Delete(labels)
 	}
 }
 

exporter_test.go

@@ -64,6 +64,99 @@ func TestNegativeCounter(t *testing.T) {
 	}
 }
 
+// TestInconsistentLabelSets validates that the exporter will register
+// and record metrics with the same metric name but inconsistent label
+// sets e.g foo{a="1"} and foo{b="1"}
+func TestInconsistentLabelSets(t *testing.T) {
+	firstLabelSet := make(map[string]string)
+	secondLabelSet := make(map[string]string)
+	metricNames := [4]string{"counter_test", "gauge_test", "histogram_test", "summary_test"}
+
+	firstLabelSet["foo"] = "1"
+	secondLabelSet["foo"] = "1"
+	secondLabelSet["bar"] = "2"
+
+	events := make(chan Events)
+	go func() {
+		c := Events{
+			&CounterEvent{
+				metricName: "counter_test",
+				value:      1,
+				labels:     firstLabelSet,
+			},
+			&CounterEvent{
+				metricName: "counter_test",
+				value:      1,
+				labels:     secondLabelSet,
+			},
+			&GaugeEvent{
+				metricName: "gauge_test",
+				value:      1,
+				labels:     firstLabelSet,
+			},
+			&GaugeEvent{
+				metricName: "gauge_test",
+				value:      1,
+				labels:     secondLabelSet,
+			},
+			&TimerEvent{
+				metricName: "histogram.test",
+				value:      1,
+				labels:     firstLabelSet,
+			},
+			&TimerEvent{
+				metricName: "histogram.test",
+				value:      1,
+				labels:     secondLabelSet,
+			},
+			&TimerEvent{
+				metricName: "summary_test",
+				value:      1,
+				labels:     firstLabelSet,
+			},
+			&TimerEvent{
+				metricName: "summary_test",
+				value:      1,
+				labels:     secondLabelSet,
+			},
+		}
+		events <- c
+		close(events)
+	}()
+
+	config := `
+mappings:
+- match: histogram.test
+  timer_type: histogram
+  name: "histogram_test"
+`
+	testMapper := &mapper.MetricMapper{}
+	err := testMapper.InitFromYAMLString(config)
+	if err != nil {
+		t.Fatalf("Config load error: %s %s", config, err)
+	}
+
+	ex := NewExporter(testMapper)
+	ex.Listen(events)
+
+	metrics, err := prometheus.DefaultGatherer.Gather()
+	if err != nil {
+		t.Fatalf("Cannot gather from DefaultGatherer: %v", err)
+	}
+
+	for _, metricName := range metricNames {
+		firstMetric := getFloat64(metrics, metricName, firstLabelSet)
+		secondMetric := getFloat64(metrics, metricName, secondLabelSet)
+
+		if firstMetric == nil {
+			t.Fatalf("Could not find time series with first label set for metric: %s", metricName)
+		}
+		if secondMetric == nil {
+			t.Fatalf("Could not find time series with second label set for metric: %s", metricName)
+		}
+	}
+}
+
 // TestEmptyStringMetric validates when a metric name ends up
 // being the empty string after applying the match replacements
 // tha we don't panic the Exporter Listener.