Merge pull request #194 from vsakhart/register_inconsistent_metrics

Allow support for inconsistent label sets by marking metrics registered as unchecked collectors
This commit is contained in:
Matthias Rampke 2019-04-09 12:08:33 +00:00 committed by GitHub
commit 43cef6ce6a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 154 additions and 28 deletions

View file

@ -38,9 +38,7 @@ import (
const (
defaultHelp = "Metric autogenerated by statsd_exporter."
regErrF = "A change of configuration created inconsistent metrics for " +
"%q. You have to restart the statsd_exporter, and you should " +
"consider the effects on your monitoring setup. Error: %s"
regErrF = "Failed to update metric %q. Error: %s"
)
var (
@ -51,7 +49,18 @@ var (
intBuf = make([]byte, 8)
)
func labelNames(labels prometheus.Labels) []string {
// uncheckedCollector wraps a Collector but its Describe method yields no Desc.
// This allows incoming metrics to have inconsistent label sets
type uncheckedCollector struct {
c prometheus.Collector
}
func (u uncheckedCollector) Describe(_ chan<- *prometheus.Desc) {}
func (u uncheckedCollector) Collect(c chan<- prometheus.Metric) {
u.c.Collect(c)
}
func getLabelNames(labels prometheus.Labels) []string {
names := make([]string, 0, len(labels))
for labelName := range labels {
names = append(names, labelName)
@ -60,6 +69,10 @@ func labelNames(labels prometheus.Labels) []string {
return names
}
func getContainerMapKey(metricName string, labelNames []string) string {
return metricName + "," + strings.Join(labelNames, ",")
}
// hashNameAndLabels returns a hash value of the provided name string and all
// the label names and values in the provided labels map.
//
@ -87,23 +100,28 @@ func NewCounterContainer() *CounterContainer {
}
func (c *CounterContainer) Get(metricName string, labels prometheus.Labels, help string) (prometheus.Counter, error) {
counterVec, ok := c.Elements[metricName]
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
counterVec, ok := c.Elements[mapKey]
if !ok {
counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: metricName,
Help: help,
}, labelNames(labels))
if err := prometheus.Register(counterVec); err != nil {
}, labelNames)
if err := prometheus.Register(uncheckedCollector{counterVec}); err != nil {
return nil, err
}
c.Elements[metricName] = counterVec
c.Elements[mapKey] = counterVec
}
return counterVec.GetMetricWith(labels)
}
func (c *CounterContainer) Delete(metricName string, labels prometheus.Labels) {
if _, ok := c.Elements[metricName]; ok {
c.Elements[metricName].Delete(labels)
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels)
}
}
@ -118,23 +136,28 @@ func NewGaugeContainer() *GaugeContainer {
}
func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels, help string) (prometheus.Gauge, error) {
gaugeVec, ok := c.Elements[metricName]
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
gaugeVec, ok := c.Elements[mapKey]
if !ok {
gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: metricName,
Help: help,
}, labelNames(labels))
if err := prometheus.Register(gaugeVec); err != nil {
}, labelNames)
if err := prometheus.Register(uncheckedCollector{gaugeVec}); err != nil {
return nil, err
}
c.Elements[metricName] = gaugeVec
c.Elements[mapKey] = gaugeVec
}
return gaugeVec.GetMetricWith(labels)
}
func (c *GaugeContainer) Delete(metricName string, labels prometheus.Labels) {
if _, ok := c.Elements[metricName]; ok {
c.Elements[metricName].Delete(labels)
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels)
}
}
@ -151,7 +174,10 @@ func NewSummaryContainer(mapper *mapper.MetricMapper) *SummaryContainer {
}
func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) {
summaryVec, ok := c.Elements[metricName]
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
summaryVec, ok := c.Elements[mapKey]
if !ok {
quantiles := c.mapper.Defaults.Quantiles
if mapping != nil && mapping.Quantiles != nil && len(mapping.Quantiles) > 0 {
@ -166,18 +192,20 @@ func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, help
Name: metricName,
Help: help,
Objectives: objectives,
}, labelNames(labels))
if err := prometheus.Register(summaryVec); err != nil {
}, getLabelNames(labels))
if err := prometheus.Register(uncheckedCollector{summaryVec}); err != nil {
return nil, err
}
c.Elements[metricName] = summaryVec
c.Elements[mapKey] = summaryVec
}
return summaryVec.GetMetricWith(labels)
}
func (c *SummaryContainer) Delete(metricName string, labels prometheus.Labels) {
if _, ok := c.Elements[metricName]; ok {
c.Elements[metricName].Delete(labels)
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels)
}
}
@ -194,7 +222,10 @@ func NewHistogramContainer(mapper *mapper.MetricMapper) *HistogramContainer {
}
func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) {
histogramVec, ok := c.Elements[metricName]
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
histogramVec, ok := c.Elements[mapKey]
if !ok {
buckets := c.mapper.Defaults.Buckets
if mapping != nil && mapping.Buckets != nil && len(mapping.Buckets) > 0 {
@ -205,18 +236,20 @@ func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, he
Name: metricName,
Help: help,
Buckets: buckets,
}, labelNames(labels))
if err := prometheus.Register(histogramVec); err != nil {
}, labelNames)
if err := prometheus.Register(uncheckedCollector{histogramVec}); err != nil {
return nil, err
}
c.Elements[metricName] = histogramVec
c.Elements[mapKey] = histogramVec
}
return histogramVec.GetMetricWith(labels)
}
func (c *HistogramContainer) Delete(metricName string, labels prometheus.Labels) {
if _, ok := c.Elements[metricName]; ok {
c.Elements[metricName].Delete(labels)
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels)
}
}

View file

@ -64,6 +64,99 @@ func TestNegativeCounter(t *testing.T) {
}
}
// TestInconsistentLabelSets validates that the exporter will register
// and record metrics with the same metric name but inconsistent label
// sets e.g foo{a="1"} and foo{b="1"}
func TestInconsistentLabelSets(t *testing.T) {
firstLabelSet := make(map[string]string)
secondLabelSet := make(map[string]string)
metricNames := [4]string{"counter_test", "gauge_test", "histogram_test", "summary_test"}
firstLabelSet["foo"] = "1"
secondLabelSet["foo"] = "1"
secondLabelSet["bar"] = "2"
events := make(chan Events)
go func() {
c := Events{
&CounterEvent{
metricName: "counter_test",
value: 1,
labels: firstLabelSet,
},
&CounterEvent{
metricName: "counter_test",
value: 1,
labels: secondLabelSet,
},
&GaugeEvent{
metricName: "gauge_test",
value: 1,
labels: firstLabelSet,
},
&GaugeEvent{
metricName: "gauge_test",
value: 1,
labels: secondLabelSet,
},
&TimerEvent{
metricName: "histogram.test",
value: 1,
labels: firstLabelSet,
},
&TimerEvent{
metricName: "histogram.test",
value: 1,
labels: secondLabelSet,
},
&TimerEvent{
metricName: "summary_test",
value: 1,
labels: firstLabelSet,
},
&TimerEvent{
metricName: "summary_test",
value: 1,
labels: secondLabelSet,
},
}
events <- c
close(events)
}()
config := `
mappings:
- match: histogram.test
timer_type: histogram
name: "histogram_test"
`
testMapper := &mapper.MetricMapper{}
err := testMapper.InitFromYAMLString(config)
if err != nil {
t.Fatalf("Config load error: %s %s", config, err)
}
ex := NewExporter(testMapper)
ex.Listen(events)
metrics, err := prometheus.DefaultGatherer.Gather()
if err != nil {
t.Fatalf("Cannot gather from DefaultGatherer: %v", err)
}
for _, metricName := range metricNames {
firstMetric := getFloat64(metrics, metricName, firstLabelSet)
secondMetric := getFloat64(metrics, metricName, secondLabelSet)
if firstMetric == nil {
t.Fatalf("Could not find time series with first label set for metric: %s", metricName)
}
if secondMetric == nil {
t.Fatalf("Could not find time series with second label set for metric: %s", metricName)
}
}
}
// TestEmptyStringMetric validates when a metric name ends up
// being the empty string after applying the match replacements
// tha we don't panic the Exporter Listener.