mirrors/statsd_exporter
1
0
Fork 1
mirror of https://github.com/prometheus/statsd_exporter.git synced 2024-11-26 01:01:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #219 from claytono/optimize-label-handling

Browse source 
Optimize label sorting
This commit is contained in:
Matthias Rampke 2019-05-17 13:47:03 +00:00 committed by GitHub
commit c9004f8f3f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 85 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
exporter.go
View file

@ -16,7 +16,6 @@ package main
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"encoding/binary"
"fmt" "fmt"
"hash/fnv" "hash/fnv"
"io" "io"
@ -71,7 +70,7 @@ type metricChecker interface {
metricConflicts(string, metricType) bool metricConflicts(string, metricType) bool
} }
func getLabelNames(labels prometheus.Labels) []string { func getSortedLabelNames(labels prometheus.Labels) []string {
names := make([]string, 0, len(labels)) names := make([]string, 0, len(labels))
for labelName := range labels { for labelName := range labels {
names = append(names, labelName) names = append(names, labelName)
@ -89,13 +88,20 @@ func getContainerMapKey(metricName string, labelNames []string) string {
// //
// Not safe for concurrent use! (Uses a shared buffer and hasher to save on // Not safe for concurrent use! (Uses a shared buffer and hasher to save on
// allocations.) // allocations.)
func hashNameAndLabels(name string, labels prometheus.Labels) uint64 { func hashNameAndLabels(name string, labelNames []string, labels prometheus.Labels) uint64 {
hash.Reset() hash.Reset()
strBuf.Reset() strBuf.Reset()
strBuf.WriteString(name) strBuf.WriteString(name)
strBuf.WriteByte(model.SeparatorByte)
for _, labelName := range labelNames {
strBuf.WriteString(labelName)
strBuf.WriteByte(model.SeparatorByte)
strBuf.WriteString(labels[labelName])
strBuf.WriteByte(model.SeparatorByte)
}
hash.Write(strBuf.Bytes()) hash.Write(strBuf.Bytes())
binary.BigEndian.PutUint64(intBuf, model.LabelsToSignature(labels))
hash.Write(intBuf)
return hash.Sum64() return hash.Sum64()
} }
@ -110,8 +116,7 @@ func NewCounterContainer() *CounterContainer {
} }
} }
func (c *CounterContainer) Get(metricName string, labels prometheus.Labels, mc metricChecker, help string) (prometheus.Counter, error) { func (c *CounterContainer) Get(metricName string, labelNames []string, labels prometheus.Labels, mc metricChecker, help string) (prometheus.Counter, error) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
counterVec, ok := c.Elements[mapKey] counterVec, ok := c.Elements[mapKey]
@ -132,8 +137,7 @@ func (c *CounterContainer) Get(metricName string, labels prometheus.Labels, mc m
return counterVec.GetMetricWith(labels) return counterVec.GetMetricWith(labels)
} }
func (c *CounterContainer) Delete(metricName string, labels prometheus.Labels) { func (c *CounterContainer) Delete(metricName string, labelNames []string, labels prometheus.Labels) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok { if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels) c.Elements[mapKey].Delete(labels)
@ -151,8 +155,7 @@ func NewGaugeContainer() *GaugeContainer {
} }
} }
func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels, mc metricChecker, help string) (prometheus.Gauge, error) { func (c *GaugeContainer) Get(metricName string, labelNames []string, labels prometheus.Labels, mc metricChecker, help string) (prometheus.Gauge, error) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
gaugeVec, ok := c.Elements[mapKey] gaugeVec, ok := c.Elements[mapKey]
@ -173,8 +176,7 @@ func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels, mc met
return gaugeVec.GetMetricWith(labels) return gaugeVec.GetMetricWith(labels)
} }
func (c *GaugeContainer) Delete(metricName string, labels prometheus.Labels) { func (c *GaugeContainer) Delete(metricName string, labelNames []string, labels prometheus.Labels) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok { if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels) c.Elements[mapKey].Delete(labels)
@ -194,8 +196,7 @@ func NewSummaryContainer(mapper *mapper.MetricMapper) *SummaryContainer {
} }
} }
func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, mc metricChecker, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) { func (c *SummaryContainer) Get(metricName string, labelNames []string, labels prometheus.Labels, mc metricChecker, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
summaryVec, ok := c.Elements[mapKey] summaryVec, ok := c.Elements[mapKey]
@ -227,7 +228,7 @@ func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, mc m
Name: metricName, Name: metricName,
Help: help, Help: help,
Objectives: objectives, Objectives: objectives,
}, getLabelNames(labels)) }, labelNames)
if err := prometheus.Register(uncheckedCollector{summaryVec}); err != nil { if err := prometheus.Register(uncheckedCollector{summaryVec}); err != nil {
return nil, err return nil, err
} }
@ -236,8 +237,7 @@ func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, mc m
return summaryVec.GetMetricWith(labels) return summaryVec.GetMetricWith(labels)
} }
func (c *SummaryContainer) Delete(metricName string, labels prometheus.Labels) { func (c *SummaryContainer) Delete(metricName string, labelNames []string, labels prometheus.Labels) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok { if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels) c.Elements[mapKey].Delete(labels)
@ -257,8 +257,7 @@ func NewHistogramContainer(mapper *mapper.MetricMapper) *HistogramContainer {
} }
} }
func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, mc metricChecker, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) { func (c *HistogramContainer) Get(metricName string, labelNames []string, labels prometheus.Labels, mc metricChecker, help string, mapping *mapper.MetricMapping) (prometheus.Observer, error) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
histogramVec, ok := c.Elements[mapKey] histogramVec, ok := c.Elements[mapKey]
@ -294,8 +293,7 @@ func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, mc
return histogramVec.GetMetricWith(labels) return histogramVec.GetMetricWith(labels)
} }
func (c *HistogramContainer) Delete(metricName string, labels prometheus.Labels) { func (c *HistogramContainer) Delete(metricName string, labelNames []string, labels prometheus.Labels) {
labelNames := getLabelNames(labels)
mapKey := getContainerMapKey(metricName, labelNames) mapKey := getContainerMapKey(metricName, labelNames)
if _, ok := c.Elements[mapKey]; ok { if _, ok := c.Elements[mapKey]; ok {
c.Elements[mapKey].Delete(labels) c.Elements[mapKey].Delete(labels)
@ -461,6 +459,7 @@ func (b *Exporter) handleEvent(event Event) {
metricName := "" metricName := ""
prometheusLabels := event.Labels() prometheusLabels := event.Labels()
sortedLabelNames := getSortedLabelNames(prometheusLabels)
if present { if present {
if mapping.Name == "" { if mapping.Name == "" {
log.Debugf("The mapping of '%s' for match '%s' generates an empty metric name", event.MetricName(), mapping.Match) log.Debugf("The mapping of '%s' for match '%s' generates an empty metric name", event.MetricName(), mapping.Match)
@ -489,13 +488,14 @@ func (b *Exporter) handleEvent(event Event) {
counter, err := b.Counters.Get( counter, err := b.Counters.Get(
metricName, metricName,
sortedLabelNames,
prometheusLabels, prometheusLabels,
b, b,
help, help,
) )
if err == nil { if err == nil {
counter.Add(event.Value()) counter.Add(event.Value())
b.saveLabelValues(metricName, CounterMetricType, prometheusLabels, mapping.Ttl) b.saveLabelValues(metricName, CounterMetricType, sortedLabelNames, prometheusLabels, mapping.Ttl)
eventStats.WithLabelValues("counter").Inc() eventStats.WithLabelValues("counter").Inc()
} else { } else {
log.Debugf(regErrF, metricName, err) log.Debugf(regErrF, metricName, err)
@ -505,6 +505,7 @@ func (b *Exporter) handleEvent(event Event) {
case *GaugeEvent: case *GaugeEvent:
gauge, err := b.Gauges.Get( gauge, err := b.Gauges.Get(
metricName, metricName,
sortedLabelNames,
prometheusLabels, prometheusLabels,
b, b,
help, help,
@ -516,7 +517,7 @@ func (b *Exporter) handleEvent(event Event) {
} else { } else {
gauge.Set(event.Value()) gauge.Set(event.Value())
} }
b.saveLabelValues(metricName, GaugeMetricType, prometheusLabels, mapping.Ttl) b.saveLabelValues(metricName, GaugeMetricType, sortedLabelNames, prometheusLabels, mapping.Ttl)
eventStats.WithLabelValues("gauge").Inc() eventStats.WithLabelValues("gauge").Inc()
} else { } else {
log.Debugf(regErrF, metricName, err) log.Debugf(regErrF, metricName, err)
@ -536,6 +537,7 @@ func (b *Exporter) handleEvent(event Event) {
case mapper.TimerTypeHistogram: case mapper.TimerTypeHistogram:
histogram, err := b.Histograms.Get( histogram, err := b.Histograms.Get(
metricName, metricName,
sortedLabelNames,
prometheusLabels, prometheusLabels,
b, b,
help, help,
@ -543,7 +545,7 @@ func (b *Exporter) handleEvent(event Event) {
) )
if err == nil { if err == nil {
histogram.Observe(event.Value() / 1000) // prometheus presumes seconds, statsd millisecond histogram.Observe(event.Value() / 1000) // prometheus presumes seconds, statsd millisecond
b.saveLabelValues(metricName, HistogramMetricType, prometheusLabels, mapping.Ttl) b.saveLabelValues(metricName, HistogramMetricType, sortedLabelNames, prometheusLabels, mapping.Ttl)
eventStats.WithLabelValues("timer").Inc() eventStats.WithLabelValues("timer").Inc()
} else { } else {
log.Debugf(regErrF, metricName, err) log.Debugf(regErrF, metricName, err)
@ -553,6 +555,7 @@ func (b *Exporter) handleEvent(event Event) {
case mapper.TimerTypeDefault, mapper.TimerTypeSummary: case mapper.TimerTypeDefault, mapper.TimerTypeSummary:
summary, err := b.Summaries.Get( summary, err := b.Summaries.Get(
metricName, metricName,
sortedLabelNames,
prometheusLabels, prometheusLabels,
b, b,
help, help,
@ -560,7 +563,7 @@ func (b *Exporter) handleEvent(event Event) {
) )
if err == nil { if err == nil {
summary.Observe(event.Value() / 1000) // prometheus presumes seconds, statsd millisecond summary.Observe(event.Value() / 1000) // prometheus presumes seconds, statsd millisecond
b.saveLabelValues(metricName, SummaryMetricType, prometheusLabels, mapping.Ttl) b.saveLabelValues(metricName, SummaryMetricType, sortedLabelNames, prometheusLabels, mapping.Ttl)
eventStats.WithLabelValues("timer").Inc() eventStats.WithLabelValues("timer").Inc()
} else { } else {
log.Debugf(regErrF, metricName, err) log.Debugf(regErrF, metricName, err)
@ -587,10 +590,11 @@ func (b *Exporter) removeStaleMetrics() {
continue continue
} }
if lvs.lastRegisteredAt.Add(lvs.ttl).Before(now) { if lvs.lastRegisteredAt.Add(lvs.ttl).Before(now) {
b.Counters.Delete(metricName, lvs.labels) sortedLabelNames := getSortedLabelNames(lvs.labels)
b.Gauges.Delete(metricName, lvs.labels) b.Counters.Delete(metricName, sortedLabelNames, lvs.labels)
b.Summaries.Delete(metricName, lvs.labels) b.Gauges.Delete(metricName, sortedLabelNames, lvs.labels)
b.Histograms.Delete(metricName, lvs.labels) b.Summaries.Delete(metricName, sortedLabelNames, lvs.labels)
b.Histograms.Delete(metricName, sortedLabelNames, lvs.labels)
delete(b.labelValues[metricName], hash) delete(b.labelValues[metricName], hash)
} }
} }
@ -598,13 +602,13 @@ func (b *Exporter) removeStaleMetrics() {
} }
// saveLabelValues stores label values set to labelValues and update lastRegisteredAt time and ttl value // saveLabelValues stores label values set to labelValues and update lastRegisteredAt time and ttl value
func (b *Exporter) saveLabelValues(metricName string, metricType metricType, labels prometheus.Labels, ttl time.Duration) { func (b *Exporter) saveLabelValues(metricName string, metricType metricType, labelNames []string, labels prometheus.Labels, ttl time.Duration) {
metric, hasMetric := b.labelValues[metricName] metric, hasMetric := b.labelValues[metricName]
if !hasMetric { if !hasMetric {
metric = make(map[uint64]*LabelValues) metric = make(map[uint64]*LabelValues)
b.labelValues[metricName] = metric b.labelValues[metricName] = metric
} }
hash := hashNameAndLabels(metricName, labels) hash := hashNameAndLabels(metricName, labelNames, labels)
metricLabelValues, ok := metric[hash] metricLabelValues, ok := metric[hash]
if !ok { if !ok {
metricLabelValues = &LabelValues{ metricLabelValues = &LabelValues{

51
exporter_test.go
View file

@ -724,9 +724,12 @@ func getFloat64(metrics []*dto.MetricFamily, name string, labels prometheus.Labe
} }
var metric *dto.Metric var metric *dto.Metric
labelsHash := hashNameAndLabels(name, labels) sortedLabelNames := getSortedLabelNames(labels)
labelsHash := hashNameAndLabels(name, sortedLabelNames, labels)
for _, m := range metricFamily.Metric { for _, m := range metricFamily.Metric {
h := hashNameAndLabels(name, labelPairsAsLabels(m.GetLabel())) l := labelPairsAsLabels(m.GetLabel())
sln := getSortedLabelNames(l)
h := hashNameAndLabels(name, sln, l)
if h == labelsHash { if h == labelsHash {
metric = m metric = m
break break
@ -822,3 +825,47 @@ func BenchmarkParseDogStatsDTagsToLabels(b *testing.B) {
}) })
} }
} }
func BenchmarkHashNameAndLabels(b *testing.B) {
scenarios := []struct {
name string
metric string
labels map[string]string
}{
{
name: "no labels",
metric: "counter",
labels: map[string]string{},
}, {
name: "one label",
metric: "counter",
labels: map[string]string{
"label": "value",
},
}, {
name: "many labels",
metric: "counter",
labels: map[string]string{
"label0": "value",
"label1": "value",
"label2": "value",
"label3": "value",
"label4": "value",
"label5": "value",
"label6": "value",
"label7": "value",
"label8": "value",
"label9": "value",
},
},
}
for _, s := range scenarios {
sortedLabelNames := getSortedLabelNames(s.labels)
b.Run(s.name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
hashNameAndLabels(s.metric, sortedLabelNames, s.labels)
}
})
}
}