rafaelcaricio/statsd_exporter
1
0
Fork 0
forked from mirrors/statsd_exporter
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #314 from glightfoot/histogram-type

Browse source 
[307] Add support for histograms and distributions without unit conversion
This commit is contained in:
Matthias Rampke 2020-06-19 13:00:06 +02:00 committed by GitHub
commit 7ba3550f8f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 449 additions and 224 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

184
bridge_test.go
View file

@ -83,60 +83,60 @@ func TestHandlePacket(t *testing.T) {
name: "simple timer", name: "simple timer",
in: "foo:200|ms", in: "foo:200|ms",
out: event.Events{ out: event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 200, OValue: 0.2,
TLabels: map[string]string{}, OLabels: map[string]string{},
}, },
}, },
}, { }, {
name: "simple histogram", name: "simple histogram",
in: "foo:200|h", in: "foo:200|h",
out: event.Events{ out: event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 200, OValue: 200,
TLabels: map[string]string{}, OLabels: map[string]string{},
}, },
}, },
}, { }, {
name: "simple distribution", name: "simple distribution",
in: "foo:200|d", in: "foo:200|d",
out: event.Events{ out: event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 200, OValue: 200,
TLabels: map[string]string{}, OLabels: map[string]string{},
}, },
}, },
}, { }, {
name: "distribution with sampling", name: "distribution with sampling",
in: "foo:0.01|d|@0.2|#tag1:bar,#tag2:baz", in: "foo:0.01|d|@0.2|#tag1:bar,#tag2:baz",
out: event.Events{ out: event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
}, },
}, { }, {
@ -265,30 +265,30 @@ func TestHandlePacket(t *testing.T) {
name: "histogram with sampling", name: "histogram with sampling",
in: "foo:0.01|h|@0.2|#tag1:bar,#tag2:baz", in: "foo:0.01|h|@0.2|#tag1:bar,#tag2:baz",
out: event.Events{ out: event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 0.01, OValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
}, },
}, },
}, { }, {
@ -321,15 +321,15 @@ func TestHandlePacket(t *testing.T) {
name: "combined multiline metrics", name: "combined multiline metrics",
in: "foo:200|ms:300|ms:5|c|@0.1:6|g\nbar:1|c:5|ms", in: "foo:200|ms:300|ms:5|c|@0.1:6|g\nbar:1|c:5|ms",
out: event.Events{ out: event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 200, OValue: .200,
TLabels: map[string]string{}, OLabels: map[string]string{},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "foo", OMetricName: "foo",
TValue: 300, OValue: .300,
TLabels: map[string]string{}, OLabels: map[string]string{},
}, },
&event.CounterEvent{ &event.CounterEvent{
CMetricName: "foo", CMetricName: "foo",
@ -346,26 +346,26 @@ func TestHandlePacket(t *testing.T) {
CValue: 1, CValue: 1,
CLabels: map[string]string{}, CLabels: map[string]string{},
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "bar", OMetricName: "bar",
TValue: 5, OValue: .005,
TLabels: map[string]string{}, OLabels: map[string]string{},
}, },
}, },
}, { }, {
name: "timings with sampling factor", name: "timings with sampling factor",
in: "foo.timing:0.5|ms|@0.1", in: "foo.timing:0.5|ms|@0.1",
out: event.Events{ out: event.Events{
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
}, },
}, { }, {
name: "bad line", name: "bad line",
@ -422,6 +422,36 @@ func TestHandlePacket(t *testing.T) {
CLabels: map[string]string{}, CLabels: map[string]string{},
}, },
}, },
}, {
name: "ms timer with conversion to seconds",
in: "foo:200|ms",
out: event.Events{
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.2,
OLabels: map[string]string{},
},
},
}, {
name: "histogram with no unit conversion",
in: "foo:200|h",
out: event.Events{
&event.ObserverEvent{
OMetricName: "foo",
OValue: 200,
OLabels: map[string]string{},
},
},
}, {
name: "distribution with no unit conversion",
in: "foo:200|d",
out: event.Events{
&event.ObserverEvent{
OMetricName: "foo",
OValue: 200,
OLabels: map[string]string{},
},
},
}, },
} }
@ -554,9 +584,9 @@ mappings:
GValue: 200, GValue: 200,
}, },
// event with ttl = 2s from a mapping // event with ttl = 2s from a mapping
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "bazqux.main", OMetricName: "bazqux.main",
TValue: 42000, OValue: 42,
}, },
} }

24
exporter_benchmark_test.go
View file

@ -18,6 +18,7 @@ import (
"testing" "testing"
"github.com/go-kit/kit/log" "github.com/go-kit/kit/log"
"github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/event"
"github.com/prometheus/statsd_exporter/pkg/exporter" "github.com/prometheus/statsd_exporter/pkg/exporter"
"github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/listener"
@ -38,6 +39,7 @@ func benchmarkUDPListener(times int, b *testing.B) {
"some_very_useful_metrics_with_quite_a_log_name:13|c", "some_very_useful_metrics_with_quite_a_log_name:13|c",
} }
bytesInput := make([]string, len(input)*times) bytesInput := make([]string, len(input)*times)
logger := log.NewNopLogger()
for run := 0; run < times; run++ { for run := 0; run < times; run++ {
for i := 0; i < len(input); i++ { for i := 0; i < len(input); i++ {
bytesInput[run*len(input)+i] = fmt.Sprintf("run%d%s", run, input[i]) bytesInput[run*len(input)+i] = fmt.Sprintf("run%d%s", run, input[i])
@ -46,7 +48,15 @@ func benchmarkUDPListener(times int, b *testing.B) {
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
// there are more events than input lines, need bigger buffer // there are more events than input lines, need bigger buffer
events := make(chan event.Events, len(bytesInput)*times*2) events := make(chan event.Events, len(bytesInput)*times*2)
l := listener.StatsDUDPListener{EventHandler: &event.UnbufferedEventHandler{C: events}}
l := listener.StatsDUDPListener{
EventHandler: &event.UnbufferedEventHandler{C: events},
Logger: logger,
UDPPackets: udpPackets,
LinesReceived: linesReceived,
SamplesReceived: samplesReceived,
TagsReceived: tagsReceived,
}
for i := 0; i < times; i++ { for i := 0; i < times; i++ {
for _, line := range bytesInput { for _, line := range bytesInput {
@ -76,13 +86,13 @@ func BenchmarkExporterListener(b *testing.B) {
GMetricName: "gauge", GMetricName: "gauge",
GValue: 10, GValue: 10,
}, },
&event.TimerEvent{ // simple timer &event.ObserverEvent{ // simple timer
TMetricName: "timer", OMetricName: "timer",
TValue: 200, OValue: 200,
}, },
&event.TimerEvent{ // simple histogram &event.ObserverEvent{ // simple histogram
TMetricName: "histogram.test", OMetricName: "histogram.test",
TValue: 200, OValue: 200,
}, },
&event.CounterEvent{ // simple_tags &event.CounterEvent{ // simple_tags
CMetricName: "simple_tags", CMetricName: "simple_tags",

16
pkg/event/event.go
View file

@ -52,16 +52,16 @@ func (g *GaugeEvent) Value() float64 { return g.GValue }
func (c *GaugeEvent) Labels() map[string]string { return c.GLabels } func (c *GaugeEvent) Labels() map[string]string { return c.GLabels }
func (c *GaugeEvent) MetricType() mapper.MetricType { return mapper.MetricTypeGauge } func (c *GaugeEvent) MetricType() mapper.MetricType { return mapper.MetricTypeGauge }
type TimerEvent struct { type ObserverEvent struct {
TMetricName string OMetricName string
TValue float64 OValue float64
TLabels map[string]string OLabels map[string]string
} }
func (t *TimerEvent) MetricName() string { return t.TMetricName } func (t *ObserverEvent) MetricName() string { return t.OMetricName }
func (t *TimerEvent) Value() float64 { return t.TValue } func (t *ObserverEvent) Value() float64 { return t.OValue }
func (c *TimerEvent) Labels() map[string]string { return c.TLabels } func (c *ObserverEvent) Labels() map[string]string { return c.OLabels }
func (c *TimerEvent) MetricType() mapper.MetricType { return mapper.MetricTypeTimer } func (c *ObserverEvent) MetricType() mapper.MetricType { return mapper.MetricTypeObserver }
type Events []Event type Events []Event

28
pkg/exporter/exporter.go
View file

@ -140,38 +140,38 @@ func (b *Exporter) handleEvent(thisEvent event.Event) {
b.ConflictingEventStats.WithLabelValues("gauge").Inc() b.ConflictingEventStats.WithLabelValues("gauge").Inc()
} }
case *event.TimerEvent: case *event.ObserverEvent:
t := mapper.TimerTypeDefault t := mapper.ObserverTypeDefault
if mapping != nil { if mapping != nil {
t = mapping.TimerType t = mapping.ObserverType
} }
if t == mapper.TimerTypeDefault { if t == mapper.ObserverTypeDefault {
t = b.Mapper.Defaults.TimerType t = b.Mapper.Defaults.ObserverType
} }
switch t { switch t {
case mapper.TimerTypeHistogram: case mapper.ObserverTypeHistogram:
histogram, err := b.Registry.GetHistogram(metricName, prometheusLabels, help, mapping, b.MetricsCount) histogram, err := b.Registry.GetHistogram(metricName, prometheusLabels, help, mapping, b.MetricsCount)
if err == nil { if err == nil {
histogram.Observe(thisEvent.Value() / 1000) // prometheus presumes seconds, statsd millisecond histogram.Observe(thisEvent.Value())
b.EventStats.WithLabelValues("timer").Inc() b.EventStats.WithLabelValues("observer").Inc()
} else { } else {
level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err) level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err)
b.ConflictingEventStats.WithLabelValues("timer").Inc() b.ConflictingEventStats.WithLabelValues("observer").Inc()
} }
case mapper.TimerTypeDefault, mapper.TimerTypeSummary: case mapper.ObserverTypeDefault, mapper.ObserverTypeSummary:
summary, err := b.Registry.GetSummary(metricName, prometheusLabels, help, mapping, b.MetricsCount) summary, err := b.Registry.GetSummary(metricName, prometheusLabels, help, mapping, b.MetricsCount)
if err == nil { if err == nil {
summary.Observe(thisEvent.Value() / 1000) // prometheus presumes seconds, statsd millisecond summary.Observe(thisEvent.Value())
b.EventStats.WithLabelValues("timer").Inc() b.EventStats.WithLabelValues("observer").Inc()
} else { } else {
level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err) level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err)
b.ConflictingEventStats.WithLabelValues("timer").Inc() b.ConflictingEventStats.WithLabelValues("observer").Inc()
} }
default: default:
level.Error(b.Logger).Log("msg", "unknown timer type", "type", t) level.Error(b.Logger).Log("msg", "unknown observer type", "type", t)
os.Exit(1) os.Exit(1)
} }

98
pkg/exporter/exporter_test.go
View file

@ -228,25 +228,25 @@ func TestInconsistentLabelSets(t *testing.T) {
GValue: 1, GValue: 1,
GLabels: secondLabelSet, GLabels: secondLabelSet,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "histogram.test", OMetricName: "histogram.test",
TValue: 1, OValue: 1,
TLabels: firstLabelSet, OLabels: firstLabelSet,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "histogram.test", OMetricName: "histogram.test",
TValue: 1, OValue: 1,
TLabels: secondLabelSet, OLabels: secondLabelSet,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "summary_test", OMetricName: "summary_test",
TValue: 1, OValue: 1,
TLabels: firstLabelSet, OLabels: firstLabelSet,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "summary_test", OMetricName: "summary_test",
TValue: 1, OValue: 1,
TLabels: secondLabelSet, OLabels: secondLabelSet,
}, },
} }
events <- c events <- c
@ -427,9 +427,9 @@ func TestConflictingMetrics(t *testing.T) {
CMetricName: "histogram_test1", CMetricName: "histogram_test1",
CValue: 1, CValue: 1,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "histogram.test1", OMetricName: "histogram.test1",
TValue: 2, OValue: 2,
}, },
}, },
}, },
@ -441,9 +441,9 @@ func TestConflictingMetrics(t *testing.T) {
CMetricName: "histogram_test1_sum", CMetricName: "histogram_test1_sum",
CValue: 1, CValue: 1,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "histogram.test1", OMetricName: "histogram.test1",
TValue: 2, OValue: 2,
}, },
}, },
}, },
@ -455,9 +455,9 @@ func TestConflictingMetrics(t *testing.T) {
CMetricName: "histogram_test2_count", CMetricName: "histogram_test2_count",
CValue: 1, CValue: 1,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "histogram.test2", OMetricName: "histogram.test2",
TValue: 2, OValue: 2,
}, },
}, },
}, },
@ -469,9 +469,9 @@ func TestConflictingMetrics(t *testing.T) {
CMetricName: "histogram_test3_bucket", CMetricName: "histogram_test3_bucket",
CValue: 1, CValue: 1,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "histogram.test3", OMetricName: "histogram.test3",
TValue: 2, OValue: 2,
}, },
}, },
}, },
@ -483,9 +483,9 @@ func TestConflictingMetrics(t *testing.T) {
CMetricName: "cvsq_test", CMetricName: "cvsq_test",
CValue: 1, CValue: 1,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "cvsq_test", OMetricName: "cvsq_test",
TValue: 2, OValue: 2,
}, },
}, },
}, },
@ -497,9 +497,9 @@ func TestConflictingMetrics(t *testing.T) {
CMetricName: "cvsc_count", CMetricName: "cvsc_count",
CValue: 1, CValue: 1,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "cvsc", OMetricName: "cvsc",
TValue: 2, OValue: 2,
}, },
}, },
}, },
@ -511,9 +511,9 @@ func TestConflictingMetrics(t *testing.T) {
CMetricName: "cvss_sum", CMetricName: "cvss_sum",
CValue: 1, CValue: 1,
}, },
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "cvss", OMetricName: "cvss",
TValue: 2, OValue: 2,
}, },
}, },
}, },
@ -672,9 +672,9 @@ func TestSummaryWithQuantilesEmptyMapping(t *testing.T) {
name := "default_foo" name := "default_foo"
c := event.Events{ c := event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: name, OMetricName: name,
TValue: 300, OValue: 300,
}, },
} }
events <- c events <- c
@ -711,7 +711,7 @@ func TestHistogramUnits(t *testing.T) {
testMapper := mapper.MetricMapper{} testMapper := mapper.MetricMapper{}
testMapper.InitCache(0) testMapper.InitCache(0)
ex := NewExporter(&testMapper, log.NewNopLogger(), eventsActions, eventsUnmapped, errorEventStats, eventStats, conflictingEventStats, metricsCount) ex := NewExporter(&testMapper, log.NewNopLogger(), eventsActions, eventsUnmapped, errorEventStats, eventStats, conflictingEventStats, metricsCount)
ex.Mapper.Defaults.TimerType = mapper.TimerTypeHistogram ex.Mapper.Defaults.ObserverType = mapper.ObserverTypeHistogram
ex.Listen(events) ex.Listen(events)
}() }()
@ -719,9 +719,9 @@ func TestHistogramUnits(t *testing.T) {
// Then close events channel to stop a listener. // Then close events channel to stop a listener.
name := "foo" name := "foo"
c := event.Events{ c := event.Events{
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: name, OMetricName: name,
TValue: 300, OValue: .300,
}, },
} }
events <- c events <- c
@ -737,9 +737,7 @@ func TestHistogramUnits(t *testing.T) {
if value == nil { if value == nil {
t.Fatal("Histogram value should not be nil") t.Fatal("Histogram value should not be nil")
} }
if *value == 300 { if *value != .300 {
t.Fatalf("Histogram observations not scaled into Seconds")
} else if *value != .300 {
t.Fatalf("Received unexpected value for histogram observation %f != .300", *value) t.Fatalf("Received unexpected value for histogram observation %f != .300", *value)
} }
} }
@ -869,9 +867,9 @@ mappings:
GValue: 200, GValue: 200,
}, },
// event with ttl = 2s from a mapping // event with ttl = 2s from a mapping
&event.TimerEvent{ &event.ObserverEvent{
TMetricName: "bazqux.main", OMetricName: "bazqux.main",
TValue: 42000, OValue: 42,
}, },
} }

16
pkg/line/line.go
View file

@ -42,11 +42,17 @@ func buildEvent(statType, metric string, value float64, relative bool, labels ma
GRelative: relative, GRelative: relative,
GLabels: labels, GLabels: labels,
}, nil }, nil
case "ms", "h", "d": case "ms":
return &event.TimerEvent{ return &event.ObserverEvent{
TMetricName: metric, OMetricName: metric,
TValue: float64(value), OValue: float64(value) / 1000, // prometheus presumes seconds, statsd millisecond
TLabels: labels, OLabels: labels,
}, nil
case "h", "d":
return &event.ObserverEvent{
OMetricName: metric,
OValue: float64(value),
OLabels: labels,
}, nil }, nil
case "s": case "s":
return nil, fmt.Errorf("no support for StatsD sets") return nil, fmt.Errorf("no support for StatsD sets")

10
pkg/mapper/fsm/README.md
View file

@ -40,7 +40,7 @@ At first, the FSM only contains three states, representing three possible metric
/ /
(start)---- [counter] (start)---- [counter]
\ \
'--- [ timer ] '--- [observer]
Adding a rule `client.*.request.count` with type `counter` will make the FSM to be: Adding a rule `client.*.request.count` with type `counter` will make the FSM to be:
@ -50,7 +50,7 @@ Adding a rule `client.*.request.count` with type `counter` will make the FSM to
/ /
(start)---- [counter] -- [client] -- [*] -- [request] -- [count] -- {R1} (start)---- [counter] -- [client] -- [*] -- [request] -- [count] -- {R1}
\ \
'--- [timer] '--- [observer]
`{R1}` is short for result 1, which is the match result for `client.*.request.count`. `{R1}` is short for result 1, which is the match result for `client.*.request.count`.
@ -60,7 +60,7 @@ Adding a rule `client.*.*.size` with type `counter` will make the FSM to be:
/ / / /
(start)---- [counter] -- [client] -- [*] (start)---- [counter] -- [client] -- [*]
\ \__ [*] -- [size] -- {R2} \ \__ [*] -- [size] -- {R2}
'--- [timer] '--- [observer]
### Finding a result state in FSM ### Finding a result state in FSM
@ -76,7 +76,7 @@ FSM, the `^1` to `^7` symbols indicate how FSM will traversal in its tree:
/ / ^5 ^6 ^7 / / ^5 ^6 ^7
(start)---- [counter] -- [client] -- [*] (start)---- [counter] -- [client] -- [*]
^1 \ ^2 ^3 \__ [*] -- [size] -- {R2} ^1 \ ^2 ^3 \__ [*] -- [size] -- {R2}
'--- [timer] ^4 '--- [observer] ^4
To map `client.bbb.request.size`, FSM will do a backtracking: To map `client.bbb.request.size`, FSM will do a backtracking:
@ -86,7 +86,7 @@ To map `client.bbb.request.size`, FSM will do a backtracking:
/ / ^5 ^6 / / ^5 ^6
(start)---- [counter] -- [client] -- [*] (start)---- [counter] -- [client] -- [*]
^1 \ ^2 ^3 \__ [*] -- [size] -- {R2} ^1 \ ^2 ^3 \__ [*] -- [size] -- {R2}
'--- [timer] ^4 '--- [observer] ^4
^7 ^8 ^9 ^7 ^8 ^9

43
pkg/mapper/mapper.go
View file

@ -35,15 +35,6 @@ var (
labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`) labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`)
) )
type mapperConfigDefaults struct {
TimerType TimerType `yaml:"timer_type"`
Buckets []float64 `yaml:"buckets"`
Quantiles []metricObjective `yaml:"quantiles"`
MatchType MatchType `yaml:"match_type"`
GlobDisableOrdering bool `yaml:"glob_disable_ordering"`
Ttl time.Duration `yaml:"ttl"`
}
type MetricMapper struct { type MetricMapper struct {
Defaults mapperConfigDefaults `yaml:"defaults"` Defaults mapperConfigDefaults `yaml:"defaults"`
Mappings []MetricMapping `yaml:"mappings"` Mappings []MetricMapping `yaml:"mappings"`
@ -56,26 +47,6 @@ type MetricMapper struct {
MappingsCount prometheus.Gauge MappingsCount prometheus.Gauge
} }
type MetricMapping struct {
Match string `yaml:"match"`
Name string `yaml:"name"`
nameFormatter *fsm.TemplateFormatter
regex *regexp.Regexp
Labels prometheus.Labels `yaml:"labels"`
labelKeys []string
labelFormatters []*fsm.TemplateFormatter
TimerType TimerType `yaml:"timer_type"`
LegacyBuckets []float64 `yaml:"buckets"`
LegacyQuantiles []metricObjective `yaml:"quantiles"`
MatchType MatchType `yaml:"match_type"`
HelpText string `yaml:"help"`
Action ActionType `yaml:"action"`
MatchMetricType MetricType `yaml:"match_metric_type"`
Ttl time.Duration `yaml:"ttl"`
SummaryOptions *SummaryOptions `yaml:"summary_options"`
HistogramOptions *HistogramOptions `yaml:"histogram_options"`
}
type SummaryOptions struct { type SummaryOptions struct {
Quantiles []metricObjective `yaml:"quantiles"` Quantiles []metricObjective `yaml:"quantiles"`
MaxAge time.Duration `yaml:"max_age"` MaxAge time.Duration `yaml:"max_age"`
@ -119,7 +90,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op
remainingMappingsCount := len(n.Mappings) remainingMappingsCount := len(n.Mappings)
n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeTimer)}, n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)},
remainingMappingsCount, n.Defaults.GlobDisableOrdering) remainingMappingsCount, n.Defaults.GlobDisableOrdering)
for i := range n.Mappings { for i := range n.Mappings {
@ -181,8 +152,8 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op
n.doRegex = true n.doRegex = true
} }
if currentMapping.TimerType == "" { if currentMapping.ObserverType == "" {
currentMapping.TimerType = n.Defaults.TimerType currentMapping.ObserverType = n.Defaults.ObserverType
} }
if currentMapping.LegacyQuantiles != nil && if currentMapping.LegacyQuantiles != nil &&
@ -207,9 +178,9 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op
return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match) return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match)
} }
if currentMapping.TimerType == TimerTypeHistogram { if currentMapping.ObserverType == ObserverTypeHistogram {
if currentMapping.SummaryOptions != nil { if currentMapping.SummaryOptions != nil {
return fmt.Errorf("cannot use histogram timer and summary options at the same time") return fmt.Errorf("cannot use histogram observer and summary options at the same time")
} }
if currentMapping.HistogramOptions == nil { if currentMapping.HistogramOptions == nil {
currentMapping.HistogramOptions = &HistogramOptions{} currentMapping.HistogramOptions = &HistogramOptions{}
@ -222,9 +193,9 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op
} }
} }
if currentMapping.TimerType == TimerTypeSummary { if currentMapping.ObserverType == ObserverTypeSummary {
if currentMapping.HistogramOptions != nil { if currentMapping.HistogramOptions != nil {
return fmt.Errorf("cannot use summary timer and histogram options at the same time") return fmt.Errorf("cannot use summary observer and histogram options at the same time")
} }
if currentMapping.SummaryOptions == nil { if currentMapping.SummaryOptions == nil {
currentMapping.SummaryOptions = &SummaryOptions{} currentMapping.SummaryOptions = &SummaryOptions{}

51
pkg/mapper/mapper_defaults.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import "time"
type mapperConfigDefaults struct {
ObserverType ObserverType `yaml:"observer_type"`
TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty
Buckets []float64 `yaml:"buckets"`
Quantiles []metricObjective `yaml:"quantiles"`
MatchType MatchType `yaml:"match_type"`
GlobDisableOrdering bool `yaml:"glob_disable_ordering"`
Ttl time.Duration `yaml:"ttl"`
}
// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys
// observer_type will override timer_type
func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error {
type mapperConfigDefaultsAlias mapperConfigDefaults
var tmp mapperConfigDefaultsAlias
if err := unmarshal(&tmp); err != nil {
return err
}
// Copy defaults
d.ObserverType = tmp.ObserverType
d.Buckets = tmp.Buckets
d.Quantiles = tmp.Quantiles
d.MatchType = tmp.MatchType
d.GlobDisableOrdering = tmp.GlobDisableOrdering
d.Ttl = tmp.Ttl
// Use deprecated TimerType if necessary
if tmp.ObserverType == "" {
d.ObserverType = tmp.TimerType
}
return nil
}

94
pkg/mapper/mapper_test.go
View file

@ -441,12 +441,39 @@ mappings:
}, },
}, },
}, },
// Config with good timer type. // Config with good observer type.
{ {
config: `--- config: `---
mappings: mappings:
- match: test.*.* - match: test.*.*
timer_type: summary observer_type: summary
name: "foo"
labels: {}
quantiles:
- quantile: 0.42
error: 0.04
- quantile: 0.7
error: 0.002
`,
mappings: mappings{
{
statsdMetric: "test.*.*",
name: "foo",
labels: map[string]string{},
quantiles: []metricObjective{
{Quantile: 0.42, Error: 0.04},
{Quantile: 0.7, Error: 0.002},
},
},
},
},
// Config with good observer type and unused timer type
{
config: `---
mappings:
- match: test.*.*
observer_type: summary
timer_type: histogram
name: "foo" name: "foo"
labels: {} labels: {}
quantiles: quantiles:
@ -470,6 +497,28 @@ mappings:
{ {
config: `--- config: `---
mappings: mappings:
- match: test1.*.*
observer_type: summary
name: "foo"
labels: {}
`,
mappings: mappings{
{
statsdMetric: "test1.*.*",
name: "foo",
labels: map[string]string{},
quantiles: []metricObjective{
{Quantile: 0.5, Error: 0.05},
{Quantile: 0.9, Error: 0.01},
{Quantile: 0.99, Error: 0.001},
},
},
},
},
// Config with good deprecated timer type
{
config: `---
mappings:
- match: test1.*.* - match: test1.*.*
timer_type: summary timer_type: summary
name: "foo" name: "foo"
@ -488,7 +537,18 @@ mappings:
}, },
}, },
}, },
// Config with bad timer type. // Config with bad observer type.
{
config: `---
mappings:
- match: test.*.*
observer_type: wrong
name: "foo"
labels: {}
`,
configBad: true,
},
// Config with bad deprecated timer type.
{ {
config: `--- config: `---
mappings: mappings:
@ -504,7 +564,7 @@ mappings:
config: `--- config: `---
mappings: mappings:
- match: test.*.* - match: test.*.*
timer_type: summary observer_type: summary
name: "foo" name: "foo"
labels: {} labels: {}
summary_options: summary_options:
@ -531,7 +591,7 @@ mappings:
config: `--- config: `---
mappings: mappings:
- match: test.*.* - match: test.*.*
timer_type: summary observer_type: summary
name: "foo" name: "foo"
labels: {} labels: {}
summary_options: summary_options:
@ -564,7 +624,7 @@ mappings:
config: `--- config: `---
mappings: mappings:
- match: test.*.* - match: test.*.*
timer_type: summary observer_type: summary
name: "foo" name: "foo"
labels: {} labels: {}
quantiles: quantiles:
@ -584,6 +644,26 @@ mappings:
- match: test.*.* - match: test.*.*
match_metric_type: counter match_metric_type: counter
name: "foo" name: "foo"
labels: {}
`,
},
// Config with good metric type observer.
{
config: `---
mappings:
- match: test.*.*
match_metric_type: observer
name: "foo"
labels: {}
`,
},
// Config with good metric type timer.
{
config: `---
mappings:
- match: test.*.*
match_metric_type: timer
name: "foo"
labels: {} labels: {}
`, `,
}, },
@ -638,7 +718,7 @@ mappings:
config: `--- config: `---
mappings: mappings:
- match: foo.*.* - match: foo.*.*
timer_type: summary observer_type: summary
name: "foo" name: "foo"
labels: {} labels: {}
`, `,

76
pkg/mapper/mapping.go Normal file
View file

@ -0,0 +1,76 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either xpress or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import (
"regexp"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
)
type MetricMapping struct {
Match string `yaml:"match"`
Name string `yaml:"name"`
nameFormatter *fsm.TemplateFormatter
regex *regexp.Regexp
Labels prometheus.Labels `yaml:"labels"`
labelKeys []string
labelFormatters []*fsm.TemplateFormatter
ObserverType ObserverType `yaml:"observer_type"`
TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty
LegacyBuckets []float64 `yaml:"buckets"`
LegacyQuantiles []metricObjective `yaml:"quantiles"`
MatchType MatchType `yaml:"match_type"`
HelpText string `yaml:"help"`
Action ActionType `yaml:"action"`
MatchMetricType MetricType `yaml:"match_metric_type"`
Ttl time.Duration `yaml:"ttl"`
SummaryOptions *SummaryOptions `yaml:"summary_options"`
HistogramOptions *HistogramOptions `yaml:"histogram_options"`
}
// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys
// observer_type will override timer_type
func (m *MetricMapping) UnmarshalYAML(unmarshal func(interface{}) error) error {
type MetricMappingAlias MetricMapping
var tmp MetricMappingAlias
if err := unmarshal(&tmp); err != nil {
return err
}
// Copy defaults
m.Match = tmp.Match
m.Name = tmp.Name
m.Labels = tmp.Labels
m.ObserverType = tmp.ObserverType
m.LegacyBuckets = tmp.LegacyBuckets
m.LegacyQuantiles = tmp.LegacyQuantiles
m.MatchType = tmp.MatchType
m.HelpText = tmp.HelpText
m.Action = tmp.Action
m.MatchMetricType = tmp.MatchMetricType
m.Ttl = tmp.Ttl
m.SummaryOptions = tmp.SummaryOptions
m.HistogramOptions = tmp.HistogramOptions
// Use deprecated TimerType if necessary
if tmp.ObserverType == "" {
m.ObserverType = tmp.TimerType
}
return nil
}

7
pkg/mapper/metric_type.go
View file

@ -20,7 +20,8 @@ type MetricType string
const ( const (
MetricTypeCounter MetricType = "counter" MetricTypeCounter MetricType = "counter"
MetricTypeGauge MetricType = "gauge" MetricTypeGauge MetricType = "gauge"
MetricTypeTimer MetricType = "timer" MetricTypeObserver MetricType = "observer"
MetricTypeTimer MetricType = "timer" // DEPRECATED
) )
func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error { func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error {
@ -34,8 +35,10 @@ func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error {
*m = MetricTypeCounter *m = MetricTypeCounter
case MetricTypeGauge: case MetricTypeGauge:
*m = MetricTypeGauge *m = MetricTypeGauge
case MetricTypeObserver:
*m = MetricTypeObserver
case MetricTypeTimer: case MetricTypeTimer:
*m = MetricTypeTimer *m = MetricTypeObserver
default: default:
return fmt.Errorf("invalid metric type '%s'", v) return fmt.Errorf("invalid metric type '%s'", v)
} }

22
pkg/mapper/timer.go → pkg/mapper/observer.go
View file

@ -15,27 +15,27 @@ package mapper
import "fmt" import "fmt"
type TimerType string type ObserverType string
const ( const (
TimerTypeHistogram TimerType = "histogram" ObserverTypeHistogram ObserverType = "histogram"
TimerTypeSummary TimerType = "summary" ObserverTypeSummary ObserverType = "summary"
TimerTypeDefault TimerType = "" ObserverTypeDefault ObserverType = ""
) )
func (t *TimerType) UnmarshalYAML(unmarshal func(interface{}) error) error { func (t *ObserverType) UnmarshalYAML(unmarshal func(interface{}) error) error {
var v string var v string
if err := unmarshal(&v); err != nil { if err := unmarshal(&v); err != nil {
return err return err
} }
switch TimerType(v) { switch ObserverType(v) {
case TimerTypeHistogram: case ObserverTypeHistogram:
*t = TimerTypeHistogram *t = ObserverTypeHistogram
case TimerTypeSummary, TimerTypeDefault: case ObserverTypeSummary, ObserverTypeDefault:
*t = TimerTypeSummary *t = ObserverTypeSummary
default: default:
return fmt.Errorf("invalid timer type '%s'", v) return fmt.Errorf("invalid observer type '%s'", v)
} }
return nil return nil
} }