From 38414f106a9b67bc2e8ff9a87a40082ddb8ebf72 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Tue, 16 Jun 2020 16:31:06 -0400 Subject: [PATCH 1/6] add benchmark for LineToEvents Signed-off-by: glightfoot --- line_benchmark_test.go | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 line_benchmark_test.go diff --git a/line_benchmark_test.go b/line_benchmark_test.go new file mode 100644 index 0000000..68bbdec --- /dev/null +++ b/line_benchmark_test.go @@ -0,0 +1,59 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/go-kit/kit/log" + + "github.com/prometheus/statsd_exporter/pkg/line" +) + +func benchmarkLineToEvents(times int, b *testing.B) { + input := []string{ + "foo1:2|c", + "foo2:3|g", + "foo3:200|ms", + "foo4:100|c|#tag1:bar,tag2:baz", + "foo5:100|c|#tag1:bar,#tag2:baz", + "foo6:100|c|#09digits:0,tag.with.dots:1", + "foo10:100|c|@0.1|#tag1:bar,#tag2:baz", + "foo11:100|c|@0.1|#tag1:foo:bar", + "foo.[foo=bar,dim=val]test:1|g", + "foo15:200|ms:300|ms:5|c|@0.1:6|g\nfoo15a:1|c:5|ms", + "some_very_useful_metrics_with_quite_a_log_name:13|c", + } + logger := log.NewNopLogger() + + for n := 0; n < b.N; n++ { + + for i := 0; i < times; i++ { + for _, l := range input { + line.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, logger) + } + } + } +} + + +func BenchmarkLineToEvents1(b *testing.B) { + benchmarkLineToEvents(1, b) +} +func BenchmarkLineToEvents5(b *testing.B) { + benchmarkLineToEvents(5, b) +} +func BenchmarkLineToEvents50(b *testing.B) { + benchmarkLineToEvents(50, b) +} \ No newline at end of file From 32f1677f81c1962c93b25b95fa8fd3fa971fa472 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Tue, 16 Jun 2020 16:32:59 -0400 Subject: [PATCH 2/6] add signalfx tag parsing Signed-off-by: glightfoot --- bridge_test.go | 10 ++++++++++ pkg/line/line.go | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/bridge_test.go b/bridge_test.go index f45cefa..f008d1d 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -169,6 +169,16 @@ func TestHandlePacket(t *testing.T) { CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, + }, { + name: "SignalFx tag extension", + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, }, { name: "influxdb tag extension with tag keys unsupported by prometheus", in: "foo,09digits=0,tag.with.dots=1:100|c", diff --git a/pkg/line/line.go b/pkg/line/line.go index 730dbcf..b1bd478 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -21,8 +21,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/mapper" ) @@ -126,6 +126,18 @@ func ParseDogStatsDTags(component string, labels map[string]string, tagErrors pr } func parseNameAndTags(name string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) string { + // check for SignalFx tags first + // `[` delimits start of tags by SignalFx + // `]` delimits end of tags by SignalFx + // https://docs.signalfx.com/en/latest/integrations/agent/monitors/collectd-statsd.html + startIdx := strings.IndexRune(name, '[') + endIdx := strings.IndexRune(name, ']') + + if startIdx != -1 && endIdx != -1 { + parseNameTags(name[startIdx+1:endIdx], labels, tagErrors, logger) + return name[:startIdx] + name[endIdx+1:] + } + for i, c := range name { // `#` delimits start of tags by Librato // https://www.librato.com/docs/kb/collect/collection_agents/stastd/#stat-level-tags From b4ce2913fabfdabddca20d1270261bb8d2c76be0 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Tue, 16 Jun 2020 16:35:55 -0400 Subject: [PATCH 3/6] add signalfx tag parsing to main benchmark tests Signed-off-by: glightfoot --- bridge_test.go | 20 ++++++++++++++++++++ exporter_benchmark_test.go | 13 ++++++++++++- line_benchmark_test.go | 3 +-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/bridge_test.go b/bridge_test.go index f008d1d..3def37a 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -179,6 +179,26 @@ func TestHandlePacket(t *testing.T) { CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, + }, { + name: "SignalFx tag extension, tags at end of name", + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, { + name: "SignalFx tag extension, tags at beginning of name", + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, }, { name: "influxdb tag extension with tag keys unsupported by prometheus", in: "foo,09digits=0,tag.with.dots=1:100|c", diff --git a/exporter_benchmark_test.go b/exporter_benchmark_test.go index 900941b..8b0fcc8 100644 --- a/exporter_benchmark_test.go +++ b/exporter_benchmark_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/go-kit/kit/log" + "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" "github.com/prometheus/statsd_exporter/pkg/listener" @@ -34,10 +35,12 @@ func benchmarkUDPListener(times int, b *testing.B) { "foo6:100|c|#09digits:0,tag.with.dots:1", "foo10:100|c|@0.1|#tag1:bar,#tag2:baz", "foo11:100|c|@0.1|#tag1:foo:bar", + "foo.[foo=bar,dim=val]test:1|g", "foo15:200|ms:300|ms:5|c|@0.1:6|g\nfoo15a:1|c:5|ms", "some_very_useful_metrics_with_quite_a_log_name:13|c", } bytesInput := make([]string, len(input)*times) + logger := log.NewNopLogger() for run := 0; run < times; run++ { for i := 0; i < len(input); i++ { bytesInput[run*len(input)+i] = fmt.Sprintf("run%d%s", run, input[i]) @@ -46,7 +49,15 @@ func benchmarkUDPListener(times int, b *testing.B) { for n := 0; n < b.N; n++ { // there are more events than input lines, need bigger buffer events := make(chan event.Events, len(bytesInput)*times*2) - l := listener.StatsDUDPListener{EventHandler: &event.UnbufferedEventHandler{C: events}} + + l := listener.StatsDUDPListener{ + EventHandler: &event.UnbufferedEventHandler{C: events}, + Logger: logger, + UDPPackets: udpPackets, + LinesReceived: linesReceived, + SamplesReceived: samplesReceived, + TagsReceived: tagsReceived, + } for i := 0; i < times; i++ { for _, line := range bytesInput { diff --git a/line_benchmark_test.go b/line_benchmark_test.go index 68bbdec..26b3508 100644 --- a/line_benchmark_test.go +++ b/line_benchmark_test.go @@ -47,7 +47,6 @@ func benchmarkLineToEvents(times int, b *testing.B) { } } - func BenchmarkLineToEvents1(b *testing.B) { benchmarkLineToEvents(1, b) } @@ -56,4 +55,4 @@ func BenchmarkLineToEvents5(b *testing.B) { } func BenchmarkLineToEvents50(b *testing.B) { benchmarkLineToEvents(50, b) -} \ No newline at end of file +} From aa591e3e8e1594446728c0ea0f38079e4ea931e3 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Wed, 17 Jun 2020 13:14:39 -0400 Subject: [PATCH 4/6] add test for signalfx/dogstatsd mixed tags Signed-off-by: glightfoot --- bridge_test.go | 5 +++++ line_benchmark_test.go | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bridge_test.go b/bridge_test.go index 3def37a..1ce8120 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -23,6 +23,7 @@ import ( "github.com/go-kit/kit/log" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" + "github.com/prometheus/statsd_exporter/pkg/clock" "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" @@ -283,6 +284,10 @@ func TestHandlePacket(t *testing.T) { name: "librato/dogstatsd mixed tag styles without sampling", in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", out: event.Events{}, + }, { + name: "signalfx/dogstatsd mixed tag styles without sampling", + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, }, { name: "influxdb/dogstatsd mixed tag styles without sampling", in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", diff --git a/line_benchmark_test.go b/line_benchmark_test.go index 26b3508..1c4dbae 100644 --- a/line_benchmark_test.go +++ b/line_benchmark_test.go @@ -38,7 +38,6 @@ func benchmarkLineToEvents(times int, b *testing.B) { logger := log.NewNopLogger() for n := 0; n < b.N; n++ { - for i := 0; i < times; i++ { for _, l := range input { line.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, logger) From 5b863848ddc48b413c8b1d44c2d43f5d87940ded Mon Sep 17 00:00:00 2001 From: glightfoot Date: Fri, 19 Jun 2020 09:10:19 -0400 Subject: [PATCH 5/6] adding more tests for malformed signalfx tags Signed-off-by: glightfoot --- bridge_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bridge_test.go b/bridge_test.go index 6b4dff6..cb4b7af 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -200,6 +200,26 @@ func TestHandlePacket(t *testing.T) { CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, + }, { + name: "SignalFx tag extension, no tags", + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, { + name: "SignalFx tag extension, non-kv tags", + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, }, { name: "influxdb tag extension with tag keys unsupported by prometheus", in: "foo,09digits=0,tag.with.dots=1:100|c", From 9faa4898dea3a1651cbeb4a712f2a37e4aabecc6 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Fri, 19 Jun 2020 14:39:47 -0400 Subject: [PATCH 6/6] abort parsing tags if malformed signalfx format, add tests Signed-off-by: glightfoot --- bridge_test.go | 20 ++++++++++++++++++++ pkg/line/line.go | 9 ++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/bridge_test.go b/bridge_test.go index cb4b7af..aae7bf2 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -220,6 +220,26 @@ func TestHandlePacket(t *testing.T) { CLabels: map[string]string{}, }, }, + }, { + name: "SignalFx tag extension, missing closing bracket", + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, { + name: "SignalFx tag extension, missing opening bracket", + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, }, { name: "influxdb tag extension with tag keys unsupported by prometheus", in: "foo,09digits=0,tag.with.dots=1:100|c", diff --git a/pkg/line/line.go b/pkg/line/line.go index ddea1da..de299be 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -139,9 +139,16 @@ func parseNameAndTags(name string, labels map[string]string, tagErrors prometheu startIdx := strings.IndexRune(name, '[') endIdx := strings.IndexRune(name, ']') - if startIdx != -1 && endIdx != -1 { + switch { + case startIdx != -1 && endIdx != -1: + // good signalfx tags parseNameTags(name[startIdx+1:endIdx], labels, tagErrors, logger) return name[:startIdx] + name[endIdx+1:] + case (startIdx != -1) != (endIdx != -1): + // only one bracket, return unparsed + level.Debug(logger).Log("msg", "invalid SignalFx tags, not parsing", "metric", name) + tagErrors.Inc() + return name } for i, c := range name {