From 3f3ab233590cc0eee8b7130f56728aaecc2ca583 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Tue, 28 Jul 2020 11:43:02 -0400 Subject: [PATCH 1/5] add globals for disabling individual tag parsing formats Signed-off-by: glightfoot --- main.go | 11 + pkg/line/line.go | 68 ++-- pkg/line/line_test.go | 749 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 800 insertions(+), 28 deletions(-) create mode 100644 pkg/line/line_test.go diff --git a/main.go b/main.go index 5ce9ff3..ec3bc9c 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( "github.com/prometheus/statsd_exporter/pkg/address" "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" + "github.com/prometheus/statsd_exporter/pkg/line" "github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/mapper" ) @@ -268,6 +269,10 @@ func main() { eventFlushInterval = kingpin.Flag("statsd.event-flush-interval", "Number of events to hold in queue before flushing").Default("200ms").Duration() dumpFSMPath = kingpin.Flag("debug.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String() checkConfig = kingpin.Flag("check-config", "Check configuration and exit.").Default("false").Bool() + dogstatsdTagsEnabled = kingpin.Flag("statsd.dogstatsd-tags-enabled", "Parse DogStatsd style tags").Default("true").Bool() + influxdbTagsEnabled = kingpin.Flag("statsd.influxdb-tags-enabled", "Parse InfluxDB style tags").Default("true").Bool() + libratoTagsEnabled = kingpin.Flag("statsd.librato-tags-enabled", "Parse Librato style tags").Default("true").Bool() + signalFXTagsEnabled = kingpin.Flag("statsd.signalfx-tags-enabled", "Parse SignalFX style tags").Default("true").Bool() ) promlogConfig := &promlog.Config{} @@ -277,6 +282,12 @@ func main() { kingpin.Parse() logger := promlog.New(promlogConfig) + // Set line parsing options + line.DogstatsdTagsEnabled = *dogstatsdTagsEnabled + line.InfluxdbTagsEnabled = *influxdbTagsEnabled + line.LibratoTagsEnabled = *libratoTagsEnabled + line.SignalFXTagsEnabled = *signalFXTagsEnabled + cacheOption := mapper.WithCacheType(*cacheType) if *statsdListenUDP == "" && *statsdListenTCP == "" && *statsdListenUnixgram == "" { diff --git a/pkg/line/line.go b/pkg/line/line.go index de299be..50f93fd 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -27,6 +27,14 @@ import ( "github.com/prometheus/statsd_exporter/pkg/mapper" ) +// These globals can be used to control parsing behavior +var ( + DogstatsdTagsEnabled = true + InfluxdbTagsEnabled = true + LibratoTagsEnabled = true + SignalFXTagsEnabled = true +) + func buildEvent(statType, metric string, value float64, relative bool, labels map[string]string) (event.Event, error) { switch statType { case "c": @@ -115,40 +123,44 @@ func trimLeftHash(s string) string { } func ParseDogStatsDTags(component string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) { - lastTagEndIndex := 0 - for i, c := range component { - if c == ',' { - tag := component[lastTagEndIndex:i] - lastTagEndIndex = i + 1 + if DogstatsdTagsEnabled { + lastTagEndIndex := 0 + for i, c := range component { + if c == ',' { + tag := component[lastTagEndIndex:i] + lastTagEndIndex = i + 1 + parseTag(component, trimLeftHash(tag), ':', labels, tagErrors, logger) + } + } + + // If we're not off the end of the string, add the last tag + if lastTagEndIndex < len(component) { + tag := component[lastTagEndIndex:] parseTag(component, trimLeftHash(tag), ':', labels, tagErrors, logger) } } - - // If we're not off the end of the string, add the last tag - if lastTagEndIndex < len(component) { - tag := component[lastTagEndIndex:] - parseTag(component, trimLeftHash(tag), ':', labels, tagErrors, logger) - } } func parseNameAndTags(name string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) string { - // check for SignalFx tags first - // `[` delimits start of tags by SignalFx - // `]` delimits end of tags by SignalFx - // https://docs.signalfx.com/en/latest/integrations/agent/monitors/collectd-statsd.html - startIdx := strings.IndexRune(name, '[') - endIdx := strings.IndexRune(name, ']') + if SignalFXTagsEnabled { + // check for SignalFx tags first + // `[` delimits start of tags by SignalFx + // `]` delimits end of tags by SignalFx + // https://docs.signalfx.com/en/latest/integrations/agent/monitors/collectd-statsd.html + startIdx := strings.IndexRune(name, '[') + endIdx := strings.IndexRune(name, ']') - switch { - case startIdx != -1 && endIdx != -1: - // good signalfx tags - parseNameTags(name[startIdx+1:endIdx], labels, tagErrors, logger) - return name[:startIdx] + name[endIdx+1:] - case (startIdx != -1) != (endIdx != -1): - // only one bracket, return unparsed - level.Debug(logger).Log("msg", "invalid SignalFx tags, not parsing", "metric", name) - tagErrors.Inc() - return name + switch { + case startIdx != -1 && endIdx != -1: + // good signalfx tags + parseNameTags(name[startIdx+1:endIdx], labels, tagErrors, logger) + return name[:startIdx] + name[endIdx+1:] + case (startIdx != -1) != (endIdx != -1): + // only one bracket, return unparsed + level.Debug(logger).Log("msg", "invalid SignalFx tags, not parsing", "metric", name) + tagErrors.Inc() + return name + } } for i, c := range name { @@ -156,7 +168,7 @@ func parseNameAndTags(name string, labels map[string]string, tagErrors prometheu // https://www.librato.com/docs/kb/collect/collection_agents/stastd/#stat-level-tags // `,` delimits start of tags by InfluxDB // https://www.influxdata.com/blog/getting-started-with-sending-statsd-metrics-to-telegraf-influxdb/#introducing-influx-statsd - if c == '#' || c == ',' { + if (c == '#' && LibratoTagsEnabled) || (c == ',' && InfluxdbTagsEnabled) { parseNameTags(name[i+1:], labels, tagErrors, logger) return name[:i] } diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go new file mode 100644 index 0000000..0bb0c42 --- /dev/null +++ b/pkg/line/line_test.go @@ -0,0 +1,749 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package line + +import ( + "reflect" + "testing" + + "github.com/go-kit/kit/log" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/statsd_exporter/pkg/event" +) + +var ( + nopSamplesReceived = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "statsd_exporter_samples_total", + Help: "The total number of StatsD samples received.", + }, + ) + nopSampleErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "statsd_exporter_sample_errors_total", + Help: "The total number of errors parsing StatsD samples.", + }, + []string{"reason"}, + ) + nopTagsReceived = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "statsd_exporter_tags_total", + Help: "The total number of DogStatsD tags processed.", + }, + ) + nopTagErrors = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "statsd_exporter_tag_errors_total", + Help: "The number of errors parsing DogStatsD tags.", + }, + ) + nopLogger = log.NewNopLogger() +) + +func TestLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "empty": {}, + "simple counter": { + in: "foo:2|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 2, + CLabels: map[string]string{}, + }, + }, + }, + "simple gauge": { + in: "foo:3|g", + out: event.Events{ + &event.GaugeEvent{ + GMetricName: "foo", + GValue: 3, + GLabels: map[string]string{}, + }, + }, + }, + "gauge with sampling": { + in: "foo:3|g|@0.2", + out: event.Events{ + &event.GaugeEvent{ + GMetricName: "foo", + GValue: 3, + GLabels: map[string]string{}, + }, + }, + }, + "gauge decrement": { + in: "foo:-10|g", + out: event.Events{ + &event.GaugeEvent{ + GMetricName: "foo", + GValue: -10, + GRelative: true, + GLabels: map[string]string{}, + }, + }, + }, + "simple timer": { + in: "foo:200|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.2, + OLabels: map[string]string{}, + }, + }, + }, + "simple histogram": { + in: "foo:200|h", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + "simple distribution": { + in: "foo:200|d", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + "distribution with sampling": { + in: "foo:0.01|d|@0.2|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "histogram with sampling": { + in: "foo:0.01|h|@0.2|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + "timings with sampling factor": { + in: "foo.timing:0.5|ms|@0.1", + out: event.Events{ + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + }, + }, + "bad line": { + in: "foo", + }, + "bad component": { + in: "foo:1", + }, + "bad value": { + in: "foo:1o|c", + }, + "illegal sampling factor": { + in: "foo:1|c|@bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1, + CLabels: map[string]string{}, + }, + }, + }, + "zero sampling factor": { + in: "foo:2|c|@0", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 2, + CLabels: map[string]string{}, + }, + }, + }, + "illegal stat type": { + in: "foo:2|t", + }, + "empty metric name": { + in: ":100|ms", + }, + "empty component": { + in: "foo:1|c|", + }, + "invalid utf8": { + in: "invalid\xc3\x28utf8:1|c", + }, + "ms timer with conversion to seconds": { + in: "foo:200|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.2, + OLabels: map[string]string{}, + }, + }, + }, + "histogram with no unit conversion": { + in: "foo:200|h", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + "distribution with no unit conversion": { + in: "foo:200|d", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + } + + DogstatsdTagsEnabled = true + InfluxdbTagsEnabled = true + SignalFXTagsEnabled = true + LibratoTagsEnabled = true + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#tag1=bar,tag2=baz", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#09digits=0,tag.with.dots=1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,tag1=bar,tag2=baz", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1=bar,tag2=baz]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test[tag1=bar,tag2=baz]", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1,tag2]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,09digits=0,tag.with.dots=1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + DogstatsdTagsEnabled = false + InfluxdbTagsEnabled = false + SignalFXTagsEnabled = false + LibratoTagsEnabled = false + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} From db25b1d65824281f90a7d531704b87d5cec11b58 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Tue, 28 Jul 2020 13:29:40 -0400 Subject: [PATCH 2/5] add line tests for disabling individual tag formats Signed-off-by: glightfoot --- pkg/line/line_test.go | 956 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 956 insertions(+) diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index 0bb0c42..2d444cc 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -747,3 +747,959 @@ func TestDisableParsingLineToEvents(t *testing.T) { }) } } + +func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + DogstatsdTagsEnabled = false + InfluxdbTagsEnabled = true + SignalFXTagsEnabled = true + LibratoTagsEnabled = true + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingInfluxdbLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,tag1=bar,tag2=baz", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,09digits=0,tag.with.dots=1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + DogstatsdTagsEnabled = true + InfluxdbTagsEnabled = false + SignalFXTagsEnabled = true + LibratoTagsEnabled = true + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingSignalfxLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { // parsed as influxdb tags + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]test"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { // parsed as influxdb tags + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { // parsed as influxdb tags + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]foo.test"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { // parsed as influxdb tags + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { // parsed as influxdb tags + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "bazfoo.test"}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { // parsed as influxdb tags + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]foo.test"}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + DogstatsdTagsEnabled = true + InfluxdbTagsEnabled = true + SignalFXTagsEnabled = false + LibratoTagsEnabled = true + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingLibratoLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { // parsed as influxdb tags + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { // parsed as influxdb tags + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#09digits=0", + CValue: 100, + CLabels: map[string]string{"tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + DogstatsdTagsEnabled = true + InfluxdbTagsEnabled = true + SignalFXTagsEnabled = true + LibratoTagsEnabled = false + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} From 6942b5a4f38c5ba5b72daae841a7921e76b79862 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 10 Aug 2020 12:44:23 -0400 Subject: [PATCH 3/5] move to line parser struct and option functions Signed-off-by: glightfoot --- bridge_test.go | 9 ++++++ exporter_benchmark_test.go | 8 +++++ line_benchmark_test.go | 19 ++++++++++-- main.go | 21 ++++++++++--- pkg/exporter/exporter_test.go | 15 ++++++++- pkg/line/line.go | 56 ++++++++++++++++++++++++--------- pkg/line/line_test.go | 58 +++++++++++++++++------------------ pkg/listener/listener.go | 9 ++++-- 8 files changed, 139 insertions(+), 56 deletions(-) diff --git a/bridge_test.go b/bridge_test.go index aae7bf2..f4e2476 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -27,6 +27,7 @@ import ( "github.com/prometheus/statsd_exporter/pkg/clock" "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" + "github.com/prometheus/statsd_exporter/pkg/line" "github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/mapper" ) @@ -530,10 +531,17 @@ func TestHandlePacket(t *testing.T) { }, } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + for k, l := range []statsDPacketHandler{&listener.StatsDUDPListener{ Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, @@ -545,6 +553,7 @@ func TestHandlePacket(t *testing.T) { Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, SampleErrors: *sampleErrors, diff --git a/exporter_benchmark_test.go b/exporter_benchmark_test.go index 6d58569..5b27fe0 100644 --- a/exporter_benchmark_test.go +++ b/exporter_benchmark_test.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" + "github.com/prometheus/statsd_exporter/pkg/line" "github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/mapper" ) @@ -47,6 +48,12 @@ func benchmarkUDPListener(times int, b *testing.B) { } } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + // reset benchmark timer to not measure startup costs b.ResetTimer() @@ -60,6 +67,7 @@ func benchmarkUDPListener(times int, b *testing.B) { l := listener.StatsDUDPListener{ EventHandler: &event.UnbufferedEventHandler{C: events}, Logger: logger, + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, SamplesReceived: samplesReceived, diff --git a/line_benchmark_test.go b/line_benchmark_test.go index 28166fa..a8bf07d 100644 --- a/line_benchmark_test.go +++ b/line_benchmark_test.go @@ -43,10 +43,19 @@ func benchmarkLinesToEvents(times int, b *testing.B, input []string) { // always report allocations since this is a hot path b.ReportAllocs() + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + + // reset benchmark timer to not measure startup costs + b.ResetTimer() + for n := 0; n < b.N; n++ { for i := 0; i < times; i++ { for _, l := range input { - line.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) + parser.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) } } } @@ -75,6 +84,12 @@ func BenchmarkLineFormats(b *testing.B) { "invalidInfluxDb": "foo3,tag1=bar,tag2:100|c", } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + // reset benchmark timer to not measure startup costs b.ResetTimer() @@ -83,7 +98,7 @@ func BenchmarkLineFormats(b *testing.B) { // always report allocations since this is a hot path b.ReportAllocs() for n := 0; n < b.N; n++ { - line.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) + parser.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) } }) } diff --git a/main.go b/main.go index ec3bc9c..6d8e09d 100644 --- a/main.go +++ b/main.go @@ -282,11 +282,19 @@ func main() { kingpin.Parse() logger := promlog.New(promlogConfig) - // Set line parsing options - line.DogstatsdTagsEnabled = *dogstatsdTagsEnabled - line.InfluxdbTagsEnabled = *influxdbTagsEnabled - line.LibratoTagsEnabled = *libratoTagsEnabled - line.SignalFXTagsEnabled = *signalFXTagsEnabled + parser := line.NewParser() + if *dogstatsdTagsEnabled { + parser.EnableDogstatsdParsing() + } + if *influxdbTagsEnabled { + parser.EnableInfluxdbParsing() + } + if *libratoTagsEnabled { + parser.EnableLibratoParsing() + } + if *signalFXTagsEnabled { + parser.EnableSignalFXParsing() + } cacheOption := mapper.WithCacheType(*cacheType) @@ -330,6 +338,7 @@ func main() { Conn: uconn, EventHandler: eventQueue, Logger: logger, + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, @@ -359,6 +368,7 @@ func main() { Conn: tconn, EventHandler: eventQueue, Logger: logger, + LineParser: parser, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, SampleErrors: *sampleErrors, @@ -402,6 +412,7 @@ func main() { Conn: uxgconn, EventHandler: eventQueue, Logger: logger, + LineParser: parser, UnixgramPackets: unixgramPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, diff --git a/pkg/exporter/exporter_test.go b/pkg/exporter/exporter_test.go index c92221b..644c5d7 100644 --- a/pkg/exporter/exporter_test.go +++ b/pkg/exporter/exporter_test.go @@ -617,11 +617,18 @@ func TestInvalidUtf8InDatadogTagValue(t *testing.T) { events := make(chan event.Events) ueh := &event.UnbufferedEventHandler{C: events} + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + go func() { for _, l := range []statsDPacketHandler{&listener.StatsDUDPListener{ Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, @@ -633,6 +640,7 @@ func TestInvalidUtf8InDatadogTagValue(t *testing.T) { Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, SampleErrors: *sampleErrors, @@ -1059,11 +1067,16 @@ func BenchmarkParseDogStatsDTags(b *testing.B) { "a-z tags": "a:0,b:1,c:2,d:3,e:4,f:5,g:6,h:7,i:8,j:9,k:0,l:1,m:2,n:3,o:4,p:5,q:6,r:7,s:8,t:9,u:0,v:1,w:2,x:3,y:4,z:5", } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + + b.ResetTimer() + for name, tags := range scenarios { b.Run(name, func(b *testing.B) { for n := 0; n < b.N; n++ { labels := map[string]string{} - line.ParseDogStatsDTags(tags, labels, tagErrors, log.NewNopLogger()) + parser.ParseDogStatsDTags(tags, labels, tagErrors, log.NewNopLogger()) } }) } diff --git a/pkg/line/line.go b/pkg/line/line.go index 50f93fd..baebe84 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -27,13 +27,39 @@ import ( "github.com/prometheus/statsd_exporter/pkg/mapper" ) -// These globals can be used to control parsing behavior -var ( - DogstatsdTagsEnabled = true - InfluxdbTagsEnabled = true - LibratoTagsEnabled = true - SignalFXTagsEnabled = true -) +// Parser is a struct to hold configuration for parsing behavior +type Parser struct { + DogstatsdTagsEnabled bool + InfluxdbTagsEnabled bool + LibratoTagsEnabled bool + SignalFXTagsEnabled bool +} + +// NewParser returns a new line parser +func NewParser() *Parser { + p := Parser{} + return &p +} + +// EnableDogstatsdParsing option to enable dogstatsd tag parsing +func (p *Parser) EnableDogstatsdParsing() { + p.DogstatsdTagsEnabled = true +} + +// EnableInfluxdbParsing option to enable influxdb tag parsing +func (p *Parser) EnableInfluxdbParsing() { + p.InfluxdbTagsEnabled = true +} + +// EnableLibratoParsing option to enable librato tag parsing +func (p *Parser) EnableLibratoParsing() { + p.LibratoTagsEnabled = true +} + +// EnableSignalFXParsing option to enable signalfx tag parsing +func (p *Parser) EnableSignalFXParsing() { + p.SignalFXTagsEnabled = true +} func buildEvent(statType, metric string, value float64, relative bool, labels map[string]string) (event.Event, error) { switch statType { @@ -122,8 +148,8 @@ func trimLeftHash(s string) string { return s } -func ParseDogStatsDTags(component string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) { - if DogstatsdTagsEnabled { +func (p *Parser) ParseDogStatsDTags(component string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) { + if p.DogstatsdTagsEnabled { lastTagEndIndex := 0 for i, c := range component { if c == ',' { @@ -141,8 +167,8 @@ func ParseDogStatsDTags(component string, labels map[string]string, tagErrors pr } } -func parseNameAndTags(name string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) string { - if SignalFXTagsEnabled { +func (p *Parser) parseNameAndTags(name string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) string { + if p.SignalFXTagsEnabled { // check for SignalFx tags first // `[` delimits start of tags by SignalFx // `]` delimits end of tags by SignalFx @@ -168,7 +194,7 @@ func parseNameAndTags(name string, labels map[string]string, tagErrors prometheu // https://www.librato.com/docs/kb/collect/collection_agents/stastd/#stat-level-tags // `,` delimits start of tags by InfluxDB // https://www.influxdata.com/blog/getting-started-with-sending-statsd-metrics-to-telegraf-influxdb/#introducing-influx-statsd - if (c == '#' && LibratoTagsEnabled) || (c == ',' && InfluxdbTagsEnabled) { + if (c == '#' && p.LibratoTagsEnabled) || (c == ',' && p.InfluxdbTagsEnabled) { parseNameTags(name[i+1:], labels, tagErrors, logger) return name[:i] } @@ -176,7 +202,7 @@ func parseNameAndTags(name string, labels map[string]string, tagErrors prometheu return name } -func LineToEvents(line string, sampleErrors prometheus.CounterVec, samplesReceived prometheus.Counter, tagErrors prometheus.Counter, tagsReceived prometheus.Counter, logger log.Logger) event.Events { +func (p *Parser) LineToEvents(line string, sampleErrors prometheus.CounterVec, samplesReceived prometheus.Counter, tagErrors prometheus.Counter, tagsReceived prometheus.Counter, logger log.Logger) event.Events { events := event.Events{} if line == "" { return events @@ -190,7 +216,7 @@ func LineToEvents(line string, sampleErrors prometheus.CounterVec, samplesReceiv } labels := map[string]string{} - metric := parseNameAndTags(elements[0], labels, tagErrors, logger) + metric := p.parseNameAndTags(elements[0], labels, tagErrors, logger) var samples []string if strings.Contains(elements[1], "|#") { @@ -264,7 +290,7 @@ samples: multiplyEvents = int(1 / samplingFactor) } case '#': - ParseDogStatsDTags(component[1:], labels, tagErrors, logger) + p.ParseDogStatsDTags(component[1:], labels, tagErrors, logger) default: level.Debug(logger).Log("msg", "Invalid sampling factor or tag section", "component", components[2], "line", line) sampleErrors.WithLabelValues("invalid_sample_factor").Inc() diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go index 2d444cc..b4b67aa 100644 --- a/pkg/line/line_test.go +++ b/pkg/line/line_test.go @@ -491,14 +491,15 @@ func TestLineToEvents(t *testing.T) { }, } - DogstatsdTagsEnabled = true - InfluxdbTagsEnabled = true - SignalFXTagsEnabled = true - LibratoTagsEnabled = true + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { - events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) for j, expected := range testCase.out { if !reflect.DeepEqual(&expected, &events[j]) { @@ -730,14 +731,11 @@ func TestDisableParsingLineToEvents(t *testing.T) { }, } - DogstatsdTagsEnabled = false - InfluxdbTagsEnabled = false - SignalFXTagsEnabled = false - LibratoTagsEnabled = false + parser := NewParser() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { - events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) for j, expected := range testCase.out { if !reflect.DeepEqual(&expected, &events[j]) { @@ -969,14 +967,14 @@ func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { }, } - DogstatsdTagsEnabled = false - InfluxdbTagsEnabled = true - SignalFXTagsEnabled = true - LibratoTagsEnabled = true + parser := NewParser() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { - events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) for j, expected := range testCase.out { if !reflect.DeepEqual(&expected, &events[j]) { @@ -1208,14 +1206,14 @@ func TestDisableParsingInfluxdbLineToEvents(t *testing.T) { }, } - DogstatsdTagsEnabled = true - InfluxdbTagsEnabled = false - SignalFXTagsEnabled = true - LibratoTagsEnabled = true + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { - events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) for j, expected := range testCase.out { if !reflect.DeepEqual(&expected, &events[j]) { @@ -1447,14 +1445,14 @@ func TestDisableParsingSignalfxLineToEvents(t *testing.T) { }, } - DogstatsdTagsEnabled = true - InfluxdbTagsEnabled = true - SignalFXTagsEnabled = false - LibratoTagsEnabled = true + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { - events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) for j, expected := range testCase.out { if !reflect.DeepEqual(&expected, &events[j]) { @@ -1686,14 +1684,14 @@ func TestDisableParsingLibratoLineToEvents(t *testing.T) { }, } - DogstatsdTagsEnabled = true - InfluxdbTagsEnabled = true - SignalFXTagsEnabled = true - LibratoTagsEnabled = false + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableSignalFXParsing() for name, testCase := range testCases { t.Run(name, func(t *testing.T) { - events := LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) for j, expected := range testCase.out { if !reflect.DeepEqual(&expected, &events[j]) { diff --git a/pkg/listener/listener.go b/pkg/listener/listener.go index 5c31f53..0607180 100644 --- a/pkg/listener/listener.go +++ b/pkg/listener/listener.go @@ -31,6 +31,7 @@ type StatsDUDPListener struct { Conn *net.UDPConn EventHandler event.EventHandler Logger log.Logger + LineParser *pkgLine.Parser UDPPackets prometheus.Counter LinesReceived prometheus.Counter EventsFlushed prometheus.Counter @@ -67,7 +68,7 @@ func (l *StatsDUDPListener) HandlePacket(packet []byte) { for _, line := range lines { level.Debug(l.Logger).Log("msg", "Incoming line", "proto", "udp", "line", line) l.LinesReceived.Inc() - l.EventHandler.Queue(pkgLine.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) + l.EventHandler.Queue(l.LineParser.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) } } @@ -75,6 +76,7 @@ type StatsDTCPListener struct { Conn *net.TCPListener EventHandler event.EventHandler Logger log.Logger + LineParser *pkgLine.Parser LinesReceived prometheus.Counter EventsFlushed prometheus.Counter SampleErrors prometheus.CounterVec @@ -128,7 +130,7 @@ func (l *StatsDTCPListener) HandleConn(c *net.TCPConn) { break } l.LinesReceived.Inc() - l.EventHandler.Queue(pkgLine.LineToEvents(string(line), l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) + l.EventHandler.Queue(l.LineParser.LineToEvents(string(line), l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) } } @@ -136,6 +138,7 @@ type StatsDUnixgramListener struct { Conn *net.UnixConn EventHandler event.EventHandler Logger log.Logger + LineParser *pkgLine.Parser UnixgramPackets prometheus.Counter LinesReceived prometheus.Counter EventsFlushed prometheus.Counter @@ -172,6 +175,6 @@ func (l *StatsDUnixgramListener) HandlePacket(packet []byte) { for _, line := range lines { level.Debug(l.Logger).Log("msg", "Incoming line", "proto", "unixgram", "line", line) l.LinesReceived.Inc() - l.EventHandler.Queue(pkgLine.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) + l.EventHandler.Queue(l.LineParser.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) } } From afa40f4adacf430d4f86dec9af533edf49072613 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 10 Aug 2020 12:56:23 -0400 Subject: [PATCH 4/5] rename cli flags for parsing and add docs to README Signed-off-by: glightfoot --- README.md | 8 ++++++++ main.go | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cbe0369..9995c11 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,14 @@ metric.name[tagName=val,tag2Name=val2]:0|c Be aware: If you mix tag styles (e.g., Librato/InfluxDB with DogStatsD), the exporter will consider this an error and the behavior is undefined. Also, tags without values (`#some_tag`) are not supported and will be ignored. +The exporter parses all tagging formats by default, but individual tagging formats can be disabled with command line flags: +``` +--no-statsd.parse-dogstatsd-tags +--no-statsd.parse-influxdb-tags +--no-statsd.parse-librato-tags +--no-statsd.parse-signalfx-tags +``` + ## Building and Running NOTE: Version 0.7.0 switched to the [kingpin](https://github.com/alecthomas/kingpin) flags library. With this change, flag behaviour is POSIX-ish: diff --git a/main.go b/main.go index 6d8e09d..100045f 100644 --- a/main.go +++ b/main.go @@ -269,10 +269,10 @@ func main() { eventFlushInterval = kingpin.Flag("statsd.event-flush-interval", "Number of events to hold in queue before flushing").Default("200ms").Duration() dumpFSMPath = kingpin.Flag("debug.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String() checkConfig = kingpin.Flag("check-config", "Check configuration and exit.").Default("false").Bool() - dogstatsdTagsEnabled = kingpin.Flag("statsd.dogstatsd-tags-enabled", "Parse DogStatsd style tags").Default("true").Bool() - influxdbTagsEnabled = kingpin.Flag("statsd.influxdb-tags-enabled", "Parse InfluxDB style tags").Default("true").Bool() - libratoTagsEnabled = kingpin.Flag("statsd.librato-tags-enabled", "Parse Librato style tags").Default("true").Bool() - signalFXTagsEnabled = kingpin.Flag("statsd.signalfx-tags-enabled", "Parse SignalFX style tags").Default("true").Bool() + dogstatsdTagsEnabled = kingpin.Flag("statsd.parse-dogstatsd-tags", "Parse DogStatsd style tags").Default("true").Bool() + influxdbTagsEnabled = kingpin.Flag("statsd.parse-influxdb-tags", "Parse InfluxDB style tags").Default("true").Bool() + libratoTagsEnabled = kingpin.Flag("statsd.parse-librato-tags", "Parse Librato style tags").Default("true").Bool() + signalFXTagsEnabled = kingpin.Flag("statsd.parse-signalfx-tags", "Parse SignalFX style tags").Default("true").Bool() ) promlogConfig := &promlog.Config{} From c162b349b7dde1ae54a15b807cb2dec32a373238 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 10 Aug 2020 14:54:59 -0400 Subject: [PATCH 5/5] add parsing flags to README Signed-off-by: glightfoot --- README.md | 8 ++++++++ main.go | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9995c11..1d6a4f2 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,14 @@ NOTE: Version 0.7.0 switched to the [kingpin](https://github.com/alecthomas/king --debug.dump-fsm="" The path to dump internal FSM generated for glob matching as Dot file. --check-config Check configuration and exit. + --statsd.parse-dogstatsd-tags + Parse DogStatsd style tags. Enabled by default. + --statsd.parse-influxdb-tags + Parse InfluxDB style tags. Enabled by default. + --statsd.parse-librato-tags + Parse Librato style tags. Enabled by default. + --statsd.parse-signalfx-tags + Parse SignalFX style tags. Enabled by default. --log.level=info Only log messages with the given severity or above. One of: [debug, info, warn, error] --log.format=logfmt Output format of log messages. One of: [logfmt, diff --git a/main.go b/main.go index 100045f..7ea1ac0 100644 --- a/main.go +++ b/main.go @@ -269,10 +269,10 @@ func main() { eventFlushInterval = kingpin.Flag("statsd.event-flush-interval", "Number of events to hold in queue before flushing").Default("200ms").Duration() dumpFSMPath = kingpin.Flag("debug.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String() checkConfig = kingpin.Flag("check-config", "Check configuration and exit.").Default("false").Bool() - dogstatsdTagsEnabled = kingpin.Flag("statsd.parse-dogstatsd-tags", "Parse DogStatsd style tags").Default("true").Bool() - influxdbTagsEnabled = kingpin.Flag("statsd.parse-influxdb-tags", "Parse InfluxDB style tags").Default("true").Bool() - libratoTagsEnabled = kingpin.Flag("statsd.parse-librato-tags", "Parse Librato style tags").Default("true").Bool() - signalFXTagsEnabled = kingpin.Flag("statsd.parse-signalfx-tags", "Parse SignalFX style tags").Default("true").Bool() + dogstatsdTagsEnabled = kingpin.Flag("statsd.parse-dogstatsd-tags", "Parse DogStatsd style tags. Enabled by default.").Default("true").Bool() + influxdbTagsEnabled = kingpin.Flag("statsd.parse-influxdb-tags", "Parse InfluxDB style tags. Enabled by default.").Default("true").Bool() + libratoTagsEnabled = kingpin.Flag("statsd.parse-librato-tags", "Parse Librato style tags. Enabled by default.").Default("true").Bool() + signalFXTagsEnabled = kingpin.Flag("statsd.parse-signalfx-tags", "Parse SignalFX style tags. Enabled by default.").Default("true").Bool() ) promlogConfig := &promlog.Config{}