diff --git a/README.md b/README.md index cbe0369..1d6a4f2 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,14 @@ metric.name[tagName=val,tag2Name=val2]:0|c Be aware: If you mix tag styles (e.g., Librato/InfluxDB with DogStatsD), the exporter will consider this an error and the behavior is undefined. Also, tags without values (`#some_tag`) are not supported and will be ignored. +The exporter parses all tagging formats by default, but individual tagging formats can be disabled with command line flags: +``` +--no-statsd.parse-dogstatsd-tags +--no-statsd.parse-influxdb-tags +--no-statsd.parse-librato-tags +--no-statsd.parse-signalfx-tags +``` + ## Building and Running NOTE: Version 0.7.0 switched to the [kingpin](https://github.com/alecthomas/kingpin) flags library. With this change, flag behaviour is POSIX-ish: @@ -131,6 +139,14 @@ NOTE: Version 0.7.0 switched to the [kingpin](https://github.com/alecthomas/king --debug.dump-fsm="" The path to dump internal FSM generated for glob matching as Dot file. --check-config Check configuration and exit. + --statsd.parse-dogstatsd-tags + Parse DogStatsd style tags. Enabled by default. + --statsd.parse-influxdb-tags + Parse InfluxDB style tags. Enabled by default. + --statsd.parse-librato-tags + Parse Librato style tags. Enabled by default. + --statsd.parse-signalfx-tags + Parse SignalFX style tags. Enabled by default. --log.level=info Only log messages with the given severity or above. One of: [debug, info, warn, error] --log.format=logfmt Output format of log messages. One of: [logfmt, diff --git a/bridge_test.go b/bridge_test.go index aae7bf2..f4e2476 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -27,6 +27,7 @@ import ( "github.com/prometheus/statsd_exporter/pkg/clock" "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" + "github.com/prometheus/statsd_exporter/pkg/line" "github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/mapper" ) @@ -530,10 +531,17 @@ func TestHandlePacket(t *testing.T) { }, } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + for k, l := range []statsDPacketHandler{&listener.StatsDUDPListener{ Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, @@ -545,6 +553,7 @@ func TestHandlePacket(t *testing.T) { Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, SampleErrors: *sampleErrors, diff --git a/exporter_benchmark_test.go b/exporter_benchmark_test.go index 6d58569..5b27fe0 100644 --- a/exporter_benchmark_test.go +++ b/exporter_benchmark_test.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" + "github.com/prometheus/statsd_exporter/pkg/line" "github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/mapper" ) @@ -47,6 +48,12 @@ func benchmarkUDPListener(times int, b *testing.B) { } } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + // reset benchmark timer to not measure startup costs b.ResetTimer() @@ -60,6 +67,7 @@ func benchmarkUDPListener(times int, b *testing.B) { l := listener.StatsDUDPListener{ EventHandler: &event.UnbufferedEventHandler{C: events}, Logger: logger, + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, SamplesReceived: samplesReceived, diff --git a/line_benchmark_test.go b/line_benchmark_test.go index 28166fa..a8bf07d 100644 --- a/line_benchmark_test.go +++ b/line_benchmark_test.go @@ -43,10 +43,19 @@ func benchmarkLinesToEvents(times int, b *testing.B, input []string) { // always report allocations since this is a hot path b.ReportAllocs() + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + + // reset benchmark timer to not measure startup costs + b.ResetTimer() + for n := 0; n < b.N; n++ { for i := 0; i < times; i++ { for _, l := range input { - line.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) + parser.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) } } } @@ -75,6 +84,12 @@ func BenchmarkLineFormats(b *testing.B) { "invalidInfluxDb": "foo3,tag1=bar,tag2:100|c", } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + // reset benchmark timer to not measure startup costs b.ResetTimer() @@ -83,7 +98,7 @@ func BenchmarkLineFormats(b *testing.B) { // always report allocations since this is a hot path b.ReportAllocs() for n := 0; n < b.N; n++ { - line.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) + parser.LineToEvents(l, *sampleErrors, samplesReceived, tagErrors, tagsReceived, nopLogger) } }) } diff --git a/main.go b/main.go index 5ce9ff3..7ea1ac0 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ import ( "github.com/prometheus/statsd_exporter/pkg/address" "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" + "github.com/prometheus/statsd_exporter/pkg/line" "github.com/prometheus/statsd_exporter/pkg/listener" "github.com/prometheus/statsd_exporter/pkg/mapper" ) @@ -268,6 +269,10 @@ func main() { eventFlushInterval = kingpin.Flag("statsd.event-flush-interval", "Number of events to hold in queue before flushing").Default("200ms").Duration() dumpFSMPath = kingpin.Flag("debug.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String() checkConfig = kingpin.Flag("check-config", "Check configuration and exit.").Default("false").Bool() + dogstatsdTagsEnabled = kingpin.Flag("statsd.parse-dogstatsd-tags", "Parse DogStatsd style tags. Enabled by default.").Default("true").Bool() + influxdbTagsEnabled = kingpin.Flag("statsd.parse-influxdb-tags", "Parse InfluxDB style tags. Enabled by default.").Default("true").Bool() + libratoTagsEnabled = kingpin.Flag("statsd.parse-librato-tags", "Parse Librato style tags. Enabled by default.").Default("true").Bool() + signalFXTagsEnabled = kingpin.Flag("statsd.parse-signalfx-tags", "Parse SignalFX style tags. Enabled by default.").Default("true").Bool() ) promlogConfig := &promlog.Config{} @@ -277,6 +282,20 @@ func main() { kingpin.Parse() logger := promlog.New(promlogConfig) + parser := line.NewParser() + if *dogstatsdTagsEnabled { + parser.EnableDogstatsdParsing() + } + if *influxdbTagsEnabled { + parser.EnableInfluxdbParsing() + } + if *libratoTagsEnabled { + parser.EnableLibratoParsing() + } + if *signalFXTagsEnabled { + parser.EnableSignalFXParsing() + } + cacheOption := mapper.WithCacheType(*cacheType) if *statsdListenUDP == "" && *statsdListenTCP == "" && *statsdListenUnixgram == "" { @@ -319,6 +338,7 @@ func main() { Conn: uconn, EventHandler: eventQueue, Logger: logger, + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, @@ -348,6 +368,7 @@ func main() { Conn: tconn, EventHandler: eventQueue, Logger: logger, + LineParser: parser, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, SampleErrors: *sampleErrors, @@ -391,6 +412,7 @@ func main() { Conn: uxgconn, EventHandler: eventQueue, Logger: logger, + LineParser: parser, UnixgramPackets: unixgramPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, diff --git a/pkg/exporter/exporter_test.go b/pkg/exporter/exporter_test.go index c92221b..644c5d7 100644 --- a/pkg/exporter/exporter_test.go +++ b/pkg/exporter/exporter_test.go @@ -617,11 +617,18 @@ func TestInvalidUtf8InDatadogTagValue(t *testing.T) { events := make(chan event.Events) ueh := &event.UnbufferedEventHandler{C: events} + parser := line.NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + go func() { for _, l := range []statsDPacketHandler{&listener.StatsDUDPListener{ Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, UDPPackets: udpPackets, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, @@ -633,6 +640,7 @@ func TestInvalidUtf8InDatadogTagValue(t *testing.T) { Conn: nil, EventHandler: nil, Logger: log.NewNopLogger(), + LineParser: parser, LinesReceived: linesReceived, EventsFlushed: eventsFlushed, SampleErrors: *sampleErrors, @@ -1059,11 +1067,16 @@ func BenchmarkParseDogStatsDTags(b *testing.B) { "a-z tags": "a:0,b:1,c:2,d:3,e:4,f:5,g:6,h:7,i:8,j:9,k:0,l:1,m:2,n:3,o:4,p:5,q:6,r:7,s:8,t:9,u:0,v:1,w:2,x:3,y:4,z:5", } + parser := line.NewParser() + parser.EnableDogstatsdParsing() + + b.ResetTimer() + for name, tags := range scenarios { b.Run(name, func(b *testing.B) { for n := 0; n < b.N; n++ { labels := map[string]string{} - line.ParseDogStatsDTags(tags, labels, tagErrors, log.NewNopLogger()) + parser.ParseDogStatsDTags(tags, labels, tagErrors, log.NewNopLogger()) } }) } diff --git a/pkg/line/line.go b/pkg/line/line.go index de299be..baebe84 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -27,6 +27,40 @@ import ( "github.com/prometheus/statsd_exporter/pkg/mapper" ) +// Parser is a struct to hold configuration for parsing behavior +type Parser struct { + DogstatsdTagsEnabled bool + InfluxdbTagsEnabled bool + LibratoTagsEnabled bool + SignalFXTagsEnabled bool +} + +// NewParser returns a new line parser +func NewParser() *Parser { + p := Parser{} + return &p +} + +// EnableDogstatsdParsing option to enable dogstatsd tag parsing +func (p *Parser) EnableDogstatsdParsing() { + p.DogstatsdTagsEnabled = true +} + +// EnableInfluxdbParsing option to enable influxdb tag parsing +func (p *Parser) EnableInfluxdbParsing() { + p.InfluxdbTagsEnabled = true +} + +// EnableLibratoParsing option to enable librato tag parsing +func (p *Parser) EnableLibratoParsing() { + p.LibratoTagsEnabled = true +} + +// EnableSignalFXParsing option to enable signalfx tag parsing +func (p *Parser) EnableSignalFXParsing() { + p.SignalFXTagsEnabled = true +} + func buildEvent(statType, metric string, value float64, relative bool, labels map[string]string) (event.Event, error) { switch statType { case "c": @@ -114,41 +148,45 @@ func trimLeftHash(s string) string { return s } -func ParseDogStatsDTags(component string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) { - lastTagEndIndex := 0 - for i, c := range component { - if c == ',' { - tag := component[lastTagEndIndex:i] - lastTagEndIndex = i + 1 +func (p *Parser) ParseDogStatsDTags(component string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) { + if p.DogstatsdTagsEnabled { + lastTagEndIndex := 0 + for i, c := range component { + if c == ',' { + tag := component[lastTagEndIndex:i] + lastTagEndIndex = i + 1 + parseTag(component, trimLeftHash(tag), ':', labels, tagErrors, logger) + } + } + + // If we're not off the end of the string, add the last tag + if lastTagEndIndex < len(component) { + tag := component[lastTagEndIndex:] parseTag(component, trimLeftHash(tag), ':', labels, tagErrors, logger) } } - - // If we're not off the end of the string, add the last tag - if lastTagEndIndex < len(component) { - tag := component[lastTagEndIndex:] - parseTag(component, trimLeftHash(tag), ':', labels, tagErrors, logger) - } } -func parseNameAndTags(name string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) string { - // check for SignalFx tags first - // `[` delimits start of tags by SignalFx - // `]` delimits end of tags by SignalFx - // https://docs.signalfx.com/en/latest/integrations/agent/monitors/collectd-statsd.html - startIdx := strings.IndexRune(name, '[') - endIdx := strings.IndexRune(name, ']') +func (p *Parser) parseNameAndTags(name string, labels map[string]string, tagErrors prometheus.Counter, logger log.Logger) string { + if p.SignalFXTagsEnabled { + // check for SignalFx tags first + // `[` delimits start of tags by SignalFx + // `]` delimits end of tags by SignalFx + // https://docs.signalfx.com/en/latest/integrations/agent/monitors/collectd-statsd.html + startIdx := strings.IndexRune(name, '[') + endIdx := strings.IndexRune(name, ']') - switch { - case startIdx != -1 && endIdx != -1: - // good signalfx tags - parseNameTags(name[startIdx+1:endIdx], labels, tagErrors, logger) - return name[:startIdx] + name[endIdx+1:] - case (startIdx != -1) != (endIdx != -1): - // only one bracket, return unparsed - level.Debug(logger).Log("msg", "invalid SignalFx tags, not parsing", "metric", name) - tagErrors.Inc() - return name + switch { + case startIdx != -1 && endIdx != -1: + // good signalfx tags + parseNameTags(name[startIdx+1:endIdx], labels, tagErrors, logger) + return name[:startIdx] + name[endIdx+1:] + case (startIdx != -1) != (endIdx != -1): + // only one bracket, return unparsed + level.Debug(logger).Log("msg", "invalid SignalFx tags, not parsing", "metric", name) + tagErrors.Inc() + return name + } } for i, c := range name { @@ -156,7 +194,7 @@ func parseNameAndTags(name string, labels map[string]string, tagErrors prometheu // https://www.librato.com/docs/kb/collect/collection_agents/stastd/#stat-level-tags // `,` delimits start of tags by InfluxDB // https://www.influxdata.com/blog/getting-started-with-sending-statsd-metrics-to-telegraf-influxdb/#introducing-influx-statsd - if c == '#' || c == ',' { + if (c == '#' && p.LibratoTagsEnabled) || (c == ',' && p.InfluxdbTagsEnabled) { parseNameTags(name[i+1:], labels, tagErrors, logger) return name[:i] } @@ -164,7 +202,7 @@ func parseNameAndTags(name string, labels map[string]string, tagErrors prometheu return name } -func LineToEvents(line string, sampleErrors prometheus.CounterVec, samplesReceived prometheus.Counter, tagErrors prometheus.Counter, tagsReceived prometheus.Counter, logger log.Logger) event.Events { +func (p *Parser) LineToEvents(line string, sampleErrors prometheus.CounterVec, samplesReceived prometheus.Counter, tagErrors prometheus.Counter, tagsReceived prometheus.Counter, logger log.Logger) event.Events { events := event.Events{} if line == "" { return events @@ -178,7 +216,7 @@ func LineToEvents(line string, sampleErrors prometheus.CounterVec, samplesReceiv } labels := map[string]string{} - metric := parseNameAndTags(elements[0], labels, tagErrors, logger) + metric := p.parseNameAndTags(elements[0], labels, tagErrors, logger) var samples []string if strings.Contains(elements[1], "|#") { @@ -252,7 +290,7 @@ samples: multiplyEvents = int(1 / samplingFactor) } case '#': - ParseDogStatsDTags(component[1:], labels, tagErrors, logger) + p.ParseDogStatsDTags(component[1:], labels, tagErrors, logger) default: level.Debug(logger).Log("msg", "Invalid sampling factor or tag section", "component", components[2], "line", line) sampleErrors.WithLabelValues("invalid_sample_factor").Inc() diff --git a/pkg/line/line_test.go b/pkg/line/line_test.go new file mode 100644 index 0000000..b4b67aa --- /dev/null +++ b/pkg/line/line_test.go @@ -0,0 +1,1703 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package line + +import ( + "reflect" + "testing" + + "github.com/go-kit/kit/log" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/statsd_exporter/pkg/event" +) + +var ( + nopSamplesReceived = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "statsd_exporter_samples_total", + Help: "The total number of StatsD samples received.", + }, + ) + nopSampleErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "statsd_exporter_sample_errors_total", + Help: "The total number of errors parsing StatsD samples.", + }, + []string{"reason"}, + ) + nopTagsReceived = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "statsd_exporter_tags_total", + Help: "The total number of DogStatsD tags processed.", + }, + ) + nopTagErrors = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "statsd_exporter_tag_errors_total", + Help: "The number of errors parsing DogStatsD tags.", + }, + ) + nopLogger = log.NewNopLogger() +) + +func TestLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "empty": {}, + "simple counter": { + in: "foo:2|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 2, + CLabels: map[string]string{}, + }, + }, + }, + "simple gauge": { + in: "foo:3|g", + out: event.Events{ + &event.GaugeEvent{ + GMetricName: "foo", + GValue: 3, + GLabels: map[string]string{}, + }, + }, + }, + "gauge with sampling": { + in: "foo:3|g|@0.2", + out: event.Events{ + &event.GaugeEvent{ + GMetricName: "foo", + GValue: 3, + GLabels: map[string]string{}, + }, + }, + }, + "gauge decrement": { + in: "foo:-10|g", + out: event.Events{ + &event.GaugeEvent{ + GMetricName: "foo", + GValue: -10, + GRelative: true, + GLabels: map[string]string{}, + }, + }, + }, + "simple timer": { + in: "foo:200|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.2, + OLabels: map[string]string{}, + }, + }, + }, + "simple histogram": { + in: "foo:200|h", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + "simple distribution": { + in: "foo:200|d", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + "distribution with sampling": { + in: "foo:0.01|d|@0.2|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "histogram with sampling": { + in: "foo:0.01|h|@0.2|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + "timings with sampling factor": { + in: "foo.timing:0.5|ms|@0.1", + out: event.Events{ + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + }, + }, + "bad line": { + in: "foo", + }, + "bad component": { + in: "foo:1", + }, + "bad value": { + in: "foo:1o|c", + }, + "illegal sampling factor": { + in: "foo:1|c|@bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1, + CLabels: map[string]string{}, + }, + }, + }, + "zero sampling factor": { + in: "foo:2|c|@0", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 2, + CLabels: map[string]string{}, + }, + }, + }, + "illegal stat type": { + in: "foo:2|t", + }, + "empty metric name": { + in: ":100|ms", + }, + "empty component": { + in: "foo:1|c|", + }, + "invalid utf8": { + in: "invalid\xc3\x28utf8:1|c", + }, + "ms timer with conversion to seconds": { + in: "foo:200|ms", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.2, + OLabels: map[string]string{}, + }, + }, + }, + "histogram with no unit conversion": { + in: "foo:200|h", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + "distribution with no unit conversion": { + in: "foo:200|d", + out: event.Events{ + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, + }, + }, + }, + } + + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#tag1=bar,tag2=baz", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#09digits=0,tag.with.dots=1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,tag1=bar,tag2=baz", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1=bar,tag2=baz]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test[tag1=bar,tag2=baz]", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1,tag2]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,09digits=0,tag.with.dots=1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + parser := NewParser() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingDogstatsdLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + parser := NewParser() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingInfluxdbLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,tag1=bar,tag2=baz", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo,09digits=0,tag.with.dots=1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableLibratoParsing() + parser.EnableSignalFXParsing() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingSignalfxLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { // parsed as influxdb tags + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]test"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { // parsed as influxdb tags + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { // parsed as influxdb tags + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]foo.test"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[]test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { // parsed as influxdb tags + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.[tag1", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { // parsed as influxdb tags + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "bazfoo.test"}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { // parsed as influxdb tags + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz]foo.test"}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableLibratoParsing() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} + +func TestDisableParsingLibratoLineToEvents(t *testing.T) { + type testCase struct { + in string + out event.Events + } + + testCases := map[string]testCase{ + "librato tag extension": { // parsed as influxdb tags + in: "foo#tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#tag1=bar", + CValue: 100, + CLabels: map[string]string{"tag2": "baz"}, + }, + }, + }, + "librato tag extension with tag keys unsupported by prometheus": { // parsed as influxdb tags + in: "foo#09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo#09digits=0", + CValue: 100, + CLabels: map[string]string{"tag_with_dots": "1"}, + }, + }, + }, + "influxdb tag extension": { + in: "foo,tag1=bar,tag2=baz:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension": { + in: "foo.[tag1=bar,tag2=baz]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at end of name": { + in: "foo.test[tag1=bar,tag2=baz]:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, tags at beginning of name": { + in: "[tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "SignalFx tag extension, no tags": { + in: "foo.[]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, non-kv tags": { + in: "foo.[tag1,tag2]test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing closing bracket": { + in: "[tag1=bar,tag2=bazfoo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "[tag1=bar,tag2=bazfoo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "SignalFx tag extension, missing opening bracket": { + in: "tag1=bar,tag2=baz]foo.test:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "tag1=bar,tag2=baz]foo.test", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "influxdb tag extension with tag keys unsupported by prometheus": { + in: "foo,09digits=0,tag.with.dots=1:100|c", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension": { + in: "foo:100|c|#tag1:bar,tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with # in all keys (as sent by datadog php client)": { + in: "foo:100|c|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "datadog tag extension with tag keys unsupported by prometheus": { + in: "foo:100|c|#09digits:0,tag.with.dots:1", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"_09digits": "0", "tag_with_dots": "1"}, + }, + }, + }, + "datadog tag extension with valueless tags: ignored": { + in: "foo:100|c|#tag_without_a_value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{}, + }, + }, + }, + "datadog tag extension with valueless tags (edge case)": { + in: "foo:100|c|#tag_without_a_value,tag:value", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with empty tags (edge case)": { + in: "foo:100|c|#tag:value,,", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 100, + CLabels: map[string]string{"tag": "value"}, + }, + }, + }, + "datadog tag extension with sampling": { + in: "foo:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + }, + }, + "librato/dogstatsd mixed tag styles without sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "signalfx/dogstatsd mixed tag styles without sampling": { + in: "foo[tag1=foo,tag3=bing]:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "influxdb/dogstatsd mixed tag styles without sampling": { + in: "foo,tag1=foo,tag3=bing:100|c|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "mixed tag styles with sampling": { + in: "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz", + out: event.Events{}, + }, + "datadog tag extension with multiple colons": { + in: "foo:100|c|@0.1|#tag1:foo:bar", + out: event.Events{ + &event.CounterEvent{ + CMetricName: "foo", + CValue: 1000, + CLabels: map[string]string{"tag1": "foo:bar"}, + }, + }, + }, + "datadog tag extension with invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag:\xc3\x28invalid", + }, + "datadog tag extension with both valid and invalid utf8 tag values": { + in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid", + }, + } + + parser := NewParser() + parser.EnableDogstatsdParsing() + parser.EnableInfluxdbParsing() + parser.EnableSignalFXParsing() + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + events := parser.LineToEvents(testCase.in, *nopSampleErrors, nopSamplesReceived, nopTagErrors, nopTagsReceived, nopLogger) + + for j, expected := range testCase.out { + if !reflect.DeepEqual(&expected, &events[j]) { + t.Fatalf("Expected %#v, got %#v in scenario '%s'", expected, events[j], name) + } + } + }) + } +} diff --git a/pkg/listener/listener.go b/pkg/listener/listener.go index 5c31f53..0607180 100644 --- a/pkg/listener/listener.go +++ b/pkg/listener/listener.go @@ -31,6 +31,7 @@ type StatsDUDPListener struct { Conn *net.UDPConn EventHandler event.EventHandler Logger log.Logger + LineParser *pkgLine.Parser UDPPackets prometheus.Counter LinesReceived prometheus.Counter EventsFlushed prometheus.Counter @@ -67,7 +68,7 @@ func (l *StatsDUDPListener) HandlePacket(packet []byte) { for _, line := range lines { level.Debug(l.Logger).Log("msg", "Incoming line", "proto", "udp", "line", line) l.LinesReceived.Inc() - l.EventHandler.Queue(pkgLine.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) + l.EventHandler.Queue(l.LineParser.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) } } @@ -75,6 +76,7 @@ type StatsDTCPListener struct { Conn *net.TCPListener EventHandler event.EventHandler Logger log.Logger + LineParser *pkgLine.Parser LinesReceived prometheus.Counter EventsFlushed prometheus.Counter SampleErrors prometheus.CounterVec @@ -128,7 +130,7 @@ func (l *StatsDTCPListener) HandleConn(c *net.TCPConn) { break } l.LinesReceived.Inc() - l.EventHandler.Queue(pkgLine.LineToEvents(string(line), l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) + l.EventHandler.Queue(l.LineParser.LineToEvents(string(line), l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) } } @@ -136,6 +138,7 @@ type StatsDUnixgramListener struct { Conn *net.UnixConn EventHandler event.EventHandler Logger log.Logger + LineParser *pkgLine.Parser UnixgramPackets prometheus.Counter LinesReceived prometheus.Counter EventsFlushed prometheus.Counter @@ -172,6 +175,6 @@ func (l *StatsDUnixgramListener) HandlePacket(packet []byte) { for _, line := range lines { level.Debug(l.Logger).Log("msg", "Incoming line", "proto", "unixgram", "line", line) l.LinesReceived.Inc() - l.EventHandler.Queue(pkgLine.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) + l.EventHandler.Queue(l.LineParser.LineToEvents(line, l.SampleErrors, l.SamplesReceived, l.TagErrors, l.TagsReceived, l.Logger)) } }