Added tagless and sampling handling in extended aggregation

Signed-off-by: Greg Chambers <gregory.w.chambers@gmail.com>

pkg/line/line.go

@@ -217,50 +217,46 @@ func (p *Parser) LineToEvents(line string, sampleErrors prometheus.CounterVec, s
 
 	labels := map[string]string{}
 	metric := p.parseNameAndTags(elements[0], labels, tagErrors, logger)
-
+	usingDogStatsDTags := strings.Contains(line, "|#")
-	var samples []string
+	if usingDogStatsDTags && len(labels) > 0 {
-	if strings.Contains(elements[1], "|#") {
 		// using DogStatsD tags
 
 		// don't allow mixed tagging styles
-		if len(labels) > 0 {
+		sampleErrors.WithLabelValues("mixed_tagging_styles").Inc()
-			sampleErrors.WithLabelValues("mixed_tagging_styles").Inc()
+		level.Debug(logger).Log("msg", "Bad line (multiple tagging styles) from StatsD", "line", line)
-			level.Debug(logger).Log("msg", "Bad line (multiple tagging styles) from StatsD", "line", line)
+		return events
+	}
+
+	var samples []string
+	lineParts := strings.SplitN(elements[1], "|", 3)
+	if strings.Contains(lineParts[0], ":") {
+		// handle DogStatsD extended aggregation
+		isValidAggType := false
+		switch lineParts[1] {
+		case
+			"ms", // timer
+			"h",  // histogram
+			"d":  // distribution
+			isValidAggType = true
+		}
+
+		if isValidAggType {
+			aggValues := strings.Split(lineParts[0], ":")
+			aggLines := make([]string, len(aggValues))
+			_, aggLineSuffix, _ := strings.Cut(elements[1], "|")
+
+			for i, aggValue := range aggValues {
+				aggLines[i] = strings.Join([]string{aggValue, aggLineSuffix}, "|")
+			}
+			samples = aggLines
+		} else {
+			sampleErrors.WithLabelValues("invalid_extended_aggregate_type").Inc()
+			level.Debug(logger).Log("msg", "Bad line (invalid extended aggregate type) from StatsD", "line", line)
 			return events
 		}
-
+	} else if usingDogStatsDTags {
-		lineParts := strings.SplitN(elements[1], "|", 3)
+		// disable multi-metrics
-		if strings.Contains(lineParts[0], ":") {
+		samples = elements[1:]
-			// handle DogStatsD extended aggregation
-			var isValidAggType bool
-			switch lineParts[1] {
-			case
-				"ms", // timer
-				"h",  // histogram
-				"d":  // distribution
-				isValidAggType = true
-			default:
-				isValidAggType = false
-			}
-
-			if isValidAggType {
-				aggValues := strings.Split(lineParts[0], ":")
-				aggLines := make([]string, len(aggValues))
-
-				for i, aggValue := range aggValues {
-					aggLines[i] = strings.Join([]string{aggValue, lineParts[1], lineParts[2]}, "|")
-				}
-
-				samples = aggLines
-			} else {
-				sampleErrors.WithLabelValues("invalid_extended_aggregate_type").Inc()
-				level.Debug(logger).Log("msg", "Bad line (invalid extended aggregate type) from StatsD", "line", line)
-				return events
-			}
-		} else {
-			// disable multi-metrics
-			samples = elements[1:]
-		}
 	} else {
 		samples = strings.Split(elements[1], ":")
 	}

pkg/line/line_test.go

@@ -404,57 +404,334 @@ func TestLineToEvents(t *testing.T) {
 			in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid",
 		},
 		"datadog timings with extended aggregation values": {
-			in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,#tag2:baz",
+			in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|#tag1:bar,tag2:baz",
 			out: event.Events{
-				&event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.0005, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				&event.ObserverEvent{
-				&event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OMetricName: "foo_timing",
-				&event.ObserverEvent{OMetricName: "foo_timing", OValue: 3, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OValue:      0.0005,
-				&event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
-				&event.ObserverEvent{OMetricName: "foo_timing", OValue: 20, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				},
-				&event.ObserverEvent{OMetricName: "foo_timing", OValue: 0.00001, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.120,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      3,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.01,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      20,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.00001,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
 			},
 		},
-		"datadog histogram with extended aggregation values": {
+		"datadog timings with extended aggregation values without tags": {
-			in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,#tag2:baz",
+			in: "foo_timing:0.5:120:3000:10:20000:0.01|ms",
 			out: event.Events{
-				&event.ObserverEvent{OMetricName: "foo_histogram", OValue: 0.5, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				&event.ObserverEvent{
-				&event.ObserverEvent{OMetricName: "foo_histogram", OValue: 120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OMetricName: "foo_timing",
-				&event.ObserverEvent{OMetricName: "foo_histogram", OValue: 3000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OValue:      0.0005,
-				&event.ObserverEvent{OMetricName: "foo_histogram", OValue: 10, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OLabels:     map[string]string{},
-				&event.ObserverEvent{OMetricName: "foo_histogram", OValue: 20000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				},
-				&event.ObserverEvent{OMetricName: "foo_histogram", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.120,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      3,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.01,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      20,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.00001,
+					OLabels:     map[string]string{},
+				},
+			},
+		},
+		"datadog timings with extended aggregation values and sampling but without tags": {
+			in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5",
+			out: event.Events{
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.120,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.120,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      3,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      3,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.01,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.01,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      20,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      20,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.00001,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.00001,
+					OLabels:     map[string]string{},
+				},
+			},
+		},
+		"datadog timings with extended aggregation values, sampling, and tags": {
+			in: "foo_timing:0.5:120:3000:10:20000:0.01|ms|@0.5|#tag1:bar,tag2:baz",
+			out: event.Events{
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.120,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.120,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      3,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      3,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.01,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.01,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      20,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      20,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.00001,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_timing",
+					OValue:      0.00001,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+			},
+		},
+		"datadog histogram with extended aggregation values and tags": {
+			in: "foo_histogram:0.5:120:3000:10:20000:0.01|h|#tag1:bar,tag2:baz",
+			out: event.Events{
+				&event.ObserverEvent{
+					OMetricName: "foo_histogram",
+					OValue:      0.5,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_histogram",
+					OValue:      120,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_histogram",
+					OValue:      3000,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_histogram",
+					OValue:      10,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_histogram",
+					OValue:      20000,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_histogram",
+					OValue:      0.01,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
 			},
 		},
 		"datadog distribution with extended aggregation values": {
-			in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,#tag2:baz",
+			in: "foo_distribution:0.5:120:3000:10:20000:0.01|d|#tag1:bar,tag2:baz",
 			out: event.Events{
-				&event.ObserverEvent{OMetricName: "foo_distribution", OValue: 0.5, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				&event.ObserverEvent{
-				&event.ObserverEvent{OMetricName: "foo_distribution", OValue: 120, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OMetricName: "foo_distribution",
-				&event.ObserverEvent{OMetricName: "foo_distribution", OValue: 3000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OValue:      0.5,
-				&event.ObserverEvent{OMetricName: "foo_distribution", OValue: 10, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
-				&event.ObserverEvent{OMetricName: "foo_distribution", OValue: 20000, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				},
-				&event.ObserverEvent{OMetricName: "foo_distribution", OValue: 0.01, OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}},
+				&event.ObserverEvent{
+					OMetricName: "foo_distribution",
+					OValue:      120,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_distribution",
+					OValue:      3000,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_distribution",
+					OValue:      10,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_distribution",
+					OValue:      20000,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo_distribution",
+					OValue:      0.01,
+					OLabels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
 			},
 		},
 		"datadog counter with invalid extended aggregation values": {
-			in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,#tag2:baz",
+			in: "foo_counter:0.5:120:3000:10:20000:0.01|c|#tag1:bar,tag2:baz",
 		},
 		"datadog gauge with invalid extended aggregation values": {
-			in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,#tag2:baz",
+			in: "foo_gauge:0.5:120:3000:10:20000:0.01|g|#tag1:bar,tag2:baz",
 		},
 		"timings with sampling factor": {
 			in: "foo.timing:0.5|ms|@0.1",
 			out: event.Events{
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+				&event.ObserverEvent{
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+					OMetricName: "foo.timing",
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+					OValue:      0.0005,
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+					OLabels:     map[string]string{},
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+				},
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+				&event.ObserverEvent{
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+					OMetricName: "foo.timing",
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+					OValue:      0.0005,
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+					OLabels:     map[string]string{},
-				&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
+				&event.ObserverEvent{
+					OMetricName: "foo.timing",
+					OValue:      0.0005,
+					OLabels:     map[string]string{},
+				},
 			},
 		},
 		"bad line": {