Add support for Librato-style tags

Signed-off-by: Tony Wooster <twooster@gmail.com>

README.md

@@ -29,15 +29,37 @@ We recommend this only as an intermediate solution and recommend switching to
 [native Prometheus instrumentation](http://prometheus.io/docs/instrumenting/clientlibs/)
 in the long term.
 
-### DogStatsD extensions
+### Tagging Extensions
 
-The exporter will convert DogStatsD-style tags to prometheus labels. See
+The exporter supports both Librato-style tags and DogStatsD-style tags,
-[Tags](https://docs.datadoghq.com/developers/dogstatsd/data_types/#tagging) in the DogStatsD
+which will be converted into Prometheus labels.
-documentation for the concept description and
+
-[Datagram Format](https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/)
+For Librato-style tags, they must be appended to the metric name, as so:
-for specifics. It boils down to appending
+
-`|#tag:value,another_tag:another_value` to the normal StatsD format.  Tags
+```
-without values (`#some_tag`) are not supported.
+metric.name#tagName=val,tag2Name=val2:0|c
+```
+
+See the [statsd-librato-backend README](https://github.com/librato/statsd-librato-backend#tags)
+for a more complete description.
+
+For DogStatsD-style tags, they're appended as another section at the end of the
+metric, as so:
+
+```
+metric.name:0|c|#tagName=val,tag2Name=val2
+```
+
+See [Tags](https://docs.datadoghq.com/developers/dogstatsd/data_types/#tagging)
+in the DogStatsD documentation for the concept description and
+[Datagram Format](https://docs.datadoghq.com/developers/dogstatsd/datagram_shell/).
+
+Although you can use both tag types simultaneously, this is not recommended.
+DogStatsD `name=value` pairs will take priority over Librato tags with the same
+name.
+
+For both Librato and DogStatsD tags, tags without values (`#some_tag`) are not
+supported.
 
 ## Building and Running
 
@@ -235,8 +257,8 @@ mappings:
 
 Note that timers will be accepted with the `ms`, `h`, and `d` statsd types.  The first two are timers and histograms and the `d` type is for DataDog's "distribution" type.  The distribution type is treated identically to timers and histograms.
 
-It should be noted that whereas timers in statsd expects the unit of timing data to be in milliseconds, 
+It should be noted that whereas timers in statsd expects the unit of timing data to be in milliseconds,
-prometheus expects the unit to be seconds. Hence, the exporter converts all timers to seconds 
+prometheus expects the unit to be seconds. Hence, the exporter converts all timers to seconds
 before exporting them.
 
 ### DogStatsD Client Behavior

bridge_test.go

@@ -127,6 +127,26 @@ func TestHandlePacket(t *testing.T) {
 					labels:     map[string]string{"tag1": "bar", "tag2": "baz"},
 				},
 			},
+		}, {
+			name: "librato tag extension",
+			in:   "foo#tag1=bar,tag2=baz:100|c",
+			out: Events{
+				&CounterEvent{
+					metricName: "foo",
+					value:      100,
+					labels:     map[string]string{"tag1": "bar", "tag2": "baz"},
+				},
+			},
+		}, {
+			name: "librato tag extension with tag keys unsupported by prometheus",
+			in:   "foo#09digits=0,tag.with.dots=1:100|c",
+			out: Events{
+				&CounterEvent{
+					metricName: "foo",
+					value:      100,
+					labels:     map[string]string{"_09digits": "0", "tag_with_dots": "1"},
+				},
+			},
 		}, {
 			name: "datadog tag extension",
 			in:   "foo:100|c|#tag1:bar,tag2:baz",
@@ -197,6 +217,16 @@ func TestHandlePacket(t *testing.T) {
 					labels:     map[string]string{"tag1": "bar", "tag2": "baz"},
 				},
 			},
+		}, {
+			name: "librato and datadog tag extension with sampling",
+			in:   "foo#tag1=foo,tag3=bing:100|c|@0.1|#tag1:bar,#tag2:baz",
+			out: Events{
+				&CounterEvent{
+					metricName: "foo",
+					value:      1000,
+					labels:     map[string]string{"tag1": "bar", "tag3": "bing", "tag2": "baz"},
+				},
+			},
 		}, {
 			name: "histogram with sampling",
 			in:   "foo:0.01|h|@0.2|#tag1:bar,#tag2:baz",

exporter.go

@@ -275,7 +275,63 @@ func buildEvent(statType, metric string, value float64, relative bool, labels ma
 	}
 }
 
-func handleDogStatsDTagToKeyValue(labels map[string]string, component, tag string) {
+func handleLibratoTag(component, tag string, labels map[string]string) {
+	// Empty tag is an error
+	if len(tag) == 0 {
+		tagErrors.Inc()
+		log.Debugf("Empty Librato tag in component %s", component)
+		return
+	}
+
+	for i, c := range tag {
+		if c == '=' {
+			k := tag[:i]
+			v := tag[i+1:]
+
+			if len(k) == 0 || len(v) == 0 {
+				// Empty key or value, so it's an error
+				tagErrors.Inc()
+				log.Debugf("Malformed Librato tag %s=%s in component %s", k, v, component)
+			} else {
+				labels[escapeMetricName(k)] = v
+			}
+			return
+		}
+	}
+
+	// Could not find an equals sign, so it's an error
+	tagErrors.Inc()
+	log.Debugf("Malformed Librato tag %s in component %s", tag, component)
+}
+
+func parseLibratoTags(component string, labels map[string]string) {
+	lastTagEndIndex := 0
+	for i, c := range component {
+		if c == ',' {
+			tag := component[lastTagEndIndex:i]
+			lastTagEndIndex = i + 1
+			handleLibratoTag(component, tag, labels)
+		}
+	}
+
+	// If we're not off the end of the string, add the last tag
+	if lastTagEndIndex < len(component) {
+		tag := component[lastTagEndIndex:]
+		handleLibratoTag(component, tag, labels)
+	}
+}
+
+func parseNameAndLibratoLabels(name string, labels map[string]string) string {
+	for i, c := range name {
+		if c == '#' {
+			parseLibratoTags(name[i+1:], labels)
+			return name[:i]
+		}
+	}
+	return name
+}
+
+func handleDogStatsDTag(component, tag string, labels map[string]string) {
 	// Bail early if the tag is empty
 	if len(tag) == 0 {
 		tagErrors.Inc()
@@ -305,30 +361,23 @@ func handleDogStatsDTagToKeyValue(labels map[string]string, component, tag strin
 	}
 
 	labels[escapeMetricName(k)] = v
-
-	return
 }
 
-func parseDogStatsDTagsToLabels(component string) map[string]string {
+func parseDogStatsDTagsToLabels(component string, labels map[string]string) {
-	labels := map[string]string{}
-	tagsReceived.Inc()
-
 	lastTagEndIndex := 0
 	for i, c := range component {
 		if c == ',' {
 			tag := component[lastTagEndIndex:i]
 			lastTagEndIndex = i + 1
-			handleDogStatsDTagToKeyValue(labels, component, tag)
+			handleDogStatsDTag(component, tag, labels)
 		}
 	}
 
 	// If we're not off the end of the string, add the last tag
 	if lastTagEndIndex < len(component) {
 		tag := component[lastTagEndIndex:]
-		handleDogStatsDTagToKeyValue(labels, component, tag)
+		handleDogStatsDTag(component, tag, labels)
 	}
-
-	return labels
 }
 
 func lineToEvents(line string) Events {
@@ -343,7 +392,9 @@ func lineToEvents(line string) Events {
 		log.Debugln("Bad line from StatsD:", line)
 		return events
 	}
-	metric := elements[0]
+
+	labels := map[string]string{}
+	metric := parseNameAndLibratoLabels(elements[0], labels)
 	var samples []string
 	if strings.Contains(elements[1], "|#") {
 		// using datadog extensions, disable multi-metrics
@@ -376,7 +427,6 @@ samples:
 		}
 
 		multiplyEvents := 1
-		labels := map[string]string{}
 		if len(components) >= 3 {
 			for _, component := range components[2:] {
 				if len(component) == 0 {
@@ -407,7 +457,7 @@ samples:
 						multiplyEvents = int(1 / samplingFactor)
 					}
 				case '#':
-					labels = parseDogStatsDTagsToLabels(component[1:])
+					parseDogStatsDTagsToLabels(component[1:], labels)
 				default:
 					log.Debugf("Invalid sampling factor or tag section %s on line %s", components[2], line)
 					sampleErrors.WithLabelValues("invalid_sample_factor").Inc()
@@ -416,6 +466,10 @@ samples:
 			}
 		}
 
+		if len(labels) > 0 {
+			tagsReceived.Inc()
+		}
+
 		for i := 0; i < multiplyEvents; i++ {
 			event, err := buildEvent(statType, metric, value, relative, labels)
 			if err != nil {

exporter_test.go

@@ -953,7 +953,8 @@ func BenchmarkParseDogStatsDTagsToLabels(b *testing.B) {
 	for name, tags := range scenarios {
 		b.Run(name, func(b *testing.B) {
 			for n := 0; n < b.N; n++ {
-				parseDogStatsDTagsToLabels(tags)
+				labels := map[string]string{}
+				parseDogStatsDTagsToLabels(tags, labels)
 			}
 		})
 	}