Merge pull request #50 from Jimdo/drop_invalid_utf8

Skip metrics with invalid utf8
This commit is contained in:
Tobias Schmidt 2016-07-27 16:53:34 -04:00 committed by GitHub
commit 35828e29e9
4 changed files with 116 additions and 1 deletions

View file

@ -136,6 +136,22 @@ func TestHandlePacket(t *testing.T) {
labels: map[string]string{"tag1": "foo:bar"},
},
},
}, {
name: "datadog tag extension with invalid utf8 tag values",
in: "foo:100|c|@0.1|#tag:\xc3\x28invalid",
}, {
name: "datadog tag extension with both valid and invalid utf8 tag values",
in: "foo:100|c|@0.1|#tag1:valid,tag2:\xc3\x28invalid",
}, {
name: "multiple metrics with invalid datadog utf8 tag values",
in: "foo:200|c|#tag:value\nfoo:300|c|#tag:\xc3\x28invalid",
out: Events{
&CounterEvent{
metricName: "foo",
value: 200,
labels: map[string]string{"tag": "value"},
},
},
}, {
name: "combined multiline metrics",
in: "foo:200|ms:300|ms:5|c|@0.1:6|g\nbar:1|c:5|ms",
@ -212,6 +228,21 @@ func TestHandlePacket(t *testing.T) {
name: "empty component",
in: "foo:1|c|",
},
{
name: "invalid utf8",
in: "invalid\xc3\x28utf8:1|c",
},
{
name: "some invalid utf8",
in: "valid_utf8:1|c\ninvalid\xc3\x28utf8:1|c",
out: Events{
&CounterEvent{
metricName: "valid_utf8",
value: 1,
labels: map[string]string{},
},
},
},
}
l := StatsDListener{}

View file

@ -22,6 +22,7 @@ import (
"regexp"
"strconv"
"strings"
"unicode/utf8"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/log"
@ -352,7 +353,7 @@ func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
}
elements := strings.SplitN(line, ":", 2)
if len(elements) < 2 || len(elements[0]) == 0 {
if len(elements) < 2 || len(elements[0]) == 0 || !utf8.ValidString(line) {
networkStats.WithLabelValues("malformed_line").Inc()
log.Errorln("Bad line from StatsD:", line)
continue

View file

@ -0,0 +1,61 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"testing"
)
func benchmarkExporter(times int, b *testing.B) {
input := []string{
"foo1:2|c",
"foo2:3|g",
"foo3:200|ms",
"foo4:100|c|#tag1:bar,tag2:baz",
"foo5:100|c|#tag1:bar,#tag2:baz",
"foo6:100|c|#09digits:0,tag.with.dots:1",
"foo10:100|c|@0.1|#tag1:bar,#tag2:baz",
"foo11:100|c|@0.1|#tag1:foo:bar",
"foo15:200|ms:300|ms:5|c|@0.1:6|g\nfoo15a:1|c:5|ms",
"some_very_useful_metrics_with_quite_a_log_name:13|c",
}
bytesInput := make([]string, len(input)*times)
for run := 0; run < times; run++ {
for i := 0; i < len(input); i++ {
bytesInput[run*len(input)+i] = fmt.Sprintf("run%d%s", run, input[i])
}
}
for n := 0; n < b.N; n++ {
l := StatsDListener{}
// there are more events than input lines, need bigger buffer
events := make(chan Events, len(bytesInput)*times*2)
for i := 0; i < times; i++ {
for _, line := range bytesInput {
l.handlePacket([]byte(line), events)
}
}
}
}
func BenchmarkExporter1(b *testing.B) {
benchmarkExporter(1, b)
}
func BenchmarkExporter5(b *testing.B) {
benchmarkExporter(5, b)
}
func BenchmarkExporter50(b *testing.B) {
benchmarkExporter(50, b)
}

View file

@ -50,3 +50,25 @@ func TestNegativeCounter(t *testing.T) {
ex.Listen(events)
}
// TestInvalidUtf8InDatadogTagValue validates robustness of exporter listener
// against datadog tags with invalid tag values.
// It sends the same tags first with a valid value, then with an invalid one.
// The exporter should not panic, but drop the invalid event
func TestInvalidUtf8InDatadogTagValue(t *testing.T) {
l := StatsDListener{}
events := make(chan Events, 2)
l.handlePacket([]byte("bar:200|c|#tag:value"), events)
l.handlePacket([]byte("bar:200|c|#tag:\xc3\x28invalid"), events)
ex := NewExporter(&metricMapper{}, true)
// Close channel to signify we are done with the listener after a short period.
go func() {
time.Sleep(time.Millisecond * 100)
close(events)
}()
ex.Listen(events)
}