statsd_exporter/pkg/mapper/mapper_test.go
Thomas Gummerer 80b77513a6 create sub-hierarchies for summary and histogram options
Currently the Buckets and Quantiles settings are top level settings per metric
in the yaml.  In a subsequent commit we're going to allow adding more such
options for summaries, at which point having them all at the top level gets
confusing.

Split the options out into separate hierarchies to allow adding more options,
without adding confusion.  We preserve backwards compatibility by still
accepting the old option, but warning when it is present.

Signed-off-by: Thomas Gummerer <t.gummerer@gmail.com>
2020-02-18 18:04:27 +00:00

988 lines
20 KiB
Go

// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import (
"testing"
"time"
)
type mappings []struct {
statsdMetric string
name string
labels map[string]string
quantiles []metricObjective
notPresent bool
ttl time.Duration
metricType MetricType
}
func TestMetricMapperYAML(t *testing.T) {
scenarios := []struct {
config string
configBad bool
mappings mappings
}{
// Empty config.
{},
// Config with several mapping definitions.
{
config: `---
mappings:
- match: test.dispatcher.*.*.*
name: "dispatch_events"
labels:
processor: "$1"
action: "$2"
result: "$3"
job: "test_dispatcher"
- match: test.my-dispatch-host01.name.dispatcher.*.*.*
name: "host_dispatch_events"
labels:
processor: "$1"
action: "$2"
result: "$3"
job: "test_dispatcher"
- match: request_time.*.*.*.*.*.*.*.*.*.*.*.*
name: "tyk_http_request"
labels:
method_and_path: "${1}"
response_code: "${2}"
apikey: "${3}"
apiversion: "${4}"
apiname: "${5}"
apiid: "${6}"
ipv4_t1: "${7}"
ipv4_t2: "${8}"
ipv4_t3: "${9}"
ipv4_t4: "${10}"
orgid: "${11}"
oauthid: "${12}"
- match: "*.*"
name: "catchall"
labels:
first: "$1"
second: "$2"
third: "$3"
job: "$1-$2-$3"
- match: (.*)\.(.*)-(.*)\.(.*)
match_type: regex
name: "proxy_requests_total"
labels:
job: "$1"
protocol: "$2"
endpoint: "$3"
result: "$4"
`,
mappings: mappings{
{
statsdMetric: "test.dispatcher.FooProcessor.send.succeeded",
name: "dispatch_events",
labels: map[string]string{
"processor": "FooProcessor",
"action": "send",
"result": "succeeded",
"job": "test_dispatcher",
},
},
{
statsdMetric: "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded",
name: "host_dispatch_events",
labels: map[string]string{
"processor": "FooProcessor",
"action": "send",
"result": "succeeded",
"job": "test_dispatcher",
},
},
{
statsdMetric: "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.",
name: "tyk_http_request",
labels: map[string]string{
"method_and_path": "get/threads/1/posts",
"response_code": "200",
"apikey": "00000000",
"apiversion": "nonversioned",
"apiname": "discussions",
"apiid": "a11bbcdf0ac64ec243658dc64b7100fb",
"ipv4_t1": "172",
"ipv4_t2": "20",
"ipv4_t3": "0",
"ipv4_t4": "1",
"orgid": "12ba97b7eaa1a50001000001",
"oauthid": "",
},
},
{
statsdMetric: "foo.bar",
name: "catchall",
labels: map[string]string{
"first": "foo",
"second": "bar",
"third": "",
"job": "foo-bar-",
},
},
{
statsdMetric: "foo.bar.baz",
},
{
statsdMetric: "proxy-1.http-goober.success",
name: "proxy_requests_total",
labels: map[string]string{
"job": "proxy-1",
"protocol": "http",
"endpoint": "goober",
"result": "success",
},
},
},
},
//Config with backtracking
{
config: `
defaults:
glob_disable_ordering: true
mappings:
- match: backtrack.*.bbb
name: "testb"
labels:
label: "${1}_foo"
- match: backtrack.justatest.aaa
name: "testa"
labels:
label: "${1}_foo"
`,
mappings: mappings{
{
statsdMetric: "backtrack.good.bbb",
name: "testb",
labels: map[string]string{
"label": "good_foo",
},
},
{
statsdMetric: "backtrack.justatest.bbb",
name: "testb",
labels: map[string]string{
"label": "justatest_foo",
},
},
{
statsdMetric: "backtrack.justatest.aaa",
name: "testa",
labels: map[string]string{
"label": "_foo",
},
},
},
},
//Config with backtracking, the non-matched rule has star(s)
// A metric like full.name.anothertest will first match full.name.* and then tries
// to match *.dummy.* and then failed.
// This test case makes sure the captures in the non-matched later rule
// doesn't affect the captures in the first matched rule.
{
config: `
defaults:
glob_disable_ordering: false
mappings:
- match: '*.dummy.*'
name: metric_one
labels:
system: $1
attribute: $2
- match: 'full.name.*'
name: metric_two
labels:
system: static
attribute: $1
`,
mappings: mappings{
{
statsdMetric: "whatever.dummy.test",
name: "metric_one",
labels: map[string]string{
"system": "whatever",
"attribute": "test",
},
},
{
statsdMetric: "full.name.anothertest",
name: "metric_two",
labels: map[string]string{
"system": "static",
"attribute": "anothertest",
},
},
},
},
//Config with super sets, disables ordering
{
config: `
defaults:
glob_disable_ordering: true
mappings:
- match: noorder.*.*
name: "testa"
labels:
label: "${1}_foo"
- match: noorder.*.bbb
name: "testb"
labels:
label: "${1}_foo"
- match: noorder.ccc.bbb
name: "testc"
labels:
label: "ccc_foo"
`,
mappings: mappings{
{
statsdMetric: "noorder.good.bbb",
name: "testb",
labels: map[string]string{
"label": "good_foo",
},
},
{
statsdMetric: "noorder.ccc.bbb",
name: "testc",
labels: map[string]string{
"label": "ccc_foo",
},
},
},
},
//Config with super sets, keeps ordering
{
config: `
defaults:
glob_disable_ordering: false
mappings:
- match: order.*.*
name: "testa"
labels:
label: "${1}_foo"
- match: order.*.bbb
name: "testb"
labels:
label: "${1}_foo"
`,
mappings: mappings{
{
statsdMetric: "order.good.bbb",
name: "testa",
labels: map[string]string{
"label": "good_foo",
},
},
},
},
// Config with bad regex reference.
{
config: `---
mappings:
- match: test.*
name: "name"
labels:
label: "$1_foo"
`,
mappings: mappings{
{
statsdMetric: "test.a",
name: "name",
labels: map[string]string{
"label": "",
},
},
},
},
// Config with good regex reference.
{
config: `
mappings:
- match: test.*
name: "name"
labels:
label: "${1}_foo"
`,
mappings: mappings{
{
statsdMetric: "test.a",
name: "name",
labels: map[string]string{
"label": "a_foo",
},
},
},
},
// Config with bad metric line.
{
config: `---
mappings:
- match: bad--metric-line.*.*
name: "foo"
labels: {}
`,
configBad: true,
},
// Config with dynamic metric name.
{
config: `---
mappings:
- match: test1.*.*
name: "$1"
labels: {}
- match: test2.*.*
name: "${1}_$2"
labels: {}
- match: test3\.(\w+)\.(\w+)
match_type: regex
name: "${2}_$1"
labels: {}
`,
mappings: mappings{
{
statsdMetric: "test1.total_requests.count",
name: "total_requests",
},
{
statsdMetric: "test2.total_requests.count",
name: "total_requests_count",
},
{
statsdMetric: "test3.total_requests.count",
name: "count_total_requests",
},
},
},
// Config with bad metric name.
{
config: `---
mappings:
- match: test.*.*
name: "0foo"
labels: {}
`,
configBad: true,
},
// Config with no metric name.
{
config: `---
mappings:
- match: test.*.*
labels:
this: "$1"
`,
configBad: true,
},
// Config with no mappings.
{
config: ``,
mappings: mappings{},
},
// Config without a trailing newline.
{
config: `mappings:
- match: test.*
name: "name"
labels:
label: "${1}_foo"`,
mappings: mappings{
{
statsdMetric: "test.a",
name: "name",
labels: map[string]string{
"label": "a_foo",
},
},
},
},
// Config with an improperly escaped *.
{
config: `
mappings:
- match: *.test.*
name: "name"
labels:
label: "${1}_foo"`,
configBad: true,
},
// Config with a properly escaped *.
{
config: `
mappings:
- match: "*.test.*"
name: "name"
labels:
label: "${2}_foo"`,
mappings: mappings{
{
statsdMetric: "foo.test.a",
name: "name",
labels: map[string]string{
"label": "a_foo",
},
},
},
},
// Config with good timer type.
{
config: `---
mappings:
- match: test.*.*
timer_type: summary
name: "foo"
labels: {}
quantiles:
- quantile: 0.42
error: 0.04
- quantile: 0.7
error: 0.002
`,
mappings: mappings{
{
statsdMetric: "test.*.*",
name: "foo",
labels: map[string]string{},
quantiles: []metricObjective{
{Quantile: 0.42, Error: 0.04},
{Quantile: 0.7, Error: 0.002},
},
},
},
},
{
config: `---
mappings:
- match: test1.*.*
timer_type: summary
name: "foo"
labels: {}
`,
mappings: mappings{
{
statsdMetric: "test1.*.*",
name: "foo",
labels: map[string]string{},
quantiles: []metricObjective{
{Quantile: 0.5, Error: 0.05},
{Quantile: 0.9, Error: 0.01},
{Quantile: 0.99, Error: 0.001},
},
},
},
},
// Config with bad timer type.
{
config: `---
mappings:
- match: test.*.*
timer_type: wrong
name: "foo"
labels: {}
`,
configBad: true,
},
// new style quantiles
{
config: `---
mappings:
- match: test.*.*
timer_type: summary
name: "foo"
labels: {}
summary_options:
quantiles:
- quantile: 0.42
error: 0.04
- quantile: 0.7
error: 0.002
`,
mappings: mappings{
{
statsdMetric: "test.*.*",
name: "foo",
labels: map[string]string{},
quantiles: []metricObjective{
{Quantile: 0.42, Error: 0.04},
{Quantile: 0.7, Error: 0.002},
},
},
},
},
// duplicate quantiles are bad
{
config: `---
mappings:
- match: test.*.*
timer_type: summary
name: "foo"
labels: {}
quantiles:
- quantile: 0.42
error: 0.04
summary_options:
quantiles:
- quantile: 0.42
error: 0.04
`,
configBad: true,
},
// Config with good metric type.
{
config: `---
mappings:
- match: test.*.*
match_metric_type: counter
name: "foo"
labels: {}
`,
},
// Config with bad metric type matcher.
{
config: `---
mappings:
- match: test.*.*
match_metric_type: wrong
name: "foo"
labels: {}
`,
configBad: true,
},
// Config with multiple explicit metric types
{
config: `---
mappings:
- match: test.foo.*
name: "test_foo_sum"
match_metric_type: counter
- match: test.foo.*
name: "test_foo_current"
match_metric_type: gauge
`,
mappings: mappings{
{
statsdMetric: "test.foo.test",
name: "test_foo_sum",
metricType: MetricTypeCounter,
},
{
statsdMetric: "test.foo.test",
name: "test_foo_current",
metricType: MetricTypeGauge,
},
},
},
//Config with uncompilable regex.
{
config: `---
mappings:
- match: "*\\.foo"
match_type: regex
name: "foo"
labels: {}
`,
configBad: true,
},
//Config with non-matched metric.
{
config: `---
mappings:
- match: foo.*.*
timer_type: summary
name: "foo"
labels: {}
`,
mappings: mappings{
{
statsdMetric: "test.1.2",
name: "test_1_2",
labels: map[string]string{},
notPresent: true,
},
},
},
//Config with no name.
{
config: `---
mappings:
- match: *\.foo
match_type: regex
labels:
bar: "foo"
`,
configBad: true,
},
// Example from the README.
{
config: `
mappings:
- match: test.dispatcher.*.*.*
name: "dispatcher_events_total"
labels:
processor: "$1"
action: "$2"
outcome: "$3"
job: "test_dispatcher"
- match: "*.signup.*.*"
name: "signup_events_total"
labels:
provider: "$2"
outcome: "$3"
job: "${1}_server"
`,
mappings: mappings{
{
statsdMetric: "test.dispatcher.FooProcessor.send.success",
name: "dispatcher_events_total",
labels: map[string]string{
"processor": "FooProcessor",
"action": "send",
"outcome": "success",
"job": "test_dispatcher",
},
},
{
statsdMetric: "foo_product.signup.facebook.failure",
name: "signup_events_total",
labels: map[string]string{
"provider": "facebook",
"outcome": "failure",
"job": "foo_product_server",
},
},
{
statsdMetric: "test.web-server.foo.bar",
name: "test_web_server_foo_bar",
labels: map[string]string{},
},
},
},
// Config that drops all.
{
config: `mappings:
- match: .
match_type: regex
name: "drop"
action: drop`,
mappings: mappings{
{
statsdMetric: "test.a",
},
{
statsdMetric: "abc",
},
},
},
// Config that has a catch-all to drop all.
{
config: `mappings:
- match: web.*
name: "web"
labels:
site: "$1"
- match: .
match_type: regex
name: "drop"
action: drop`,
mappings: mappings{
{
statsdMetric: "test.a",
},
{
statsdMetric: "web.localhost",
name: "web",
labels: map[string]string{
"site": "localhost",
},
},
},
},
// Config that has a ttl.
{
config: `mappings:
- match: web.*
name: "web"
ttl: 10s
labels:
site: "$1"`,
mappings: mappings{
{
statsdMetric: "test.a",
},
{
statsdMetric: "web.localhost",
name: "web",
labels: map[string]string{
"site": "localhost",
},
ttl: time.Second * 10,
},
},
},
// Config that has a default ttl.
{
config: `defaults:
ttl: 1m2s
mappings:
- match: web.*
name: "web"
labels:
site: "$1"`,
mappings: mappings{
{
statsdMetric: "test.a",
},
{
statsdMetric: "web.localhost",
name: "web",
labels: map[string]string{
"site": "localhost",
},
ttl: time.Minute + time.Second*2,
},
},
},
// Config that override a default ttl.
{
config: `defaults:
ttl: 1m2s
mappings:
- match: web.*
name: "web"
ttl: 5s
labels:
site: "$1"`,
mappings: mappings{
{
statsdMetric: "test.a",
},
{
statsdMetric: "web.localhost",
name: "web",
labels: map[string]string{
"site": "localhost",
},
ttl: time.Second * 5,
},
},
},
}
mapper := MetricMapper{}
for i, scenario := range scenarios {
err := mapper.InitFromYAMLString(scenario.config, 1000)
if err != nil && !scenario.configBad {
t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err)
}
if err == nil && scenario.configBad {
t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config)
}
for metric, mapping := range scenario.mappings {
// exporter will call mapper.GetMapping with valid MetricType
// so we also pass a sane MetricType in testing if it's not specified
mapType := mapping.metricType
if mapType == "" {
mapType = MetricTypeCounter
}
m, labels, present := mapper.GetMapping(mapping.statsdMetric, mapType)
if present && mapping.name != "" && m.Name != mapping.name {
t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name)
}
if mapping.notPresent && present {
t.Fatalf("%d.%q: Expected metric to not be present", i, metric)
}
if len(labels) != len(mapping.labels) {
t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping.labels), len(labels))
}
for label, value := range labels {
if mapping.labels[label] != value {
t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels)
}
}
if mapping.ttl > 0 && mapping.ttl != m.Ttl {
t.Fatalf("%d.%q: Expected ttl of %s, got %s", i, metric, mapping.ttl.String(), m.Ttl.String())
}
if mapping.metricType != "" && mapType != m.MatchMetricType {
t.Fatalf("%d.%q: Expected match metric of %s, got %s", i, metric, mapType, m.MatchMetricType)
}
if len(mapping.quantiles) != 0 {
if len(mapping.quantiles) != len(m.SummaryOptions.Quantiles) {
t.Fatalf("%d.%q: Expected %d quantiles, got %d", i, metric, len(mapping.quantiles), len(m.SummaryOptions.Quantiles))
}
for i, quantile := range mapping.quantiles {
if quantile.Quantile != m.SummaryOptions.Quantiles[i].Quantile {
t.Fatalf("%d.%q: Expected quantile %v, got %v", i, metric, m.SummaryOptions.Quantiles[i].Quantile, quantile.Quantile)
}
if quantile.Error != m.SummaryOptions.Quantiles[i].Error {
t.Fatalf("%d.%q: Expected Error margin %v, got %v", i, metric, m.SummaryOptions.Quantiles[i].Error, quantile.Error)
}
}
}
}
}
}
func TestAction(t *testing.T) {
scenarios := []struct {
config string
configBad bool
expectedAction ActionType
}{
{
// no action set
config: `---
mappings:
- match: test.*.*
name: "foo"
`,
configBad: false,
expectedAction: ActionTypeMap,
},
{
// map action set
config: `---
mappings:
- match: test.*.*
name: "foo"
action: map
`,
configBad: false,
expectedAction: ActionTypeMap,
},
{
// drop action set
config: `---
mappings:
- match: test.*.*
name: "foo"
action: drop
`,
configBad: false,
expectedAction: ActionTypeDrop,
},
{
// invalid action set
config: `---
mappings:
- match: test.*.*
name: "foo"
action: xyz
`,
configBad: true,
expectedAction: ActionTypeDrop,
},
{
// valid yaml example
config: `---
mappings:
- match: "test\\.(\\w+)\\.(\\w+)\\.counter"
match_type: regex
name: "${2}_total"
labels:
provider: "$1"
`,
configBad: false,
expectedAction: ActionTypeMap,
},
{
// invalid yaml example
config: `---
mappings:
- match: "test\.(\w+)\.(\w+)\.counter"
match_type: regex
name: "${2}_total"
labels:
provider: "$1"
`,
configBad: true,
},
}
for i, scenario := range scenarios {
mapper := MetricMapper{}
err := mapper.InitFromYAMLString(scenario.config, 0)
if err != nil && !scenario.configBad {
t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err)
}
if err == nil && scenario.configBad {
t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config)
}
if !scenario.configBad {
a := mapper.Mappings[0].Action
if scenario.expectedAction != a {
t.Fatalf("%d: Expected action %v, got %v", i, scenario.expectedAction, a)
}
}
}
}
// Test for https://github.com/prometheus/statsd_exporter/issues/273
// Corrupt cache for multiple names matching in fsm
func TestMultipleMatches(t *testing.T) {
config := `---
mappings:
- match: aa.bb.*.*
name: "aa_bb_${1}_total"
labels:
app: "$2"
`
mapper := MetricMapper{}
err := mapper.InitFromYAMLString(config, 0)
if err != nil {
t.Fatalf("config load error: %s ", err)
}
names := map[string]string{
"aa.bb.aa.myapp": "aa_bb_aa_total",
"aa.bb.bb.myapp": "aa_bb_bb_total",
"aa.bb.cc.myapp": "aa_bb_cc_total",
"aa.bb.dd.myapp": "aa_bb_dd_total",
}
scenarios := []struct {
cacheSize int
}{
{
cacheSize: 0,
},
{
cacheSize: len(names),
},
}
for i, scenario := range scenarios {
mapper.InitCache(scenario.cacheSize)
// run multiple times to ensure cache works as expected
for j := 0; j < 10; j++ {
for name, expected := range names {
m, _, ok := mapper.GetMapping(name, MetricTypeCounter)
if !ok {
t.Fatalf("%d:%d Did not find match for %s", i, j, name)
}
if m.Name != expected {
t.Fatalf("%d:%d Expected name %s, got %s", i, j, expected, m.Name)
}
}
}
}
}