From ec3cc120e26a15edb03429f4bbab9ee2071f8af1 Mon Sep 17 00:00:00 2001 From: Dave Rawks Date: Fri, 29 Sep 2017 00:57:17 -0700 Subject: [PATCH] Resolves #84 adds support for regex mappers (#85) * Added `regex` match_type as valid config callout in both individual mappers and mapper defaults * Added test coverage for changes * Updated documentation to reflect usage of `match_type` parameter and provide example of a raw regex match rule --- README.md | 21 ++++++++++++++++++++- mapper.go | 32 ++++++++++++++++++++++---------- mapper_test.go | 27 +++++++++++++++++++++++++++ match.go | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 match.go diff --git a/README.md b/README.md index 69cb19e..0a189a7 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,24 @@ mappings: job: "${1}_server" ``` +Another capability when using YAML configuration is the ability to define matches +using raw regular expressions as opposed to the default globbing style of match. +This may allow for pulling structured data from otherwise poorly named statsd +metrics AND allow for more precise targetting of match rules. When no `match_type` +paramter is specified the default value of `glob` will be assumed: + +```yaml +mappings: +- match: (.*)\.(.*)--(.*)\.status\.(.*)\.count + match_type: regex + labels: + name: "request_total" + hostname: "$1" + exec: "$2" + protocol: "$3" + code: "$4" +``` + Note, that one may also set the histogram buckets. If not set, then the default [Prometheus client values](https://godoc.org/github.com/prometheus/client_golang/prometheus#pkg-variables) are used: `[.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10]`. `+Inf` is added automatically. @@ -169,13 +187,14 @@ automatically. only used when the statsd metric type is a timerand the `timer_type` is set to "histogram." -One may also set defaults for the timer type and the buckets. These will be used +One may also set defaults for the timer type, buckets and match_type. These will be used by all mappings that do not define these. ```yaml defaults: timer_type: histogram buckets: [.005, .01, .025, .05, .1, .25, .5, 1, 2.5 ] + match_type: glob mappings: # This will be a histogram using the buckets set in `defaults`. - match: test.timing.*.*.* diff --git a/mapper.go b/mapper.go index 53ce4b2..477e199 100644 --- a/mapper.go +++ b/mapper.go @@ -37,6 +37,7 @@ var ( type mapperConfigDefaults struct { TimerType timerType `yaml:"timer_type"` Buckets []float64 `yaml:"buckets"` + MatchType matchType `yaml:"match_type"` } type metricMapper struct { @@ -51,6 +52,7 @@ type metricMapping struct { Labels prometheus.Labels `yaml:"labels"` TimerType timerType `yaml:"timer_type"` Buckets []float64 `yaml:"buckets"` + MatchType matchType `yaml:"match_type"` } type configLoadStates int @@ -136,14 +138,14 @@ func (m *metricMapper) initFromYAMLString(fileContents string) error { n.Defaults.Buckets = prometheus.DefBuckets } + if n.Defaults.MatchType == matchTypeDefault { + n.Defaults.MatchType = matchTypeGlob + } + for i := range n.Mappings { currentMapping := &n.Mappings[i] - if !metricLineRE.MatchString(currentMapping.Match) { - return fmt.Errorf("invalid match: %s", currentMapping.Match) - } - - // Check that label is correct. + // check that label is correct for k, v := range currentMapping.Labels { if !metricNameRE.MatchString(k) { return fmt.Errorf("invalid label key: %s", k) @@ -156,12 +158,22 @@ func (m *metricMapper) initFromYAMLString(fileContents string) error { if _, ok := currentMapping.Labels["name"]; !ok { return fmt.Errorf("line %d: metric mapping didn't set a metric name", i) } + if currentMapping.MatchType == "" { + currentMapping.MatchType = n.Defaults.MatchType + } - // Translate the glob-style metric match line into a proper regex that we - // can use to match metrics later on. - metricRe := strings.Replace(currentMapping.Match, ".", "\\.", -1) - metricRe = strings.Replace(metricRe, "*", "([^.]+)", -1) - currentMapping.regex = regexp.MustCompile("^" + metricRe + "$") + if currentMapping.MatchType == matchTypeGlob { + if !metricLineRE.MatchString(currentMapping.Match) { + return fmt.Errorf("invalid match: %s", currentMapping.Match) + } + // Translate the glob-style metric match line into a proper regex that we + // can use to match metrics later on. + metricRe := strings.Replace(currentMapping.Match, ".", "\\.", -1) + metricRe = strings.Replace(metricRe, "*", "([^.]+)", -1) + currentMapping.regex = regexp.MustCompile("^" + metricRe + "$") + } else { + currentMapping.regex = regexp.MustCompile(currentMapping.Match) + } if currentMapping.TimerType == "" { currentMapping.TimerType = n.Defaults.TimerType diff --git a/mapper_test.go b/mapper_test.go index 149dcb3..44f8364 100644 --- a/mapper_test.go +++ b/mapper_test.go @@ -219,6 +219,15 @@ mappings: second: "$2" third: "$3" job: "$1-$2-$3" +- match: (.*)\.(.*)-(.*)\.(.*) + match_type: regex + labels: + name: "proxy_requests_total" + job: "$1" + protocol: "$2" + endpoint: "$3" + result: "$4" + `, mappings: map[string]map[string]string{ "test.dispatcher.FooProcessor.send.succeeded": map[string]string{ @@ -243,6 +252,13 @@ mappings: "job": "foo-bar-", }, "foo.bar.baz": map[string]string{}, + "proxy-1.http-goober.success": map[string]string{ + "name": "proxy_requests_total", + "job": "proxy-1", + "protocol": "http", + "endpoint": "goober", + "result": "success", + }, }, }, // Config with bad regex reference. @@ -333,6 +349,17 @@ mappings: mappings: - match: test.*.* timer_type: wrong + labels: + name: "foo" + `, + configBad: true, + }, + //Config with uncompilable regex. + { + config: `--- +mappings: +- match: *\.foo + match_type: regex labels: name: "foo" `, diff --git a/match.go b/match.go new file mode 100644 index 0000000..23357b4 --- /dev/null +++ b/match.go @@ -0,0 +1,41 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "fmt" + +type matchType string + +const ( + matchTypeGlob matchType = "glob" + matchTypeRegex matchType = "regex" + matchTypeDefault matchType = "" +) + +func (t *matchType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var v string + if err := unmarshal(&v); err != nil { + return err + } + + switch matchType(v) { + case matchTypeRegex: + *t = matchTypeRegex + case matchTypeGlob, matchTypeDefault: + *t = matchTypeGlob + default: + return fmt.Errorf("invalid match type %q", v) + } + return nil +}