From 825b734b3e45e421fa9ecec5030f7f02f139f782 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 19 Jul 2018 14:57:36 -0700 Subject: [PATCH 01/23] Implement "fsm" match type Signed-off-by: Wangchong Zhou --- pkg/mapper/mapper.go | 175 +++++++++++++++++- pkg/mapper/mapper_test.go | 363 ++++++++++++++++++++++++++++++++++++++ pkg/mapper/match.go | 3 + 3 files changed, 532 insertions(+), 9 deletions(-) diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 973607c..c37a783 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -14,13 +14,17 @@ package mapper import ( + "bufio" "fmt" "io/ioutil" + "os" "regexp" + "strconv" "strings" "sync" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" yaml "gopkg.in/yaml.v2" ) @@ -28,26 +32,41 @@ var ( statsdMetricRE = `[a-zA-Z_](-?[a-zA-Z0-9_])+` templateReplaceRE = `(\$\{?\d+\}?)` - metricLineRE = regexp.MustCompile(`^(\*\.|` + statsdMetricRE + `\.)+(\*|` + statsdMetricRE + `)$`) - metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`) - labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`) + metricLineRE = regexp.MustCompile(`^(\*\.|` + statsdMetricRE + `\.)+(\*|` + statsdMetricRE + `)$`) + metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`) + labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`) + labelValueExpansionRE = regexp.MustCompile(`\${?(\d+)}?`) ) type mapperConfigDefaults struct { - TimerType TimerType `yaml:"timer_type"` - Buckets []float64 `yaml:"buckets"` - Quantiles []metricObjective `yaml:"quantiles"` - MatchType MatchType `yaml:"match_type"` + TimerType TimerType `yaml:"timer_type"` + Buckets []float64 `yaml:"buckets"` + Quantiles []metricObjective `yaml:"quantiles"` + MatchType MatchType `yaml:"match_type"` + DumpFSM string `yaml:"dump_fsm"` + FSMFallback MatchType `yaml:"fsm_fallback"` +} + +type mappingState struct { + transitions map[string]*mappingState + // result is nil unless there's a metric ends with this state + result *MetricMapping } type MetricMapper struct { Defaults mapperConfigDefaults `yaml:"defaults"` Mappings []MetricMapping `yaml:"mappings"` + FSM *mappingState mutex sync.Mutex MappingsCount prometheus.Gauge } +type labelFormatter struct { + captureIdx int + fmtString string +} + type matchMetricType string type MetricMapping struct { @@ -55,6 +74,7 @@ type MetricMapping struct { Name string `yaml:"name"` regex *regexp.Regexp Labels prometheus.Labels `yaml:"labels"` + LabelsFormatter map[string]labelFormatter TimerType TimerType `yaml:"timer_type"` Buckets []float64 `yaml:"buckets"` Quantiles []metricObjective `yaml:"quantiles"` @@ -94,7 +114,14 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { n.Defaults.MatchType = MatchTypeGlob } + maxPossibleTransitions := len(n.Mappings) + + n.FSM = &mappingState{} + n.FSM.transitions = make(map[string]*mappingState, maxPossibleTransitions) + for i := range n.Mappings { + maxPossibleTransitions-- + currentMapping := &n.Mappings[i] // check that label is correct @@ -120,7 +147,57 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { currentMapping.Action = ActionTypeMap } - if currentMapping.MatchType == MatchTypeGlob { + if currentMapping.MatchType == MatchTypeFSM { + // first split by "." + matchFields := strings.Split(currentMapping.Match, ".") + // fill into our FSM + root := n.FSM + captureCount := 0 + for i, field := range matchFields { + state, prs := root.transitions[field] + if !prs { + state = &mappingState{} + (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) + root.transitions[field] = state + // if this is last field, set result to currentMapping instance + if i == len(matchFields)-1 { + root.transitions[field].result = currentMapping + } + } + if field == "*" { + captureCount++ + } + + // goto next state + root = state + } + currentLabelFormatter := make(map[string]labelFormatter, captureCount) + for label, valueExpr := range currentMapping.Labels { + matches := labelValueExpansionRE.FindAllStringSubmatch(valueExpr, -1) + if len(matches) == 0 { + // if no regex expansion found, keep it as it is + currentLabelFormatter[label] = labelFormatter{captureIdx: -1, fmtString: valueExpr} + continue + } else if len(matches) > 1 { + return fmt.Errorf("multiple captures is not supported in FSM matching type") + } + var valueFormatter string + idx, err := strconv.Atoi(matches[0][1]) + if err != nil { + return fmt.Errorf("invalid label value expression: %s", valueExpr) + } + if idx > captureCount || idx < 1 { + // index larger than captured count, replace all expansion with empty string + valueFormatter = labelValueExpansionRE.ReplaceAllString(valueExpr, "") + idx = 0 + } else { + valueFormatter = labelValueExpansionRE.ReplaceAllString(valueExpr, "%s") + } + currentLabelFormatter[label] = labelFormatter{captureIdx: idx - 1, fmtString: valueFormatter} + } + currentMapping.LabelsFormatter = currentLabelFormatter + } + if currentMapping.MatchType == MatchTypeGlob || n.Defaults.FSMFallback == MatchTypeGlob { if !metricLineRE.MatchString(currentMapping.Match) { return fmt.Errorf("invalid match: %s", currentMapping.Match) } @@ -133,7 +210,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { } else { currentMapping.regex = regex } - } else { + } else if currentMapping.MatchType == MatchTypeRegex || n.Defaults.FSMFallback == MatchTypeRegex { if regex, err := regexp.Compile(currentMapping.Match); err != nil { return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err) } else { @@ -154,12 +231,18 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { } } + if len(n.Defaults.DumpFSM) > 0 { + m.dumpFSM(n.Defaults.DumpFSM, n.FSM) + } m.mutex.Lock() defer m.mutex.Unlock() m.Defaults = n.Defaults m.Mappings = n.Mappings + if len(n.FSM.transitions) > 0 { + m.FSM = n.FSM + } if m.MappingsCount != nil { m.MappingsCount.Set(float64(len(n.Mappings))) @@ -168,6 +251,35 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { return nil } +func (m *MetricMapper) dumpFSM(fileName string, root *mappingState) { + log.Infoln("Start dumping FSM to", fileName) + idx := 0 + states := make(map[int]*mappingState) + states[idx] = root + + f, _ := os.Create(fileName) + w := bufio.NewWriter(f) + w.WriteString("digraph g {\n") + w.WriteString("rankdir=LR\n") // make it vertical + w.WriteString("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n") // remove label of node + + for idx < len(states) { + for field, transition := range states[idx].transitions { + states[len(states)] = transition + w.WriteString(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)) + if transition.transitions == nil || len(transition.transitions) == 0 { + w.WriteString(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)) + } + + } + idx++ + } + w.WriteString(fmt.Sprintf("0 [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n")) + w.WriteString("}") + w.Flush() + log.Infoln("Finish dumping FSM") +} + func (m *MetricMapper) InitFromFile(fileName string) error { mappingStr, err := ioutil.ReadFile(fileName) if err != nil { @@ -177,6 +289,51 @@ func (m *MetricMapper) InitFromFile(fileName string) error { } func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) { + if root := m.FSM; root != nil { + matchFields := strings.Split(statsdMetric, ".") + captures := make(map[int]string, len(matchFields)) + captureIdx := 0 + filedsCount := len(matchFields) + for i, field := range matchFields { + if root.transitions == nil { + break + } + state, prs := root.transitions[field] + if !prs { + state, prs = root.transitions["*"] + if !prs { + break + } + captures[captureIdx] = field + captureIdx++ + } + if state.result != nil && i == filedsCount-1 { + // format valueExpr + mapping := *state.result + labels := prometheus.Labels{} + for label := range mapping.Labels { + formatter := mapping.LabelsFormatter[label] + idx := formatter.captureIdx + var value string + if idx == -1 { + value = formatter.fmtString + } else { + value = fmt.Sprintf(formatter.fmtString, captures[idx]) + } + labels[label] = string(value) + } + return state.result, labels, true + } + root = state + } + + // if fsm_fallback is not defined, return immediately + if len(m.Defaults.FSMFallback) == 0 { + log.Infof("%s not matched by fsm\n", statsdMetric) + return nil, nil, false + } + } + m.mutex.Lock() defer m.mutex.Unlock() diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index 79b10df..0b74f1e 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -368,6 +368,30 @@ mappings: `, configBad: true, }, + //Config with multiple captures for fsm match type + { + config: `--- + mappings: + - match: "foo.*.*" + match_type: fsm + name: "foo" + labels: + bar: "$1-$2" + `, + configBad: true, + }, + //Config with non-numeric capture index for fsm match type + { + config: `--- + mappings: + - match: "foo.*.*" + match_type: fsm + name: "foo" + labels: + bar: "$a" + `, + configBad: true, + }, //Config with non-matched metric. { config: `--- @@ -518,6 +542,345 @@ mappings: } } +func TestFSMMatcher(t *testing.T) { + scenarios := []struct { + config string + configBad bool + mappings mappings + }{ + // Empty config. + {}, + // Config with several mapping definitions. + { + config: `--- +defaults: + match_type: "fsm" +mappings: +- match: test.dispatcher.*.*.* + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher.*.*.* + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* + name: "tyk_http_request" + labels: + method_and_path: "$1" + response_code: "$2" + apikey: "$3" + apiversion: "$4" + apiname: "$5" + apiid: "$6" + ipv4_t1: "$7" + ipv4_t2: "$8" + ipv4_t3: "$9" + ipv4_t4: "$10" + orgid: "$11" + oauthid: "$12" +- match: "*.*" + name: "catchall" + labels: + first: "$1" + second: "second_label_$2" + third: "$3" + job: "-" + `, + mappings: mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "host_dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "tyk_http_request", + labels: map[string]string{ + "method_and_path": "get/threads/1/posts", + "response_code": "200", + "apikey": "00000000", + "apiversion": "nonversioned", + "apiname": "discussions", + "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", + "ipv4_t1": "172", + "ipv4_t2": "20", + "ipv4_t3": "0", + "ipv4_t4": "1", + "orgid": "12ba97b7eaa1a50001000001", + "oauthid": "", + }, + }, + "foo.bar": { + name: "catchall", + labels: map[string]string{ + "first": "foo", + "second": "second_label_bar", + "third": "", + "job": "-", + }, + }, + "foo.bar.baz": {}, + }, + }, + // local match_type + { + config: `--- +mappings: +- match: test.dispatcher.*.*.* + name: "dispatch_events" + labels: + match_type: "fsm" + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher.*.*.* + name: "host_dispatch_events" + labels: + match_type: "fsm" + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* + name: "tyk_http_request" + labels: + match_type: "fsm" + method_and_path: "$1" + response_code: "$2" + apikey: "$3" + apiversion: "$4" + apiname: "$5" + apiid: "$6" + ipv4_t1: "$7" + ipv4_t2: "$8" + ipv4_t3: "$9" + ipv4_t4: "$10" + orgid: "$11" + oauthid: "$12" +- match: "*.*" + name: "catchall" + labels: + match_type: "fsm" + first: "$1" + second: "second_label_$2" + third: "$3" + job: "-" + `, + mappings: mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "host_dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "tyk_http_request", + labels: map[string]string{ + "method_and_path": "get/threads/1/posts", + "response_code": "200", + "apikey": "00000000", + "apiversion": "nonversioned", + "apiname": "discussions", + "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", + "ipv4_t1": "172", + "ipv4_t2": "20", + "ipv4_t3": "0", + "ipv4_t4": "1", + "orgid": "12ba97b7eaa1a50001000001", + "oauthid": "", + }, + }, + "foo.bar": { + name: "catchall", + labels: map[string]string{ + "first": "foo", + "second": "second_label_bar", + "third": "", + "job": "-", + }, + }, + "foo.bar.baz": {}, + }, + }, + } + + mapper := MetricMapper{} + for i, scenario := range scenarios { + err := mapper.InitFromYAMLString(scenario.config) + if err != nil && !scenario.configBad { + t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err) + } + if err == nil && scenario.configBad { + t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config) + } + + var dummyMetricType MetricType = "" + for metric, mapping := range scenario.mappings { + m, labels, present := mapper.GetMapping(metric, dummyMetricType) + if present && mapping.name != "" && m.Name != mapping.name { + t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name) + } + if mapping.notPresent && present { + t.Fatalf("%d.%q: Expected metric to not be present", i, metric) + } + if len(labels) != len(mapping.labels) { + t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping.labels), len(labels)) + } + for label, value := range labels { + if mapping.labels[label] != value { + t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels) + } + } + + } + } +} + +func TestFSMMatcherFallbackRegex(t *testing.T) { + scenarios := []struct { + config string + configBad bool + mappings mappings + }{ + // Config with simple matcher as default mathcer and fallback_regex to false. + { + config: `--- +defaults: + match_type: "fsm" +mappings: +- match: client.*.request.duration + name: "request_size" + labels: + client: "$1" +- match: client.*.*.size + name: "request_response_size" + labels: + client: "$1" + direction: "$2" + `, + mappings: mappings{ + "client.a.request.duration": { + name: "request_size", + labels: map[string]string{ + "client": "a", + }, + }, + "client.a.request.size": {}, + "client.a.response.size": { + name: "request_response_size", + labels: map[string]string{ + "client": "a", + "direction": "response", + }, + }, + }, + }, + // Config with simple matcher as default mathcer and fallback_regex to true. + { + config: `--- +defaults: + match_type: "fsm" + fsm_fallback: "glob" +mappings: +- match: client.*.request.duration + name: "request_size" + labels: + client: "$1" +- match: client.*.*.size + name: "request_response_size" + labels: + client: "$1" + direction: "$2" + `, + mappings: mappings{ + "client.a.request.duration": { + name: "request_size", + labels: map[string]string{ + "client": "a", + }, + }, + "client.a.request.size": { + name: "request_response_size", + labels: map[string]string{ + "client": "a", + "direction": "request", + }, + }, + "client.a.response.size": { + name: "request_response_size", + labels: map[string]string{ + "client": "a", + "direction": "response", + }, + }, + }, + }, + } + + mapper := MetricMapper{} + for i, scenario := range scenarios { + err := mapper.InitFromYAMLString(scenario.config) + if err != nil && !scenario.configBad { + t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err) + } + if err == nil && scenario.configBad { + t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config) + } + + var dummyMetricType MetricType = "" + for metric, mapping := range scenario.mappings { + m, labels, present := mapper.GetMapping(metric, dummyMetricType) + if present && mapping.name != "" && m.Name != mapping.name { + t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name) + } + if mapping.notPresent && present { + t.Fatalf("%d.%q: Expected metric to not be present", i, metric) + } + if len(labels) != len(mapping.labels) { + t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping.labels), len(labels)) + } + for label, value := range labels { + if mapping.labels[label] != value { + t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels) + } + } + + } + } +} + func TestAction(t *testing.T) { scenarios := []struct { config string diff --git a/pkg/mapper/match.go b/pkg/mapper/match.go index 12d5e8d..276b52a 100644 --- a/pkg/mapper/match.go +++ b/pkg/mapper/match.go @@ -20,6 +20,7 @@ type MatchType string const ( MatchTypeGlob MatchType = "glob" MatchTypeRegex MatchType = "regex" + MatchTypeFSM MatchType = "fsm" MatchTypeDefault MatchType = "" ) @@ -32,6 +33,8 @@ func (t *MatchType) UnmarshalYAML(unmarshal func(interface{}) error) error { switch MatchType(v) { case MatchTypeRegex: *t = MatchTypeRegex + case MatchTypeFSM: + *t = MatchTypeFSM case MatchTypeGlob, MatchTypeDefault: *t = MatchTypeGlob default: From bfe23298aa594fe37714d4ebfe9817d9ec524af0 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Tue, 11 Sep 2018 17:31:26 -0700 Subject: [PATCH 02/23] replace glob matching Signed-off-by: Wangchong Zhou --- main.go | 7 + pkg/mapper/mapper.go | 193 ++++++++++++++---------- pkg/mapper/mapper_test.go | 299 +++++--------------------------------- pkg/mapper/match.go | 3 - 4 files changed, 160 insertions(+), 342 deletions(-) diff --git a/main.go b/main.go index 5fc4881..481f9d2 100644 --- a/main.go +++ b/main.go @@ -126,6 +126,7 @@ func main() { statsdListenTCP = kingpin.Flag("statsd.listen-tcp", "The TCP address on which to receive statsd metric lines. \"\" disables it.").Default(":9125").String() mappingConfig = kingpin.Flag("statsd.mapping-config", "Metric mapping configuration file name.").String() readBuffer = kingpin.Flag("statsd.read-buffer", "Size (in bytes) of the operating system's transmit read buffer associated with the UDP connection. Please make sure the kernel parameters net.core.rmem_max is set to a value greater than the value specified.").Int() + dumpFSMPath = kingpin.Flag("statsd.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String() ) log.AddFlags(kingpin.CommandLine) @@ -178,6 +179,12 @@ func main() { } mapper := &mapper.MetricMapper{MappingsCount: mappingsCount} + if *dumpFSMPath != "" { + err := mapper.SetDumpFSMPath(*dumpFSMPath) + if err != nil { + log.Fatal("Error setting dump FSM path:", err) + } + } if *mappingConfig != "" { err := mapper.InitFromFile(*mappingConfig) if err != nil { diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index c37a783..ae1aa79 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -32,23 +32,23 @@ var ( statsdMetricRE = `[a-zA-Z_](-?[a-zA-Z0-9_])+` templateReplaceRE = `(\$\{?\d+\}?)` - metricLineRE = regexp.MustCompile(`^(\*\.|` + statsdMetricRE + `\.)+(\*|` + statsdMetricRE + `)$`) - metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`) - labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`) - labelValueExpansionRE = regexp.MustCompile(`\${?(\d+)}?`) + metricLineRE = regexp.MustCompile(`^(\*\.|` + statsdMetricRE + `\.)+(\*|` + statsdMetricRE + `)$`) + metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`) + labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`) + + templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`) ) type mapperConfigDefaults struct { - TimerType TimerType `yaml:"timer_type"` - Buckets []float64 `yaml:"buckets"` - Quantiles []metricObjective `yaml:"quantiles"` - MatchType MatchType `yaml:"match_type"` - DumpFSM string `yaml:"dump_fsm"` - FSMFallback MatchType `yaml:"fsm_fallback"` + TimerType TimerType `yaml:"timer_type"` + Buckets []float64 `yaml:"buckets"` + Quantiles []metricObjective `yaml:"quantiles"` + MatchType MatchType `yaml:"match_type"` } type mappingState struct { - transitions map[string]*mappingState + transitionsMap map[string]*mappingState + transitionsArray []*mappingState // result is nil unless there's a metric ends with this state result *MetricMapping } @@ -57,14 +57,18 @@ type MetricMapper struct { Defaults mapperConfigDefaults `yaml:"defaults"` Mappings []MetricMapping `yaml:"mappings"` FSM *mappingState - mutex sync.Mutex + // if this is true, that means at least one matching rule is regex type + doRegex bool + dumpFSMPath string + mutex sync.Mutex MappingsCount prometheus.Gauge } -type labelFormatter struct { - captureIdx int - fmtString string +type templateFormatter struct { + captureIndexes []int + captureCount int + fmtString string } type matchMetricType string @@ -72,9 +76,10 @@ type matchMetricType string type MetricMapping struct { Match string `yaml:"match"` Name string `yaml:"name"` + NameFormatter templateFormatter regex *regexp.Regexp Labels prometheus.Labels `yaml:"labels"` - LabelsFormatter map[string]labelFormatter + LabelsFormatter map[string]templateFormatter TimerType TimerType `yaml:"timer_type"` Buckets []float64 `yaml:"buckets"` Quantiles []metricObjective `yaml:"quantiles"` @@ -95,6 +100,48 @@ var defaultQuantiles = []metricObjective{ {Quantile: 0.99, Error: 0.001}, } +func generateFormatter(valueExpr string, captureCount int) (templateFormatter, error) { + matches := templateReplaceCaptureRE.FindAllStringSubmatch(valueExpr, -1) + if len(matches) == 0 { + // if no regex reference found, keep it as it is + return templateFormatter{captureCount: 0, fmtString: valueExpr}, nil + } + + var indexes []int + valueFormatter := valueExpr + for _, match := range matches { + idx, err := strconv.Atoi(match[len(match)-1]) + if err != nil || idx > captureCount || idx < 1 { + // if index larger than captured count or using unsupported named capture group, + // replace with empty string + valueFormatter = strings.Replace(valueFormatter, match[0], "", -1) + } else { + valueFormatter = strings.Replace(valueFormatter, match[0], "%s", -1) + // note: the regex reference variable $? starts from 1 + indexes = append(indexes, idx-1) + } + } + return templateFormatter{ + captureIndexes: indexes, + captureCount: len(indexes), + fmtString: valueFormatter, + }, nil +} + +func formatTemplate(formatter templateFormatter, captures map[int]string) string { + if formatter.captureCount == 0 { + // no label substitution, keep as it is + return formatter.fmtString + } else { + indexes := formatter.captureIndexes + vargs := make([]interface{}, formatter.captureCount) + for i, idx := range indexes { + vargs[i] = captures[idx] + } + return fmt.Sprintf(formatter.fmtString, vargs...) + } +} + func (m *MetricMapper) InitFromYAMLString(fileContents string) error { var n MetricMapper @@ -117,7 +164,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { maxPossibleTransitions := len(n.Mappings) n.FSM = &mappingState{} - n.FSM.transitions = make(map[string]*mappingState, maxPossibleTransitions) + n.FSM.transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) for i := range n.Mappings { maxPossibleTransitions-- @@ -147,21 +194,25 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { currentMapping.Action = ActionTypeMap } - if currentMapping.MatchType == MatchTypeFSM { + if currentMapping.MatchType == MatchTypeGlob { + if !metricLineRE.MatchString(currentMapping.Match) { + return fmt.Errorf("invalid match: %s", currentMapping.Match) + } + // first split by "." matchFields := strings.Split(currentMapping.Match, ".") // fill into our FSM root := n.FSM captureCount := 0 for i, field := range matchFields { - state, prs := root.transitions[field] + state, prs := root.transitionsMap[field] if !prs { state = &mappingState{} - (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) - root.transitions[field] = state + (*state).transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) + root.transitionsMap[field] = state // if this is last field, set result to currentMapping instance if i == len(matchFields)-1 { - root.transitions[field].result = currentMapping + root.transitionsMap[field].result = currentMapping } } if field == "*" { @@ -171,33 +222,29 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { // goto next state root = state } - currentLabelFormatter := make(map[string]labelFormatter, captureCount) + nameFmt, err := generateFormatter(currentMapping.Name, captureCount) + if err != nil { + return err + } + currentMapping.NameFormatter = nameFmt + + currentLabelFormatter := make(map[string]templateFormatter, captureCount) for label, valueExpr := range currentMapping.Labels { - matches := labelValueExpansionRE.FindAllStringSubmatch(valueExpr, -1) - if len(matches) == 0 { - // if no regex expansion found, keep it as it is - currentLabelFormatter[label] = labelFormatter{captureIdx: -1, fmtString: valueExpr} - continue - } else if len(matches) > 1 { - return fmt.Errorf("multiple captures is not supported in FSM matching type") - } - var valueFormatter string - idx, err := strconv.Atoi(matches[0][1]) + lblFmt, err := generateFormatter(valueExpr, captureCount) if err != nil { - return fmt.Errorf("invalid label value expression: %s", valueExpr) + return err } - if idx > captureCount || idx < 1 { - // index larger than captured count, replace all expansion with empty string - valueFormatter = labelValueExpansionRE.ReplaceAllString(valueExpr, "") - idx = 0 - } else { - valueFormatter = labelValueExpansionRE.ReplaceAllString(valueExpr, "%s") - } - currentLabelFormatter[label] = labelFormatter{captureIdx: idx - 1, fmtString: valueFormatter} + currentLabelFormatter[label] = lblFmt } currentMapping.LabelsFormatter = currentLabelFormatter - } - if currentMapping.MatchType == MatchTypeGlob || n.Defaults.FSMFallback == MatchTypeGlob { + } else { + if regex, err := regexp.Compile(currentMapping.Match); err != nil { + return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err) + } else { + currentMapping.regex = regex + } + m.doRegex = true + } /*else if currentMapping.MatchType == MatchTypeGlob { if !metricLineRE.MatchString(currentMapping.Match) { return fmt.Errorf("invalid match: %s", currentMapping.Match) } @@ -210,13 +257,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { } else { currentMapping.regex = regex } - } else if currentMapping.MatchType == MatchTypeRegex || n.Defaults.FSMFallback == MatchTypeRegex { - if regex, err := regexp.Compile(currentMapping.Match); err != nil { - return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err) - } else { - currentMapping.regex = regex - } - } + } */ if currentMapping.TimerType == "" { currentMapping.TimerType = n.Defaults.TimerType @@ -231,17 +272,17 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { } } - if len(n.Defaults.DumpFSM) > 0 { - m.dumpFSM(n.Defaults.DumpFSM, n.FSM) - } m.mutex.Lock() defer m.mutex.Unlock() m.Defaults = n.Defaults m.Mappings = n.Mappings - if len(n.FSM.transitions) > 0 { + if len(n.FSM.transitionsMap) > 0 || len(n.FSM.transitionsArray) > 0 { m.FSM = n.FSM + if m.dumpFSMPath != "" { + dumpFSM(m.dumpFSMPath, m.FSM) + } } if m.MappingsCount != nil { @@ -251,7 +292,12 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { return nil } -func (m *MetricMapper) dumpFSM(fileName string, root *mappingState) { +func (m *MetricMapper) SetDumpFSMPath(path string) error { + m.dumpFSMPath = path + return nil +} + +func dumpFSM(fileName string, root *mappingState) { log.Infoln("Start dumping FSM to", fileName) idx := 0 states := make(map[int]*mappingState) @@ -264,13 +310,12 @@ func (m *MetricMapper) dumpFSM(fileName string, root *mappingState) { w.WriteString("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n") // remove label of node for idx < len(states) { - for field, transition := range states[idx].transitions { + for field, transition := range states[idx].transitionsMap { states[len(states)] = transition w.WriteString(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)) - if transition.transitions == nil || len(transition.transitions) == 0 { + if transition.transitionsMap == nil || len(transition.transitionsMap) == 0 { w.WriteString(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)) } - } idx++ } @@ -289,18 +334,19 @@ func (m *MetricMapper) InitFromFile(fileName string) error { } func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) { + // glob matching if root := m.FSM; root != nil { matchFields := strings.Split(statsdMetric, ".") captures := make(map[int]string, len(matchFields)) captureIdx := 0 filedsCount := len(matchFields) for i, field := range matchFields { - if root.transitions == nil { + if root.transitionsMap == nil { break } - state, prs := root.transitions[field] + state, prs := root.transitionsMap[field] if !prs { - state, prs = root.transitions["*"] + state, prs = root.transitionsMap["*"] if !prs { break } @@ -308,36 +354,33 @@ func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricTy captureIdx++ } if state.result != nil && i == filedsCount-1 { - // format valueExpr mapping := *state.result + state.result.Name = formatTemplate(mapping.NameFormatter, captures) + labels := prometheus.Labels{} for label := range mapping.Labels { - formatter := mapping.LabelsFormatter[label] - idx := formatter.captureIdx - var value string - if idx == -1 { - value = formatter.fmtString - } else { - value = fmt.Sprintf(formatter.fmtString, captures[idx]) - } - labels[label] = string(value) + labels[label] = formatTemplate(mapping.LabelsFormatter[label], captures) } return state.result, labels, true } root = state } - // if fsm_fallback is not defined, return immediately - if len(m.Defaults.FSMFallback) == 0 { - log.Infof("%s not matched by fsm\n", statsdMetric) + // if there's no regex match type, return immediately + if !m.doRegex { return nil, nil, false } } + // regex matching m.mutex.Lock() defer m.mutex.Unlock() for _, mapping := range m.Mappings { + // if a rule don't have regex matching type, the regex field is unset + if mapping.regex == nil { + continue + } matches := mapping.regex.FindStringSubmatchIndex(statsdMetric) if len(matches) == 0 { continue diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index 0b74f1e..8f5d5e2 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -368,30 +368,6 @@ mappings: `, configBad: true, }, - //Config with multiple captures for fsm match type - { - config: `--- - mappings: - - match: "foo.*.*" - match_type: fsm - name: "foo" - labels: - bar: "$1-$2" - `, - configBad: true, - }, - //Config with non-numeric capture index for fsm match type - { - config: `--- - mappings: - - match: "foo.*.*" - match_type: fsm - name: "foo" - labels: - bar: "$a" - `, - configBad: true, - }, //Config with non-matched metric. { config: `--- @@ -542,7 +518,7 @@ mappings: } } -func TestFSMMatcher(t *testing.T) { +/*func TestRPS(t *testing.T) { scenarios := []struct { config string configBad bool @@ -553,12 +529,10 @@ func TestFSMMatcher(t *testing.T) { // Config with several mapping definitions. { config: `--- -defaults: - match_type: "fsm" mappings: - match: test.dispatcher.*.*.* name: "dispatch_events" - labels: + labels: processor: "$1" action: "$2" result: "$3" @@ -573,23 +547,23 @@ mappings: - match: request_time.*.*.*.*.*.*.*.*.*.*.*.* name: "tyk_http_request" labels: - method_and_path: "$1" - response_code: "$2" - apikey: "$3" - apiversion: "$4" - apiname: "$5" - apiid: "$6" - ipv4_t1: "$7" - ipv4_t2: "$8" - ipv4_t3: "$9" - ipv4_t4: "$10" - orgid: "$11" - oauthid: "$12" + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" - match: "*.*" name: "catchall" labels: first: "$1" - second: "second_label_$2" + second: "$2" third: "$3" job: "-" `, @@ -633,100 +607,7 @@ mappings: name: "catchall", labels: map[string]string{ "first": "foo", - "second": "second_label_bar", - "third": "", - "job": "-", - }, - }, - "foo.bar.baz": {}, - }, - }, - // local match_type - { - config: `--- -mappings: -- match: test.dispatcher.*.*.* - name: "dispatch_events" - labels: - match_type: "fsm" - processor: "$1" - action: "$2" - result: "$3" - job: "test_dispatcher" -- match: test.my-dispatch-host01.name.dispatcher.*.*.* - name: "host_dispatch_events" - labels: - match_type: "fsm" - processor: "$1" - action: "$2" - result: "$3" - job: "test_dispatcher" -- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* - name: "tyk_http_request" - labels: - match_type: "fsm" - method_and_path: "$1" - response_code: "$2" - apikey: "$3" - apiversion: "$4" - apiname: "$5" - apiid: "$6" - ipv4_t1: "$7" - ipv4_t2: "$8" - ipv4_t3: "$9" - ipv4_t4: "$10" - orgid: "$11" - oauthid: "$12" -- match: "*.*" - name: "catchall" - labels: - match_type: "fsm" - first: "$1" - second: "second_label_$2" - third: "$3" - job: "-" - `, - mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "host_dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "tyk_http_request", - labels: map[string]string{ - "method_and_path": "get/threads/1/posts", - "response_code": "200", - "apikey": "00000000", - "apiversion": "nonversioned", - "apiname": "discussions", - "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", - "ipv4_t1": "172", - "ipv4_t2": "20", - "ipv4_t3": "0", - "ipv4_t4": "1", - "orgid": "12ba97b7eaa1a50001000001", - "oauthid": "", - }, - }, - "foo.bar": { - name: "catchall", - labels: map[string]string{ - "first": "foo", - "second": "second_label_bar", + "second": "bar", "third": "", "job": "-", }, @@ -747,139 +628,29 @@ mappings: } var dummyMetricType MetricType = "" - for metric, mapping := range scenario.mappings { - m, labels, present := mapper.GetMapping(metric, dummyMetricType) - if present && mapping.name != "" && m.Name != mapping.name { - t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name) - } - if mapping.notPresent && present { - t.Fatalf("%d.%q: Expected metric to not be present", i, metric) - } - if len(labels) != len(mapping.labels) { - t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping.labels), len(labels)) - } - for label, value := range labels { - if mapping.labels[label] != value { - t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels) + start := int32(time.Now().Unix()) + for j := 1; j < 100000; j++ { + for metric, mapping := range scenario.mappings { + m, labels, present := mapper.GetMapping(metric, dummyMetricType) + if present && mapping.name != "" && m.Name != mapping.name { + t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name) + } + if mapping.notPresent && present { + t.Fatalf("%d.%q: Expected metric to not be present", i, metric) + } + if len(labels) != len(mapping.labels) { + t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping.labels), len(labels)) + } + for label, value := range labels { + if mapping.labels[label] != value { + t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels) + } } } - } + fmt.Println("finished in", int32(time.Now().Unix())-start) } -} - -func TestFSMMatcherFallbackRegex(t *testing.T) { - scenarios := []struct { - config string - configBad bool - mappings mappings - }{ - // Config with simple matcher as default mathcer and fallback_regex to false. - { - config: `--- -defaults: - match_type: "fsm" -mappings: -- match: client.*.request.duration - name: "request_size" - labels: - client: "$1" -- match: client.*.*.size - name: "request_response_size" - labels: - client: "$1" - direction: "$2" - `, - mappings: mappings{ - "client.a.request.duration": { - name: "request_size", - labels: map[string]string{ - "client": "a", - }, - }, - "client.a.request.size": {}, - "client.a.response.size": { - name: "request_response_size", - labels: map[string]string{ - "client": "a", - "direction": "response", - }, - }, - }, - }, - // Config with simple matcher as default mathcer and fallback_regex to true. - { - config: `--- -defaults: - match_type: "fsm" - fsm_fallback: "glob" -mappings: -- match: client.*.request.duration - name: "request_size" - labels: - client: "$1" -- match: client.*.*.size - name: "request_response_size" - labels: - client: "$1" - direction: "$2" - `, - mappings: mappings{ - "client.a.request.duration": { - name: "request_size", - labels: map[string]string{ - "client": "a", - }, - }, - "client.a.request.size": { - name: "request_response_size", - labels: map[string]string{ - "client": "a", - "direction": "request", - }, - }, - "client.a.response.size": { - name: "request_response_size", - labels: map[string]string{ - "client": "a", - "direction": "response", - }, - }, - }, - }, - } - - mapper := MetricMapper{} - for i, scenario := range scenarios { - err := mapper.InitFromYAMLString(scenario.config) - if err != nil && !scenario.configBad { - t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err) - } - if err == nil && scenario.configBad { - t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config) - } - - var dummyMetricType MetricType = "" - for metric, mapping := range scenario.mappings { - m, labels, present := mapper.GetMapping(metric, dummyMetricType) - if present && mapping.name != "" && m.Name != mapping.name { - t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name) - } - if mapping.notPresent && present { - t.Fatalf("%d.%q: Expected metric to not be present", i, metric) - } - if len(labels) != len(mapping.labels) { - t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping.labels), len(labels)) - } - for label, value := range labels { - if mapping.labels[label] != value { - t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels) - } - } - - } - } -} +}*/ func TestAction(t *testing.T) { scenarios := []struct { diff --git a/pkg/mapper/match.go b/pkg/mapper/match.go index 276b52a..12d5e8d 100644 --- a/pkg/mapper/match.go +++ b/pkg/mapper/match.go @@ -20,7 +20,6 @@ type MatchType string const ( MatchTypeGlob MatchType = "glob" MatchTypeRegex MatchType = "regex" - MatchTypeFSM MatchType = "fsm" MatchTypeDefault MatchType = "" ) @@ -33,8 +32,6 @@ func (t *MatchType) UnmarshalYAML(unmarshal func(interface{}) error) error { switch MatchType(v) { case MatchTypeRegex: *t = MatchTypeRegex - case MatchTypeFSM: - *t = MatchTypeFSM case MatchTypeGlob, MatchTypeDefault: *t = MatchTypeGlob default: From dad04e9c8b752b46f16f241b9a1fd7422d3b4fc1 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Wed, 12 Sep 2018 14:44:07 -0700 Subject: [PATCH 03/23] detect backtracking Signed-off-by: Wangchong Zhou --- pkg/mapper/mapper.go | 178 +++++++++++++++++++++++++++++++------- pkg/mapper/mapper_test.go | 25 ++++++ 2 files changed, 172 insertions(+), 31 deletions(-) diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index ae1aa79..8081267 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -40,24 +40,29 @@ var ( ) type mapperConfigDefaults struct { - TimerType TimerType `yaml:"timer_type"` - Buckets []float64 `yaml:"buckets"` - Quantiles []metricObjective `yaml:"quantiles"` - MatchType MatchType `yaml:"match_type"` + TimerType TimerType `yaml:"timer_type"` + Buckets []float64 `yaml:"buckets"` + Quantiles []metricObjective `yaml:"quantiles"` + MatchType MatchType `yaml:"match_type"` + GlobDisbleOrdering bool `yaml:"glob_disable_ordering"` } type mappingState struct { transitionsMap map[string]*mappingState transitionsArray []*mappingState + // use to compare length upfront to avoid unnecessary backtrack + minRemainingLength int + maxRemainingLength int // result is nil unless there's a metric ends with this state result *MetricMapping } type MetricMapper struct { - Defaults mapperConfigDefaults `yaml:"defaults"` - Mappings []MetricMapping `yaml:"mappings"` - FSM *mappingState - // if this is true, that means at least one matching rule is regex type + Defaults mapperConfigDefaults `yaml:"defaults"` + Mappings []MetricMapping `yaml:"mappings"` + FSM *mappingState + FSMNeedsBacktracking bool + // if doRegex is true, at least one matching rule is regex type doRegex bool dumpFSMPath string mutex sync.Mutex @@ -71,6 +76,11 @@ type templateFormatter struct { fmtString string } +type fsmBacktrackRecord struct { + fieldIndex int + state *mappingState +} + type matchMetricType string type MetricMapping struct { @@ -142,6 +152,20 @@ func formatTemplate(formatter templateFormatter, captures map[int]string) string } } +func min(x, y int) int { + if x < y { + return x + } + return y +} + +func max(x, y int) int { + if x > y { + return x + } + return y +} + func (m *MetricMapper) InitFromYAMLString(fileContents string) error { var n MetricMapper @@ -164,7 +188,12 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { maxPossibleTransitions := len(n.Mappings) n.FSM = &mappingState{} - n.FSM.transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) + n.FSM.transitionsMap = make(map[string]*mappingState, 3) + for _, field := range []MetricType{MetricTypeCounter, MetricTypeTimer, MetricTypeGauge, ""} { + state := &mappingState{} + (*state).transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) + n.FSM.transitionsMap[string(field)] = state + } for i := range n.Mappings { maxPossibleTransitions-- @@ -202,25 +231,40 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { // first split by "." matchFields := strings.Split(currentMapping.Match, ".") // fill into our FSM - root := n.FSM - captureCount := 0 - for i, field := range matchFields { - state, prs := root.transitionsMap[field] - if !prs { - state = &mappingState{} - (*state).transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) - root.transitionsMap[field] = state - // if this is last field, set result to currentMapping instance - if i == len(matchFields)-1 { - root.transitionsMap[field].result = currentMapping + roots := []*mappingState{} + if currentMapping.MatchMetricType == "" { + for _, metricType := range []MetricType{MetricTypeCounter, MetricTypeTimer, MetricTypeGauge, ""} { + roots = append(roots, n.FSM.transitionsMap[string(metricType)]) + } + } else { + roots = append(roots, n.FSM.transitionsMap[string(currentMapping.MatchMetricType)]) + } + var captureCount int + for _, root := range roots { + captureCount = 0 + for i, field := range matchFields { + state, prs := root.transitionsMap[field] + if !prs { + state = &mappingState{} + (*state).transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) + (*state).maxRemainingLength = len(matchFields) - i - 1 + (*state).minRemainingLength = len(matchFields) - i - 1 + root.transitionsMap[field] = state + // if this is last field, set result to currentMapping instance + if i == len(matchFields)-1 { + root.transitionsMap[field].result = currentMapping + } + } else { + (*state).maxRemainingLength = max(len(matchFields)-i-1, (*state).maxRemainingLength) + (*state).minRemainingLength = min(len(matchFields)-i-1, (*state).minRemainingLength) + } + if field == "*" { + captureCount++ } - } - if field == "*" { - captureCount++ - } - // goto next state - root = state + // goto next state + root = state + } } nameFmt, err := generateFormatter(currentMapping.Name, captureCount) if err != nil { @@ -243,7 +287,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { } else { currentMapping.regex = regex } - m.doRegex = true + n.doRegex = true } /*else if currentMapping.MatchType == MatchTypeGlob { if !metricLineRE.MatchString(currentMapping.Match) { return fmt.Errorf("invalid match: %s", currentMapping.Match) @@ -280,9 +324,22 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { m.Mappings = n.Mappings if len(n.FSM.transitionsMap) > 0 || len(n.FSM.transitionsArray) > 0 { m.FSM = n.FSM + m.doRegex = n.doRegex if m.dumpFSMPath != "" { dumpFSM(m.dumpFSMPath, m.FSM) } + + // backtracking only makes sense when we disbled ordering of rules + // where transistions are stored in (unordered) map + if m.Defaults.GlobDisbleOrdering || true { + backtrackingRules := findBacktrackRules(&n) + if len(backtrackingRules) > 0 { + for _, rule := range backtrackingRules { + log.Infof("backtracking required for match \"%s\", matching performance may be degraded\n", rule) + } + m.FSMNeedsBacktracking = true + } + } } if m.MappingsCount != nil { @@ -297,6 +354,58 @@ func (m *MetricMapper) SetDumpFSMPath(path string) error { return nil } +func findBacktrackRules(n *MetricMapper) []string { + var found []string + // rule A and B that has same length and + // A one has * in rules but is not a superset of B makes it needed for backtracking + ruleByLength := make(map[int][]string) + ruleREByLength := make(map[int][]*regexp.Regexp) + + // first sort rules by length + for _, mapping := range n.Mappings { + if mapping.MatchType != MatchTypeGlob { + continue + } + l := len(mapping.Match) + ruleByLength[l] = append(ruleByLength[l], mapping.Match) + + metricRe := strings.Replace(mapping.Match, ".", "\\.", -1) + metricRe = strings.Replace(metricRe, "*", "([^.]*)", -1) + regex, err := regexp.Compile("^" + metricRe + "$") + if err != nil { + log.Warnf("invalid match %s. cannot compile regex in mapping: %v", mapping.Match, err) + } + ruleREByLength[l] = append(ruleREByLength[l], regex) + } + + for l, rules := range ruleByLength { + if len(rules) == 1 { + continue + } + rulesRE := ruleREByLength[l] + // for each rule r1 in rules that has * inside, check if r1 is the superset of any rules + // if not then r1 is a rule that leads to backtrack + for i1, r1 := range rules { + hasSubset := false + re1 := rulesRE[i1] + if re1 == nil || strings.Index(r1, "*") == -1 { + continue + } + for _, r2 := range rules { + if r2 != r1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { + fmt.Println("subset", r1, "of", r2) + hasSubset = true + break + } + } + if !hasSubset { + found = append(found, r1) + } + } + } + return found +} + func dumpFSM(fileName string, root *mappingState) { log.Infoln("Start dumping FSM to", fileName) idx := 0 @@ -313,13 +422,18 @@ func dumpFSM(fileName string, root *mappingState) { for field, transition := range states[idx].transitionsMap { states[len(states)] = transition w.WriteString(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)) - if transition.transitionsMap == nil || len(transition.transitionsMap) == 0 { + if idx == 0 { + // color for metric types + w.WriteString(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1)) + } else if transition.transitionsMap == nil || len(transition.transitionsMap) == 0 { + // color for end state w.WriteString(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)) } } idx++ } - w.WriteString(fmt.Sprintf("0 [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n")) + // color for start state + w.WriteString(fmt.Sprintf("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];\n")) w.WriteString("}") w.Flush() log.Infoln("Finish dumping FSM") @@ -336,6 +450,7 @@ func (m *MetricMapper) InitFromFile(fileName string) error { func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) { // glob matching if root := m.FSM; root != nil { + root = root.transitionsMap[string(statsdMetricType)] matchFields := strings.Split(statsdMetric, ".") captures := make(map[int]string, len(matchFields)) captureIdx := 0 @@ -345,9 +460,10 @@ func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricTy break } state, prs := root.transitionsMap[field] - if !prs { + fieldsLeft := filedsCount - i - 1 + if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { state, prs = root.transitionsMap["*"] - if !prs { + if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { break } captures[captureIdx] = field diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index 8f5d5e2..e419fb1 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -471,6 +471,31 @@ mappings: }, }, }, + //Config with backtracking + { + config: `mappings: +- match: foo.*.ccc + name: "fooc" + labels: {} +- match: foo.bbb.aaa + name: "foob" + labels: {} + `, + mappings: mappings{ + "foo.bbb.ccc": { + name: "fooc", + labels: map[string]string{}, + }, + "foo.ddd.ccc": { + name: "fooc", + labels: map[string]string{}, + }, + "foo.bbb.aaa": { + name: "foob", + labels: map[string]string{}, + }, + }, + }, } mapper := MetricMapper{} From c2742aa299300a9583843cef6fbb641e1d7385b0 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Wed, 12 Sep 2018 18:18:12 -0700 Subject: [PATCH 04/23] implement backtrack Signed-off-by: Wangchong Zhou --- pkg/mapper/mapper.go | 134 ++++++++++++++++++++++++-------------- pkg/mapper/mapper_test.go | 75 ++++++++++++++------- 2 files changed, 136 insertions(+), 73 deletions(-) diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 8081267..7fb5ff8 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -48,9 +48,7 @@ type mapperConfigDefaults struct { } type mappingState struct { - transitionsMap map[string]*mappingState - transitionsArray []*mappingState - // use to compare length upfront to avoid unnecessary backtrack + transitions map[string]*mappingState minRemainingLength int maxRemainingLength int // result is nil unless there's a metric ends with this state @@ -76,9 +74,13 @@ type templateFormatter struct { fmtString string } -type fsmBacktrackRecord struct { - fieldIndex int - state *mappingState +type fsmBacktrackStackCursor struct { + fieldIndex int + captureIdx int + currentCapture string + state *mappingState + prev *fsmBacktrackStackCursor + next *fsmBacktrackStackCursor } type matchMetricType string @@ -188,11 +190,11 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { maxPossibleTransitions := len(n.Mappings) n.FSM = &mappingState{} - n.FSM.transitionsMap = make(map[string]*mappingState, 3) + n.FSM.transitions = make(map[string]*mappingState, 3) for _, field := range []MetricType{MetricTypeCounter, MetricTypeTimer, MetricTypeGauge, ""} { state := &mappingState{} - (*state).transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) - n.FSM.transitionsMap[string(field)] = state + (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) + n.FSM.transitions[string(field)] = state } for i := range n.Mappings { @@ -234,25 +236,25 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { roots := []*mappingState{} if currentMapping.MatchMetricType == "" { for _, metricType := range []MetricType{MetricTypeCounter, MetricTypeTimer, MetricTypeGauge, ""} { - roots = append(roots, n.FSM.transitionsMap[string(metricType)]) + roots = append(roots, n.FSM.transitions[string(metricType)]) } } else { - roots = append(roots, n.FSM.transitionsMap[string(currentMapping.MatchMetricType)]) + roots = append(roots, n.FSM.transitions[string(currentMapping.MatchMetricType)]) } var captureCount int for _, root := range roots { captureCount = 0 for i, field := range matchFields { - state, prs := root.transitionsMap[field] + state, prs := root.transitions[field] if !prs { state = &mappingState{} - (*state).transitionsMap = make(map[string]*mappingState, maxPossibleTransitions) + (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) (*state).maxRemainingLength = len(matchFields) - i - 1 (*state).minRemainingLength = len(matchFields) - i - 1 - root.transitionsMap[field] = state + root.transitions[field] = state // if this is last field, set result to currentMapping instance if i == len(matchFields)-1 { - root.transitionsMap[field].result = currentMapping + root.transitions[field].result = currentMapping } } else { (*state).maxRemainingLength = max(len(matchFields)-i-1, (*state).maxRemainingLength) @@ -322,7 +324,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { m.Defaults = n.Defaults m.Mappings = n.Mappings - if len(n.FSM.transitionsMap) > 0 || len(n.FSM.transitionsArray) > 0 { + if len(n.FSM.transitions) > 0 { m.FSM = n.FSM m.doRegex = n.doRegex if m.dumpFSMPath != "" { @@ -335,7 +337,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { backtrackingRules := findBacktrackRules(&n) if len(backtrackingRules) > 0 { for _, rule := range backtrackingRules { - log.Infof("backtracking required for match \"%s\", matching performance may be degraded\n", rule) + log.Warnf("backtracking required because of match \"%s\", matching performance may be degraded\n", rule) } m.FSMNeedsBacktracking = true } @@ -366,7 +368,7 @@ func findBacktrackRules(n *MetricMapper) []string { if mapping.MatchType != MatchTypeGlob { continue } - l := len(mapping.Match) + l := len(strings.Split(mapping.Match, ".")) ruleByLength[l] = append(ruleByLength[l], mapping.Match) metricRe := strings.Replace(mapping.Match, ".", "\\.", -1) @@ -375,6 +377,7 @@ func findBacktrackRules(n *MetricMapper) []string { if err != nil { log.Warnf("invalid match %s. cannot compile regex in mapping: %v", mapping.Match, err) } + // put into array no matter there's error or not, we will skip later if regex is nil ruleREByLength[l] = append(ruleREByLength[l], regex) } @@ -393,9 +396,8 @@ func findBacktrackRules(n *MetricMapper) []string { } for _, r2 := range rules { if r2 != r1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { - fmt.Println("subset", r1, "of", r2) + log.Warnf("rule \"%s\" is a super set of rule \"%s\", the later will never be matched\n", r1, r2) hasSubset = true - break } } if !hasSubset { @@ -419,13 +421,13 @@ func dumpFSM(fileName string, root *mappingState) { w.WriteString("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n") // remove label of node for idx < len(states) { - for field, transition := range states[idx].transitionsMap { + for field, transition := range states[idx].transitions { states[len(states)] = transition w.WriteString(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)) if idx == 0 { // color for metric types w.WriteString(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1)) - } else if transition.transitionsMap == nil || len(transition.transitionsMap) == 0 { + } else if transition.transitions == nil || len(transition.transitions) == 0 { // color for end state w.WriteString(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)) } @@ -450,41 +452,77 @@ func (m *MetricMapper) InitFromFile(fileName string) error { func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) { // glob matching if root := m.FSM; root != nil { - root = root.transitionsMap[string(statsdMetricType)] + root = root.transitions[string(statsdMetricType)] matchFields := strings.Split(statsdMetric, ".") captures := make(map[int]string, len(matchFields)) captureIdx := 0 + var backtrackCursor *fsmBacktrackStackCursor + backtrackCursor = nil filedsCount := len(matchFields) - for i, field := range matchFields { - if root.transitionsMap == nil { - break - } - state, prs := root.transitionsMap[field] - fieldsLeft := filedsCount - i - 1 - if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { - state, prs = root.transitionsMap["*"] - if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + i := 0 + for { + for i < filedsCount { + if root.transitions == nil { break } - captures[captureIdx] = field - captureIdx++ - } - if state.result != nil && i == filedsCount-1 { - mapping := *state.result - state.result.Name = formatTemplate(mapping.NameFormatter, captures) + field := matchFields[i] + state, prs := root.transitions[field] + fieldsLeft := filedsCount - i - 1 + // also compare length upfront to avoid unnecessary loop or backtrack + if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + state, prs = root.transitions["*"] + if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + break + } else { + captures[captureIdx] = field + captureIdx++ + } + } else if m.FSMNeedsBacktracking { + otherState, prs := root.transitions["*"] + if !prs || fieldsLeft > otherState.maxRemainingLength || fieldsLeft < otherState.minRemainingLength { + } else { + newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: otherState, + fieldIndex: i + 1, + captureIdx: captureIdx + 1, currentCapture: field, + } + if backtrackCursor != nil { + backtrackCursor.next = &newCursor + } + backtrackCursor = &newCursor + } - labels := prometheus.Labels{} - for label := range mapping.Labels { - labels[label] = formatTemplate(mapping.LabelsFormatter[label], captures) } - return state.result, labels, true - } - root = state - } + // found! + if state != nil && state.result != nil && i == filedsCount-1 { + mapping := *state.result + state.result.Name = formatTemplate(mapping.NameFormatter, captures) - // if there's no regex match type, return immediately - if !m.doRegex { - return nil, nil, false + labels := prometheus.Labels{} + for label := range mapping.Labels { + labels[label] = formatTemplate(mapping.LabelsFormatter[label], captures) + } + return state.result, labels, true + } + root = state + i++ + } + // if we are not doing backtracking or all path has been travesaled + if backtrackCursor == nil { + // if there's no regex match type, return immediately + if !m.doRegex { + return nil, nil, false + } else { + break + } + } else { + // pop one from stack + root = backtrackCursor.state + i = backtrackCursor.fieldIndex + captureIdx = backtrackCursor.captureIdx + // put the * capture back + captures[captureIdx-1] = backtrackCursor.currentCapture + backtrackCursor = backtrackCursor.prev + } } } diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index e419fb1..b150ece 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -139,6 +139,56 @@ mappings: }, }, }, + //Config with backtracking + { + config: ` +mappings: +- match: test.*.bbb + name: "testb" + labels: + label: "${1}_foo" +- match: test.justatest.aaa + name: "testa" + labels: + label: "${1}_foo" + `, + mappings: mappings{ + "test.good.bbb": { + name: "testb", + labels: map[string]string{ + "label": "good_foo", + }, + }, + "test.justatest.bbb": { + name: "testb", + labels: map[string]string{ + "label": "justatest_foo", + }, + }, + }, + }, + //Config with super sets + { + config: ` +mappings: +- match: test.*.bbb + name: "testb" + labels: + label: "${1}_foo" +- match: test.*.* + name: "testa" + labels: + label: "${1}_foo" + `, + mappings: mappings{ + "test.good.bbb": { + name: "testb", + labels: map[string]string{ + "label": "good_foo", + }, + }, + }, + }, // Config with bad regex reference. { config: `--- @@ -471,31 +521,6 @@ mappings: }, }, }, - //Config with backtracking - { - config: `mappings: -- match: foo.*.ccc - name: "fooc" - labels: {} -- match: foo.bbb.aaa - name: "foob" - labels: {} - `, - mappings: mappings{ - "foo.bbb.ccc": { - name: "fooc", - labels: map[string]string{}, - }, - "foo.ddd.ccc": { - name: "fooc", - labels: map[string]string{}, - }, - "foo.bbb.aaa": { - name: "foob", - labels: map[string]string{}, - }, - }, - }, } mapper := MetricMapper{} From 9ebab25dfa9361a7efbbdef9938f984ac272a13d Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 13 Sep 2018 11:46:41 -0700 Subject: [PATCH 05/23] ordering Signed-off-by: Wangchong Zhou --- pkg/mapper/mapper.go | 218 +++++++++++++++++++++++--------------- pkg/mapper/mapper_test.go | 88 +++++++++------ 2 files changed, 187 insertions(+), 119 deletions(-) diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 7fb5ff8..e003720 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -59,6 +59,7 @@ type MetricMapper struct { Defaults mapperConfigDefaults `yaml:"defaults"` Mappings []MetricMapping `yaml:"mappings"` FSM *mappingState + hasFSM bool FSMNeedsBacktracking bool // if doRegex is true, at least one matching rule is regex type doRegex bool @@ -76,7 +77,7 @@ type templateFormatter struct { type fsmBacktrackStackCursor struct { fieldIndex int - captureIdx int + captureIndex int currentCapture string state *mappingState prev *fsmBacktrackStackCursor @@ -99,6 +100,7 @@ type MetricMapping struct { HelpText string `yaml:"help"` Action ActionType `yaml:"action"` MatchMetricType MetricType `yaml:"match_metric_type"` + priority int } type metricObjective struct { @@ -225,7 +227,10 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { currentMapping.Action = ActionTypeMap } + currentMapping.priority = i + if currentMapping.MatchType == MatchTypeGlob { + n.hasFSM = true if !metricLineRE.MatchString(currentMapping.Match) { return fmt.Errorf("invalid match: %s", currentMapping.Match) } @@ -235,6 +240,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { // fill into our FSM roots := []*mappingState{} if currentMapping.MatchMetricType == "" { + // if metricType not specified, connect the state from all three types for _, metricType := range []MetricType{MetricTypeCounter, MetricTypeTimer, MetricTypeGauge, ""} { roots = append(roots, n.FSM.transitions[string(metricType)]) } @@ -290,20 +296,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { currentMapping.regex = regex } n.doRegex = true - } /*else if currentMapping.MatchType == MatchTypeGlob { - if !metricLineRE.MatchString(currentMapping.Match) { - return fmt.Errorf("invalid match: %s", currentMapping.Match) - } - // Translate the glob-style metric match line into a proper regex that we - // can use to match metrics later on. - metricRe := strings.Replace(currentMapping.Match, ".", "\\.", -1) - metricRe = strings.Replace(metricRe, "*", "([^.]*)", -1) - if regex, err := regexp.Compile("^" + metricRe + "$"); err != nil { - return fmt.Errorf("invalid match %s. cannot compile regex in mapping: %v", currentMapping.Match, err) - } else { - currentMapping.regex = regex - } - } */ + } if currentMapping.TimerType == "" { currentMapping.TimerType = n.Defaults.TimerType @@ -324,24 +317,15 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { m.Defaults = n.Defaults m.Mappings = n.Mappings - if len(n.FSM.transitions) > 0 { + if n.hasFSM { + m.hasFSM = n.hasFSM m.FSM = n.FSM m.doRegex = n.doRegex if m.dumpFSMPath != "" { dumpFSM(m.dumpFSMPath, m.FSM) } - // backtracking only makes sense when we disbled ordering of rules - // where transistions are stored in (unordered) map - if m.Defaults.GlobDisbleOrdering || true { - backtrackingRules := findBacktrackRules(&n) - if len(backtrackingRules) > 0 { - for _, rule := range backtrackingRules { - log.Warnf("backtracking required because of match \"%s\", matching performance may be degraded\n", rule) - } - m.FSMNeedsBacktracking = true - } - } + m.FSMNeedsBacktracking = needBacktracking(&n) } if m.MappingsCount != nil { @@ -356,12 +340,13 @@ func (m *MetricMapper) SetDumpFSMPath(path string) error { return nil } -func findBacktrackRules(n *MetricMapper) []string { - var found []string +func needBacktracking(n *MetricMapper) bool { + needBacktrack := false // rule A and B that has same length and // A one has * in rules but is not a superset of B makes it needed for backtracking ruleByLength := make(map[int][]string) ruleREByLength := make(map[int][]*regexp.Regexp) + rulesOrderByLength := make(map[int][]int) // first sort rules by length for _, mapping := range n.Mappings { @@ -379,6 +364,7 @@ func findBacktrackRules(n *MetricMapper) []string { } // put into array no matter there's error or not, we will skip later if regex is nil ruleREByLength[l] = append(ruleREByLength[l], regex) + rulesOrderByLength[l] = append(rulesOrderByLength[l], mapping.priority) } for l, rules := range ruleByLength { @@ -386,26 +372,49 @@ func findBacktrackRules(n *MetricMapper) []string { continue } rulesRE := ruleREByLength[l] + rulesOrder := rulesOrderByLength[l] // for each rule r1 in rules that has * inside, check if r1 is the superset of any rules // if not then r1 is a rule that leads to backtrack for i1, r1 := range rules { - hasSubset := false + currentRuleNeedBacktrack := true re1 := rulesRE[i1] if re1 == nil || strings.Index(r1, "*") == -1 { continue } - for _, r2 := range rules { - if r2 != r1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { - log.Warnf("rule \"%s\" is a super set of rule \"%s\", the later will never be matched\n", r1, r2) - hasSubset = true + for i2, r2 := range rules { + if i2 != i1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { + // log if we care about ordering and the superset occurs before + if !n.Defaults.GlobDisbleOrdering && rulesOrder[i1] < rulesOrder[i2] { + log.Warnf("match \"%s\" is a super set of match \"%s\" but in a lower order, "+ + "the first will never be matched\n", r1, r2) + } + currentRuleNeedBacktrack = false } } - if !hasSubset { - found = append(found, r1) + for i2, re2 := range rulesRE { + // especially, if r1 is a subset of other rule, we don't need backtrack + // because either we turned on ordering + // or we disabled ordering and can't match it with backtrack + if i2 != i1 && re2 != nil && len(re2.FindStringSubmatchIndex(r1)) > 0 { + currentRuleNeedBacktrack = false + } + } + if currentRuleNeedBacktrack { + log.Warnf("backtracking required because of match \"%s\", "+ + "matching performance may be degraded\n", r1) + needBacktrack = true } } } - return found + if !n.Defaults.GlobDisbleOrdering { + // backtracking only makes sense when we disbled ordering of rules + // where transistions are stored in (unordered) map + // note: don't move this branch to the beginning of this function + // since we need logs for superset rules + return true + } else { + return needBacktrack + } } func dumpFSM(fileName string, root *mappingState) { @@ -451,79 +460,112 @@ func (m *MetricMapper) InitFromFile(fileName string) error { func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) { // glob matching - if root := m.FSM; root != nil { - root = root.transitions[string(statsdMetricType)] + if m.hasFSM { matchFields := strings.Split(statsdMetric, ".") + + root := m.FSM.transitions[string(statsdMetricType)] captures := make(map[int]string, len(matchFields)) captureIdx := 0 var backtrackCursor *fsmBacktrackStackCursor backtrackCursor = nil + resumeFromBacktrack := false + var result *MetricMapping filedsCount := len(matchFields) i := 0 + var state *mappingState for { - for i < filedsCount { - if root.transitions == nil { - break - } - field := matchFields[i] - state, prs := root.transitions[field] - fieldsLeft := filedsCount - i - 1 - // also compare length upfront to avoid unnecessary loop or backtrack - if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { - state, prs = root.transitions["*"] - if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + for { + var prs bool + if !resumeFromBacktrack { + if len(root.transitions) > 0 { + field := matchFields[i] + state, prs = root.transitions[field] + fieldsLeft := filedsCount - i - 1 + // also compare length upfront to avoid unnecessary loop or backtrack + if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + state, prs = root.transitions["*"] + if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + break + } else { + captures[captureIdx] = field + captureIdx++ + } + } else if m.FSMNeedsBacktracking { + altState, prs := root.transitions["*"] + if !prs || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength { + } else { + // push to stack + newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: altState, + fieldIndex: i, + captureIndex: captureIdx, currentCapture: field, + } + if backtrackCursor != nil { + backtrackCursor.next = &newCursor + } + backtrackCursor = &newCursor + } + } + } else { // root.transitions == nil break - } else { - captures[captureIdx] = field - captureIdx++ - } - } else if m.FSMNeedsBacktracking { - otherState, prs := root.transitions["*"] - if !prs || fieldsLeft > otherState.maxRemainingLength || fieldsLeft < otherState.minRemainingLength { - } else { - newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: otherState, - fieldIndex: i + 1, - captureIdx: captureIdx + 1, currentCapture: field, - } - if backtrackCursor != nil { - backtrackCursor.next = &newCursor - } - backtrackCursor = &newCursor } + } // backtrack will resume from here - } - // found! - if state != nil && state.result != nil && i == filedsCount-1 { - mapping := *state.result - state.result.Name = formatTemplate(mapping.NameFormatter, captures) - - labels := prometheus.Labels{} - for label := range mapping.Labels { - labels[label] = formatTemplate(mapping.LabelsFormatter[label], captures) + // do we reach a final state? + if state.result != nil && i == filedsCount-1 { + if m.Defaults.GlobDisbleOrdering { + result = state.result + // do a double break + goto formatLabels + } else if result == nil || result.priority > state.result.priority { + // if we care about ordering, try to find a result with highest prioity + result = state.result } - return state.result, labels, true - } - root = state - i++ - } - // if we are not doing backtracking or all path has been travesaled - if backtrackCursor == nil { - // if there's no regex match type, return immediately - if !m.doRegex { - return nil, nil, false - } else { break } + + i++ + if i >= filedsCount { + break + } + + resumeFromBacktrack = false + root = state + } + if backtrackCursor == nil { + // if we are not doing backtracking or all path has been travesaled + break } else { // pop one from stack - root = backtrackCursor.state + state = backtrackCursor.state + root = state i = backtrackCursor.fieldIndex - captureIdx = backtrackCursor.captureIdx + captureIdx = backtrackCursor.captureIndex + 1 // put the * capture back captures[captureIdx-1] = backtrackCursor.currentCapture backtrackCursor = backtrackCursor.prev + if backtrackCursor != nil { + // deref for GC + backtrackCursor.next = nil + } + resumeFromBacktrack = true } } + + formatLabels: + // format name and labels + if result != nil { + result.Name = formatTemplate(result.NameFormatter, captures) + + labels := prometheus.Labels{} + for label := range result.Labels { + labels[label] = formatTemplate(result.LabelsFormatter[label], captures) + } + return result, labels, true + } else if !m.doRegex { + // if there's no regex match type, return immediately + return nil, nil, false + } + } // regex matching diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index b150ece..1d5f9cd 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -14,7 +14,9 @@ package mapper import ( + "fmt" "testing" + "time" ) type mappings map[string]struct { @@ -142,24 +144,26 @@ mappings: //Config with backtracking { config: ` +defaults: + glob_disable_ordering: true mappings: -- match: test.*.bbb +- match: backtrack.*.bbb name: "testb" labels: label: "${1}_foo" -- match: test.justatest.aaa +- match: backtrack.justatest.aaa name: "testa" labels: label: "${1}_foo" `, mappings: mappings{ - "test.good.bbb": { + "backtrack.good.bbb": { name: "testb", labels: map[string]string{ "label": "good_foo", }, }, - "test.justatest.bbb": { + "backtrack.justatest.bbb": { name: "testb", labels: map[string]string{ "label": "justatest_foo", @@ -167,22 +171,58 @@ mappings: }, }, }, - //Config with super sets + //Config with super sets, disables ordering { config: ` +defaults: + glob_disable_ordering: true mappings: -- match: test.*.bbb +- match: noorder.*.* + name: "testa" + labels: + label: "${1}_foo" +- match: noorder.*.bbb name: "testb" labels: label: "${1}_foo" -- match: test.*.* +- match: noorder.ccc.bbb + name: "testc" + labels: + label: "ccc_foo" + `, + mappings: mappings{ + "noorder.good.bbb": { + name: "testb", + labels: map[string]string{ + "label": "good_foo", + }, + }, + "noorder.ccc.bbb": { + name: "testc", + labels: map[string]string{ + "label": "ccc_foo", + }, + }, + }, + }, + //Config with super sets, keeps ordering + { + config: ` +defaults: + glob_disable_ordering: false +mappings: +- match: order.*.* name: "testa" labels: label: "${1}_foo" +- match: order.*.bbb + name: "testb" + labels: + label: "${1}_foo" `, mappings: mappings{ - "test.good.bbb": { - name: "testb", + "order.good.bbb": { + name: "testa", labels: map[string]string{ "label": "good_foo", }, @@ -568,17 +608,17 @@ mappings: } } -/*func TestRPS(t *testing.T) { +func TestRPS(t *testing.T) { scenarios := []struct { config string configBad bool mappings mappings }{ - // Empty config. - {}, // Config with several mapping definitions. { config: `--- +defaults: + glob_disable_ordering: true mappings: - match: test.dispatcher.*.*.* name: "dispatch_events" @@ -678,29 +718,15 @@ mappings: } var dummyMetricType MetricType = "" - start := int32(time.Now().Unix()) + start := time.Now() for j := 1; j < 100000; j++ { - for metric, mapping := range scenario.mappings { - m, labels, present := mapper.GetMapping(metric, dummyMetricType) - if present && mapping.name != "" && m.Name != mapping.name { - t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name) - } - if mapping.notPresent && present { - t.Fatalf("%d.%q: Expected metric to not be present", i, metric) - } - if len(labels) != len(mapping.labels) { - t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping.labels), len(labels)) - } - for label, value := range labels { - if mapping.labels[label] != value { - t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels) - } - } + for metric, _ := range scenario.mappings { + mapper.GetMapping(metric, dummyMetricType) } } - fmt.Println("finished in", int32(time.Now().Unix())-start) + fmt.Println("finished 100000 iterations in", time.Now().Sub(start)) } -}*/ +} func TestAction(t *testing.T) { scenarios := []struct { From a751c0c091e6b6ec71bf09fe51cd360254edda53 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Thu, 13 Sep 2018 12:23:59 -0700 Subject: [PATCH 06/23] cleanup Signed-off-by: Wangchong Zhou --- pkg/mapper/mapper.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index e003720..63926e8 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -114,11 +114,11 @@ var defaultQuantiles = []metricObjective{ {Quantile: 0.99, Error: 0.001}, } -func generateFormatter(valueExpr string, captureCount int) (templateFormatter, error) { +func generateFormatter(valueExpr string, captureCount int) templateFormatter { matches := templateReplaceCaptureRE.FindAllStringSubmatch(valueExpr, -1) if len(matches) == 0 { // if no regex reference found, keep it as it is - return templateFormatter{captureCount: 0, fmtString: valueExpr}, nil + return templateFormatter{captureCount: 0, fmtString: valueExpr} } var indexes []int @@ -139,7 +139,7 @@ func generateFormatter(valueExpr string, captureCount int) (templateFormatter, e captureIndexes: indexes, captureCount: len(indexes), fmtString: valueFormatter, - }, nil + } } func formatTemplate(formatter templateFormatter, captures map[int]string) string { @@ -274,18 +274,13 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { root = state } } - nameFmt, err := generateFormatter(currentMapping.Name, captureCount) - if err != nil { - return err - } + nameFmt := generateFormatter(currentMapping.Name, captureCount) + currentMapping.NameFormatter = nameFmt currentLabelFormatter := make(map[string]templateFormatter, captureCount) for label, valueExpr := range currentMapping.Labels { - lblFmt, err := generateFormatter(valueExpr, captureCount) - if err != nil { - return err - } + lblFmt := generateFormatter(valueExpr, captureCount) currentLabelFormatter[label] = lblFmt } currentMapping.LabelsFormatter = currentLabelFormatter @@ -317,8 +312,8 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { m.Defaults = n.Defaults m.Mappings = n.Mappings + m.hasFSM = n.hasFSM if n.hasFSM { - m.hasFSM = n.hasFSM m.FSM = n.FSM m.doRegex = n.doRegex if m.dumpFSMPath != "" { @@ -394,7 +389,7 @@ func needBacktracking(n *MetricMapper) bool { for i2, re2 := range rulesRE { // especially, if r1 is a subset of other rule, we don't need backtrack // because either we turned on ordering - // or we disabled ordering and can't match it with backtrack + // or we disabled ordering and can't match it even with backtrack if i2 != i1 && re2 != nil && len(re2.FindStringSubmatchIndex(r1)) > 0 { currentRuleNeedBacktrack = false } @@ -462,12 +457,10 @@ func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricTy // glob matching if m.hasFSM { matchFields := strings.Split(statsdMetric, ".") - root := m.FSM.transitions[string(statsdMetricType)] captures := make(map[int]string, len(matchFields)) captureIdx := 0 var backtrackCursor *fsmBacktrackStackCursor - backtrackCursor = nil resumeFromBacktrack := false var result *MetricMapping filedsCount := len(matchFields) @@ -505,7 +498,7 @@ func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricTy backtrackCursor = &newCursor } } - } else { // root.transitions == nil + } else { // no more transitions for this state break } } // backtrack will resume from here From e6349977914b6b14fcf752934bf0e1813728fa7c Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Fri, 21 Sep 2018 17:59:11 -0700 Subject: [PATCH 07/23] move fsm to sepeare pkg Signed-off-by: Wangchong Zhou --- main.go | 2 +- pkg/mapper/fsm/formatter.go | 55 +++++ pkg/mapper/fsm/fsm.go | 344 ++++++++++++++++++++++++++++++ pkg/mapper/fsm/minmax.go | 17 ++ pkg/mapper/mapper.go | 405 ++++-------------------------------- 5 files changed, 452 insertions(+), 371 deletions(-) create mode 100644 pkg/mapper/fsm/formatter.go create mode 100644 pkg/mapper/fsm/fsm.go create mode 100644 pkg/mapper/fsm/minmax.go diff --git a/main.go b/main.go index 481f9d2..5d13480 100644 --- a/main.go +++ b/main.go @@ -180,7 +180,7 @@ func main() { mapper := &mapper.MetricMapper{MappingsCount: mappingsCount} if *dumpFSMPath != "" { - err := mapper.SetDumpFSMPath(*dumpFSMPath) + err := mapper.FSM.SetDumpFSMPath(*dumpFSMPath) if err != nil { log.Fatal("Error setting dump FSM path:", err) } diff --git a/pkg/mapper/fsm/formatter.go b/pkg/mapper/fsm/formatter.go new file mode 100644 index 0000000..591f9a8 --- /dev/null +++ b/pkg/mapper/fsm/formatter.go @@ -0,0 +1,55 @@ +package fsm + +import ( + "fmt" + "strconv" + "strings" +) + +type templateFormatter struct { + captureIndexes []int + captureCount int + fmtString string +} + +func newTemplateFormatter(valueExpr string, captureCount int) *templateFormatter { + matches := templateReplaceCaptureRE.FindAllStringSubmatch(valueExpr, -1) + if len(matches) == 0 { + // if no regex reference found, keep it as it is + return &templateFormatter{captureCount: 0, fmtString: valueExpr} + } + + var indexes []int + valueFormatter := valueExpr + for _, match := range matches { + idx, err := strconv.Atoi(match[len(match)-1]) + if err != nil || idx > captureCount || idx < 1 { + // if index larger than captured count or using unsupported named capture group, + // replace with empty string + valueFormatter = strings.Replace(valueFormatter, match[0], "", -1) + } else { + valueFormatter = strings.Replace(valueFormatter, match[0], "%s", -1) + // note: the regex reference variable $? starts from 1 + indexes = append(indexes, idx-1) + } + } + return &templateFormatter{ + captureIndexes: indexes, + captureCount: len(indexes), + fmtString: valueFormatter, + } +} + +func (formatter *templateFormatter) format(captures map[int]string) string { + if formatter.captureCount == 0 { + // no label substitution, keep as it is + return formatter.fmtString + } else { + indexes := formatter.captureIndexes + vargs := make([]interface{}, formatter.captureCount) + for i, idx := range indexes { + vargs[i] = captures[idx] + } + return fmt.Sprintf(formatter.fmtString, vargs...) + } +} diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go new file mode 100644 index 0000000..79240f0 --- /dev/null +++ b/pkg/mapper/fsm/fsm.go @@ -0,0 +1,344 @@ +package fsm + +import ( + "bufio" + "fmt" + "os" + "regexp" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/log" +) + +var ( + templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`) +) + +type mappingState struct { + transitions map[string]*mappingState + minRemainingLength int + maxRemainingLength int + // result* members are nil unless there's a metric ends with this state + result interface{} + resultPriority int + resultNameFormatter *templateFormatter + resultLabelsFormatter map[string]*templateFormatter +} + +type fsmBacktrackStackCursor struct { + fieldIndex int + captureIndex int + currentCapture string + state *mappingState + prev *fsmBacktrackStackCursor + next *fsmBacktrackStackCursor +} + +type FSM struct { + root *mappingState + needsBacktracking bool + dumpFSMPath string + metricTypes []string + disableOrdering bool + statesCount int +} + +func (fsm *FSM) SetDumpFSMPath(path string) error { + fsm.dumpFSMPath = path + return nil +} + +func NewFSM(metricTypes []string, maxPossibleTransitions int, disableOrdering bool) *FSM { + fsm := FSM{} + root := &mappingState{} + root.transitions = make(map[string]*mappingState, 3) + + metricTypes = append(metricTypes, "") + for _, field := range metricTypes { + state := &mappingState{} + (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) + root.transitions[string(field)] = state + } + fsm.disableOrdering = disableOrdering + fsm.metricTypes = metricTypes + fsm.statesCount = 0 + fsm.root = root + return &fsm +} + +func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matchMetricType string, + maxPossibleTransitions int, result interface{}) { + // first split by "." + matchFields := strings.Split(match, ".") + // fill into our FSM + roots := []*mappingState{} + if matchMetricType == "" { + // if metricType not specified, connect the state from all three types + for _, metricType := range f.metricTypes { + roots = append(roots, f.root.transitions[string(metricType)]) + } + } else { + roots = append(roots, f.root.transitions[matchMetricType]) + } + var captureCount int + var finalStates []*mappingState + for _, root := range roots { + captureCount = 0 + for i, field := range matchFields { + state, prs := root.transitions[field] + if !prs { + state = &mappingState{} + (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) + (*state).maxRemainingLength = len(matchFields) - i - 1 + (*state).minRemainingLength = len(matchFields) - i - 1 + root.transitions[field] = state + // if this is last field, set result to currentMapping instance + if i == len(matchFields)-1 { + root.transitions[field].result = result + } + } else { + (*state).maxRemainingLength = max(len(matchFields)-i-1, (*state).maxRemainingLength) + (*state).minRemainingLength = min(len(matchFields)-i-1, (*state).minRemainingLength) + } + if field == "*" { + captureCount++ + } + + // goto next state + root = state + } + finalStates = append(finalStates, root) + } + nameFmt := newTemplateFormatter(name, captureCount) + + currentLabelFormatter := make(map[string]*templateFormatter, captureCount) + for label, valueExpr := range labels { + lblFmt := newTemplateFormatter(valueExpr, captureCount) + currentLabelFormatter[label] = lblFmt + } + + for _, state := range finalStates { + state.resultNameFormatter = nameFmt + state.resultLabelsFormatter = currentLabelFormatter + state.resultPriority = f.statesCount + } + + f.statesCount++ + +} + +func (f *FSM) DumpFSM() { + if f.dumpFSMPath == "" { + return + } + log.Infoln("Start dumping FSM to", f.dumpFSMPath) + idx := 0 + states := make(map[int]*mappingState) + states[idx] = f.root + + fd, _ := os.Create(f.dumpFSMPath) + w := bufio.NewWriter(fd) + w.WriteString("digraph g {\n") + w.WriteString("rankdir=LR\n") // make it vertical + w.WriteString("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n") // remove label of node + + for idx < len(states) { + for field, transition := range states[idx].transitions { + states[len(states)] = transition + w.WriteString(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)) + if idx == 0 { + // color for metric types + w.WriteString(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1)) + } else if transition.transitions == nil || len(transition.transitions) == 0 { + // color for end state + w.WriteString(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)) + } + } + idx++ + } + // color for start state + w.WriteString(fmt.Sprintf("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];\n")) + w.WriteString("}") + w.Flush() + log.Infoln("Finish dumping FSM") +} + +func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { + needBacktrack := false + // rule A and B that has same length and + // A one has * in rules but is not a superset of B makes it needed for backtracking + ruleByLength := make(map[int][]string) + ruleREByLength := make(map[int][]*regexp.Regexp) + + // first sort rules by length + for _, mapping := range mappings { + l := len(strings.Split(mapping, ".")) + ruleByLength[l] = append(ruleByLength[l], mapping) + + metricRe := strings.Replace(mapping, ".", "\\.", -1) + metricRe = strings.Replace(metricRe, "*", "([^.]*)", -1) + regex, err := regexp.Compile("^" + metricRe + "$") + if err != nil { + log.Warnf("invalid match %s. cannot compile regex in mapping: %v", mapping, err) + } + // put into array no matter there's error or not, we will skip later if regex is nil + ruleREByLength[l] = append(ruleREByLength[l], regex) + } + + for l, rules := range ruleByLength { + if len(rules) == 1 { + continue + } + rulesRE := ruleREByLength[l] + // for each rule r1 in rules that has * inside, check if r1 is the superset of any rules + // if not then r1 is a rule that leads to backtrack + for i1, r1 := range rules { + currentRuleNeedBacktrack := true + re1 := rulesRE[i1] + if re1 == nil || strings.Index(r1, "*") == -1 { + continue + } + for i2, r2 := range rules { + if i2 != i1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { + // log if we care about ordering and the superset occurs before + if !f.disableOrdering && i1 < i2 { + log.Warnf("match \"%s\" is a super set of match \"%s\" but in a lower order, "+ + "the first will never be matched\n", r1, r2) + } + currentRuleNeedBacktrack = false + } + } + for i2, re2 := range rulesRE { + // especially, if r1 is a subset of other rule, we don't need backtrack + // because either we turned on ordering + // or we disabled ordering and can't match it even with backtrack + if i2 != i1 && re2 != nil && len(re2.FindStringSubmatchIndex(r1)) > 0 { + currentRuleNeedBacktrack = false + } + } + if currentRuleNeedBacktrack { + log.Warnf("backtracking required because of match \"%s\", "+ + "matching performance may be degraded\n", r1) + needBacktrack = true + } + } + } + + // backtracking will always be needed if ordering of rules is not disabled + // since transistions are stored in (unordered) map + // note: don't move this branch to the beginning of this function + // since we need logs for superset rules + f.needsBacktracking = !f.disableOrdering || needBacktrack + + return f.needsBacktracking +} + +func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interface{}, string, prometheus.Labels, bool) { + matchFields := strings.Split(statsdMetric, ".") + currentState := f.root.transitions[statsdMetricType] + + // the cursor/pointer in the backtrack stack + var backtrackCursor *fsmBacktrackStackCursor + resumeFromBacktrack := false + + var finalState *mappingState + + captures := make(map[int]string, len(matchFields)) + captureIdx := 0 + filedsCount := len(matchFields) + i := 0 + var state *mappingState + for { // the loop for backtracking + for { // the loop for a single "depth only" search + var present bool + // if we resume from backtrack, we should skip this branch in this case + // since the state that were saved at the end of this branch + if !resumeFromBacktrack { + if len(currentState.transitions) > 0 { + field := matchFields[i] + state, present = currentState.transitions[field] + fieldsLeft := filedsCount - i - 1 + // also compare length upfront to avoid unnecessary loop or backtrack + if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + state, present = currentState.transitions["*"] + if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + break + } else { + captures[captureIdx] = field + captureIdx++ + } + } else if f.needsBacktracking { + altState, present := currentState.transitions["*"] + if !present || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength { + } else { + // push to backtracking stack + newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: altState, + fieldIndex: i, + captureIndex: captureIdx, currentCapture: field, + } + if backtrackCursor != nil { + backtrackCursor.next = &newCursor + } + backtrackCursor = &newCursor + } + } + } else { // no more transitions for this state + break + } + } // backtrack will resume from here + + // do we reach a final state? + if state.result != nil && i == filedsCount-1 { + if f.disableOrdering { + finalState = state + // do a double break + goto formatLabels + } else if finalState == nil || finalState.resultPriority > state.resultPriority { + // if we care about ordering, try to find a result with highest prioity + finalState = state + } + break + } + + i++ + if i >= filedsCount { + break + } + + resumeFromBacktrack = false + currentState = state + } + if backtrackCursor == nil { + // if we are not doing backtracking or all path has been travesaled + break + } else { + // pop one from stack + state = backtrackCursor.state + currentState = state + i = backtrackCursor.fieldIndex + captureIdx = backtrackCursor.captureIndex + 1 + // put the * capture back + captures[captureIdx-1] = backtrackCursor.currentCapture + backtrackCursor = backtrackCursor.prev + if backtrackCursor != nil { + // deref for GC + backtrackCursor.next = nil + } + resumeFromBacktrack = true + } + } + +formatLabels: + // format name and labels + if finalState != nil { + name := finalState.resultNameFormatter.format(captures) + + labels := prometheus.Labels{} + for key, formatter := range finalState.resultLabelsFormatter { + labels[key] = formatter.format(captures) + } + return finalState.result, name, labels, true + } + return nil, "", nil, false +} diff --git a/pkg/mapper/fsm/minmax.go b/pkg/mapper/fsm/minmax.go new file mode 100644 index 0000000..cf37159 --- /dev/null +++ b/pkg/mapper/fsm/minmax.go @@ -0,0 +1,17 @@ +package fsm + +// min and max implementation for integer + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +func max(x, y int) int { + if x > y { + return x + } + return y +} diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 63926e8..999c55c 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -14,17 +14,13 @@ package mapper import ( - "bufio" "fmt" "io/ioutil" - "os" "regexp" - "strconv" - "strings" "sync" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/log" + "github.com/prometheus/statsd_exporter/pkg/mapper/fsm" yaml "gopkg.in/yaml.v2" ) @@ -40,59 +36,31 @@ var ( ) type mapperConfigDefaults struct { - TimerType TimerType `yaml:"timer_type"` - Buckets []float64 `yaml:"buckets"` - Quantiles []metricObjective `yaml:"quantiles"` - MatchType MatchType `yaml:"match_type"` - GlobDisbleOrdering bool `yaml:"glob_disable_ordering"` -} - -type mappingState struct { - transitions map[string]*mappingState - minRemainingLength int - maxRemainingLength int - // result is nil unless there's a metric ends with this state - result *MetricMapping + TimerType TimerType `yaml:"timer_type"` + Buckets []float64 `yaml:"buckets"` + Quantiles []metricObjective `yaml:"quantiles"` + MatchType MatchType `yaml:"match_type"` + GlobDisableOrdering bool `yaml:"glob_disable_ordering"` } type MetricMapper struct { - Defaults mapperConfigDefaults `yaml:"defaults"` - Mappings []MetricMapping `yaml:"mappings"` - FSM *mappingState - hasFSM bool - FSMNeedsBacktracking bool - // if doRegex is true, at least one matching rule is regex type - doRegex bool - dumpFSMPath string - mutex sync.Mutex + Defaults mapperConfigDefaults `yaml:"defaults"` + Mappings []MetricMapping `yaml:"mappings"` + FSM *fsm.FSM + doFSM bool + doRegex bool + mutex sync.Mutex MappingsCount prometheus.Gauge } -type templateFormatter struct { - captureIndexes []int - captureCount int - fmtString string -} - -type fsmBacktrackStackCursor struct { - fieldIndex int - captureIndex int - currentCapture string - state *mappingState - prev *fsmBacktrackStackCursor - next *fsmBacktrackStackCursor -} - type matchMetricType string type MetricMapping struct { Match string `yaml:"match"` Name string `yaml:"name"` - NameFormatter templateFormatter regex *regexp.Regexp Labels prometheus.Labels `yaml:"labels"` - LabelsFormatter map[string]templateFormatter TimerType TimerType `yaml:"timer_type"` Buckets []float64 `yaml:"buckets"` Quantiles []metricObjective `yaml:"quantiles"` @@ -100,7 +68,6 @@ type MetricMapping struct { HelpText string `yaml:"help"` Action ActionType `yaml:"action"` MatchMetricType MetricType `yaml:"match_metric_type"` - priority int } type metricObjective struct { @@ -114,48 +81,6 @@ var defaultQuantiles = []metricObjective{ {Quantile: 0.99, Error: 0.001}, } -func generateFormatter(valueExpr string, captureCount int) templateFormatter { - matches := templateReplaceCaptureRE.FindAllStringSubmatch(valueExpr, -1) - if len(matches) == 0 { - // if no regex reference found, keep it as it is - return templateFormatter{captureCount: 0, fmtString: valueExpr} - } - - var indexes []int - valueFormatter := valueExpr - for _, match := range matches { - idx, err := strconv.Atoi(match[len(match)-1]) - if err != nil || idx > captureCount || idx < 1 { - // if index larger than captured count or using unsupported named capture group, - // replace with empty string - valueFormatter = strings.Replace(valueFormatter, match[0], "", -1) - } else { - valueFormatter = strings.Replace(valueFormatter, match[0], "%s", -1) - // note: the regex reference variable $? starts from 1 - indexes = append(indexes, idx-1) - } - } - return templateFormatter{ - captureIndexes: indexes, - captureCount: len(indexes), - fmtString: valueFormatter, - } -} - -func formatTemplate(formatter templateFormatter, captures map[int]string) string { - if formatter.captureCount == 0 { - // no label substitution, keep as it is - return formatter.fmtString - } else { - indexes := formatter.captureIndexes - vargs := make([]interface{}, formatter.captureCount) - for i, idx := range indexes { - vargs[i] = captures[idx] - } - return fmt.Sprintf(formatter.fmtString, vargs...) - } -} - func min(x, y int) int { if x < y { return x @@ -189,18 +114,13 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { n.Defaults.MatchType = MatchTypeGlob } - maxPossibleTransitions := len(n.Mappings) + remainingMappingsCount := len(n.Mappings) - n.FSM = &mappingState{} - n.FSM.transitions = make(map[string]*mappingState, 3) - for _, field := range []MetricType{MetricTypeCounter, MetricTypeTimer, MetricTypeGauge, ""} { - state := &mappingState{} - (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) - n.FSM.transitions[string(field)] = state - } + n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeTimer)}, + remainingMappingsCount, n.Defaults.GlobDisableOrdering) for i := range n.Mappings { - maxPossibleTransitions-- + remainingMappingsCount-- currentMapping := &n.Mappings[i] @@ -227,63 +147,15 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { currentMapping.Action = ActionTypeMap } - currentMapping.priority = i - if currentMapping.MatchType == MatchTypeGlob { - n.hasFSM = true + n.doFSM = true if !metricLineRE.MatchString(currentMapping.Match) { return fmt.Errorf("invalid match: %s", currentMapping.Match) } - // first split by "." - matchFields := strings.Split(currentMapping.Match, ".") - // fill into our FSM - roots := []*mappingState{} - if currentMapping.MatchMetricType == "" { - // if metricType not specified, connect the state from all three types - for _, metricType := range []MetricType{MetricTypeCounter, MetricTypeTimer, MetricTypeGauge, ""} { - roots = append(roots, n.FSM.transitions[string(metricType)]) - } - } else { - roots = append(roots, n.FSM.transitions[string(currentMapping.MatchMetricType)]) - } - var captureCount int - for _, root := range roots { - captureCount = 0 - for i, field := range matchFields { - state, prs := root.transitions[field] - if !prs { - state = &mappingState{} - (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) - (*state).maxRemainingLength = len(matchFields) - i - 1 - (*state).minRemainingLength = len(matchFields) - i - 1 - root.transitions[field] = state - // if this is last field, set result to currentMapping instance - if i == len(matchFields)-1 { - root.transitions[field].result = currentMapping - } - } else { - (*state).maxRemainingLength = max(len(matchFields)-i-1, (*state).maxRemainingLength) - (*state).minRemainingLength = min(len(matchFields)-i-1, (*state).minRemainingLength) - } - if field == "*" { - captureCount++ - } + n.FSM.AddState(currentMapping.Match, currentMapping.Name, currentMapping.Labels, string(currentMapping.MatchMetricType), + remainingMappingsCount, currentMapping) - // goto next state - root = state - } - } - nameFmt := generateFormatter(currentMapping.Name, captureCount) - - currentMapping.NameFormatter = nameFmt - - currentLabelFormatter := make(map[string]templateFormatter, captureCount) - for label, valueExpr := range currentMapping.Labels { - lblFmt := generateFormatter(valueExpr, captureCount) - currentLabelFormatter[label] = lblFmt - } - currentMapping.LabelsFormatter = currentLabelFormatter } else { if regex, err := regexp.Compile(currentMapping.Match); err != nil { return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err) @@ -312,16 +184,19 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { m.Defaults = n.Defaults m.Mappings = n.Mappings - m.hasFSM = n.hasFSM - if n.hasFSM { + if n.doFSM { + var mappings []string + for _, mapping := range n.Mappings { + if mapping.MatchType == MatchTypeGlob { + mappings = append(mappings, mapping.Match) + } + } + n.FSM.TestIfNeedBacktracking(mappings) + m.FSM = n.FSM m.doRegex = n.doRegex - if m.dumpFSMPath != "" { - dumpFSM(m.dumpFSMPath, m.FSM) - } - - m.FSMNeedsBacktracking = needBacktracking(&n) } + m.doFSM = n.doFSM if m.MappingsCount != nil { m.MappingsCount.Set(float64(len(n.Mappings))) @@ -330,121 +205,6 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { return nil } -func (m *MetricMapper) SetDumpFSMPath(path string) error { - m.dumpFSMPath = path - return nil -} - -func needBacktracking(n *MetricMapper) bool { - needBacktrack := false - // rule A and B that has same length and - // A one has * in rules but is not a superset of B makes it needed for backtracking - ruleByLength := make(map[int][]string) - ruleREByLength := make(map[int][]*regexp.Regexp) - rulesOrderByLength := make(map[int][]int) - - // first sort rules by length - for _, mapping := range n.Mappings { - if mapping.MatchType != MatchTypeGlob { - continue - } - l := len(strings.Split(mapping.Match, ".")) - ruleByLength[l] = append(ruleByLength[l], mapping.Match) - - metricRe := strings.Replace(mapping.Match, ".", "\\.", -1) - metricRe = strings.Replace(metricRe, "*", "([^.]*)", -1) - regex, err := regexp.Compile("^" + metricRe + "$") - if err != nil { - log.Warnf("invalid match %s. cannot compile regex in mapping: %v", mapping.Match, err) - } - // put into array no matter there's error or not, we will skip later if regex is nil - ruleREByLength[l] = append(ruleREByLength[l], regex) - rulesOrderByLength[l] = append(rulesOrderByLength[l], mapping.priority) - } - - for l, rules := range ruleByLength { - if len(rules) == 1 { - continue - } - rulesRE := ruleREByLength[l] - rulesOrder := rulesOrderByLength[l] - // for each rule r1 in rules that has * inside, check if r1 is the superset of any rules - // if not then r1 is a rule that leads to backtrack - for i1, r1 := range rules { - currentRuleNeedBacktrack := true - re1 := rulesRE[i1] - if re1 == nil || strings.Index(r1, "*") == -1 { - continue - } - for i2, r2 := range rules { - if i2 != i1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { - // log if we care about ordering and the superset occurs before - if !n.Defaults.GlobDisbleOrdering && rulesOrder[i1] < rulesOrder[i2] { - log.Warnf("match \"%s\" is a super set of match \"%s\" but in a lower order, "+ - "the first will never be matched\n", r1, r2) - } - currentRuleNeedBacktrack = false - } - } - for i2, re2 := range rulesRE { - // especially, if r1 is a subset of other rule, we don't need backtrack - // because either we turned on ordering - // or we disabled ordering and can't match it even with backtrack - if i2 != i1 && re2 != nil && len(re2.FindStringSubmatchIndex(r1)) > 0 { - currentRuleNeedBacktrack = false - } - } - if currentRuleNeedBacktrack { - log.Warnf("backtracking required because of match \"%s\", "+ - "matching performance may be degraded\n", r1) - needBacktrack = true - } - } - } - if !n.Defaults.GlobDisbleOrdering { - // backtracking only makes sense when we disbled ordering of rules - // where transistions are stored in (unordered) map - // note: don't move this branch to the beginning of this function - // since we need logs for superset rules - return true - } else { - return needBacktrack - } -} - -func dumpFSM(fileName string, root *mappingState) { - log.Infoln("Start dumping FSM to", fileName) - idx := 0 - states := make(map[int]*mappingState) - states[idx] = root - - f, _ := os.Create(fileName) - w := bufio.NewWriter(f) - w.WriteString("digraph g {\n") - w.WriteString("rankdir=LR\n") // make it vertical - w.WriteString("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n") // remove label of node - - for idx < len(states) { - for field, transition := range states[idx].transitions { - states[len(states)] = transition - w.WriteString(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)) - if idx == 0 { - // color for metric types - w.WriteString(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1)) - } else if transition.transitions == nil || len(transition.transitions) == 0 { - // color for end state - w.WriteString(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)) - } - } - idx++ - } - // color for start state - w.WriteString(fmt.Sprintf("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];\n")) - w.WriteString("}") - w.Flush() - log.Infoln("Finish dumping FSM") -} - func (m *MetricMapper) InitFromFile(fileName string) error { mappingStr, err := ioutil.ReadFile(fileName) if err != nil { @@ -455,110 +215,15 @@ func (m *MetricMapper) InitFromFile(fileName string) error { func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) { // glob matching - if m.hasFSM { - matchFields := strings.Split(statsdMetric, ".") - root := m.FSM.transitions[string(statsdMetricType)] - captures := make(map[int]string, len(matchFields)) - captureIdx := 0 - var backtrackCursor *fsmBacktrackStackCursor - resumeFromBacktrack := false - var result *MetricMapping - filedsCount := len(matchFields) - i := 0 - var state *mappingState - for { - for { - var prs bool - if !resumeFromBacktrack { - if len(root.transitions) > 0 { - field := matchFields[i] - state, prs = root.transitions[field] - fieldsLeft := filedsCount - i - 1 - // also compare length upfront to avoid unnecessary loop or backtrack - if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { - state, prs = root.transitions["*"] - if !prs || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { - break - } else { - captures[captureIdx] = field - captureIdx++ - } - } else if m.FSMNeedsBacktracking { - altState, prs := root.transitions["*"] - if !prs || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength { - } else { - // push to stack - newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: altState, - fieldIndex: i, - captureIndex: captureIdx, currentCapture: field, - } - if backtrackCursor != nil { - backtrackCursor.next = &newCursor - } - backtrackCursor = &newCursor - } - } - } else { // no more transitions for this state - break - } - } // backtrack will resume from here - - // do we reach a final state? - if state.result != nil && i == filedsCount-1 { - if m.Defaults.GlobDisbleOrdering { - result = state.result - // do a double break - goto formatLabels - } else if result == nil || result.priority > state.result.priority { - // if we care about ordering, try to find a result with highest prioity - result = state.result - } - break - } - - i++ - if i >= filedsCount { - break - } - - resumeFromBacktrack = false - root = state - } - if backtrackCursor == nil { - // if we are not doing backtracking or all path has been travesaled - break - } else { - // pop one from stack - state = backtrackCursor.state - root = state - i = backtrackCursor.fieldIndex - captureIdx = backtrackCursor.captureIndex + 1 - // put the * capture back - captures[captureIdx-1] = backtrackCursor.currentCapture - backtrackCursor = backtrackCursor.prev - if backtrackCursor != nil { - // deref for GC - backtrackCursor.next = nil - } - resumeFromBacktrack = true - } - } - - formatLabels: - // format name and labels - if result != nil { - result.Name = formatTemplate(result.NameFormatter, captures) - - labels := prometheus.Labels{} - for label := range result.Labels { - labels[label] = formatTemplate(result.LabelsFormatter[label], captures) - } - return result, labels, true - } else if !m.doRegex { + if m.doFSM { + mapping, mappingName, labels, matched := m.FSM.GetMapping(statsdMetric, string(statsdMetricType)) + if matched { + mapping.(*MetricMapping).Name = mappingName + return mapping.(*MetricMapping), labels, matched + } else if !matched && !m.doRegex { // if there's no regex match type, return immediately return nil, nil, false } - } // regex matching From 5e1df60d22c69836dcfa7666b0a26571c9d03621 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Fri, 21 Sep 2018 18:09:36 -0700 Subject: [PATCH 08/23] abstract dumpFsm Signed-off-by: Wangchong Zhou --- main.go | 28 ++++++++++++++++++++++------ pkg/mapper/fsm/fsm.go | 35 ++++++++++------------------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/main.go b/main.go index 5d13480..306ee2a 100644 --- a/main.go +++ b/main.go @@ -14,8 +14,10 @@ package main import ( + "bufio" "net" "net/http" + "os" "strconv" "github.com/howeyc/fsnotify" @@ -118,6 +120,20 @@ func watchConfig(fileName string, mapper *mapper.MetricMapper) { } } +func dumpFSM(mapper *mapper.MetricMapper, dumpFilename string) error { + f, err := os.Create(dumpFilename) + if err != nil { + return err + } + log.Infoln("Start dumping FSM to", dumpFilename) + w := bufio.NewWriter(f) + mapper.FSM.DumpFSM(w) + w.Flush() + f.Close() + log.Infoln("Finish dumping FSM") + return nil +} + func main() { var ( listenAddress = kingpin.Flag("web.listen-address", "The address on which to expose the web interface and generated Prometheus metrics.").Default(":9102").String() @@ -179,17 +195,17 @@ func main() { } mapper := &mapper.MetricMapper{MappingsCount: mappingsCount} - if *dumpFSMPath != "" { - err := mapper.FSM.SetDumpFSMPath(*dumpFSMPath) - if err != nil { - log.Fatal("Error setting dump FSM path:", err) - } - } if *mappingConfig != "" { err := mapper.InitFromFile(*mappingConfig) if err != nil { log.Fatal("Error loading config:", err) } + if *dumpFSMPath != "" { + err := dumpFSM(mapper, *dumpFSMPath) + if err != nil { + log.Fatal("Error dumpping FSM:", err) + } + } go watchConfig(*mappingConfig, mapper) } exporter := NewExporter(mapper) diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go index 79240f0..5a8f543 100644 --- a/pkg/mapper/fsm/fsm.go +++ b/pkg/mapper/fsm/fsm.go @@ -1,9 +1,8 @@ package fsm import ( - "bufio" "fmt" - "os" + "io" "regexp" "strings" @@ -38,17 +37,11 @@ type fsmBacktrackStackCursor struct { type FSM struct { root *mappingState needsBacktracking bool - dumpFSMPath string metricTypes []string disableOrdering bool statesCount int } -func (fsm *FSM) SetDumpFSMPath(path string) error { - fsm.dumpFSMPath = path - return nil -} - func NewFSM(metricTypes []string, maxPossibleTransitions int, disableOrdering bool) *FSM { fsm := FSM{} root := &mappingState{} @@ -128,40 +121,32 @@ func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matc } -func (f *FSM) DumpFSM() { - if f.dumpFSMPath == "" { - return - } - log.Infoln("Start dumping FSM to", f.dumpFSMPath) +func (f *FSM) DumpFSM(w io.Writer) { idx := 0 states := make(map[int]*mappingState) states[idx] = f.root - fd, _ := os.Create(f.dumpFSMPath) - w := bufio.NewWriter(fd) - w.WriteString("digraph g {\n") - w.WriteString("rankdir=LR\n") // make it vertical - w.WriteString("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n") // remove label of node + w.Write([]byte("digraph g {\n")) + w.Write([]byte("rankdir=LR\n")) // make it vertical + w.Write([]byte("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n")) // remove label of node for idx < len(states) { for field, transition := range states[idx].transitions { states[len(states)] = transition - w.WriteString(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)) + w.Write([]byte(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field))) if idx == 0 { // color for metric types - w.WriteString(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1)) + w.Write([]byte(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1))) } else if transition.transitions == nil || len(transition.transitions) == 0 { // color for end state - w.WriteString(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)) + w.Write([]byte(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1))) } } idx++ } // color for start state - w.WriteString(fmt.Sprintf("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];\n")) - w.WriteString("}") - w.Flush() - log.Infoln("Finish dumping FSM") + w.Write([]byte(fmt.Sprintf("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];\n"))) + w.Write([]byte("}")) } func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { From 668e31e5f115fc9f447b6dd2d1bab02db70aa853 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Fri, 21 Sep 2018 18:13:23 -0700 Subject: [PATCH 09/23] license header Signed-off-by: Wangchong Zhou --- pkg/mapper/fsm/formatter.go | 13 +++++++++++++ pkg/mapper/fsm/fsm.go | 13 +++++++++++++ pkg/mapper/fsm/minmax.go | 13 +++++++++++++ 3 files changed, 39 insertions(+) diff --git a/pkg/mapper/fsm/formatter.go b/pkg/mapper/fsm/formatter.go index 591f9a8..d6782ba 100644 --- a/pkg/mapper/fsm/formatter.go +++ b/pkg/mapper/fsm/formatter.go @@ -1,3 +1,16 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fsm import ( diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go index 5a8f543..458c1b7 100644 --- a/pkg/mapper/fsm/fsm.go +++ b/pkg/mapper/fsm/fsm.go @@ -1,3 +1,16 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fsm import ( diff --git a/pkg/mapper/fsm/minmax.go b/pkg/mapper/fsm/minmax.go index cf37159..95bd9c5 100644 --- a/pkg/mapper/fsm/minmax.go +++ b/pkg/mapper/fsm/minmax.go @@ -1,3 +1,16 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fsm // min and max implementation for integer From f387766cc2d84a26b8578d5aacfb952e1ffee3e2 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Mon, 24 Sep 2018 13:36:01 -0700 Subject: [PATCH 10/23] cleanup and add comments Signed-off-by: Wangchong Zhou --- pkg/mapper/fsm/fsm.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go index 458c1b7..28d90dd 100644 --- a/pkg/mapper/fsm/fsm.go +++ b/pkg/mapper/fsm/fsm.go @@ -55,6 +55,7 @@ type FSM struct { statesCount int } +// NewFSM creates a new FSM instance func NewFSM(metricTypes []string, maxPossibleTransitions int, disableOrdering bool) *FSM { fsm := FSM{} root := &mappingState{} @@ -73,6 +74,7 @@ func NewFSM(metricTypes []string, maxPossibleTransitions int, disableOrdering bo return &fsm } +// AddState adds a state into the existing FSM func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matchMetricType string, maxPossibleTransitions int, result interface{}) { // first split by "." @@ -134,6 +136,7 @@ func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matc } +// DumpFSM accepts a io.writer and write the current FSM into dot file format func (f *FSM) DumpFSM(w io.Writer) { idx := 0 states := make(map[int]*mappingState) @@ -162,6 +165,7 @@ func (f *FSM) DumpFSM(w io.Writer) { w.Write([]byte("}")) } +// TestIfNeedBacktracking test if backtrack is needed for current FSM func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { needBacktrack := false // rule A and B that has same length and @@ -232,17 +236,20 @@ func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { return f.needsBacktracking } +// GetMapping implements a mapping algorithm for Glob pattern func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interface{}, string, prometheus.Labels, bool) { matchFields := strings.Split(statsdMetric, ".") currentState := f.root.transitions[statsdMetricType] - // the cursor/pointer in the backtrack stack + // the cursor/pointer in the backtrack stack implemented as a double-linked list var backtrackCursor *fsmBacktrackStackCursor resumeFromBacktrack := false + // the return variable var finalState *mappingState captures := make(map[int]string, len(matchFields)) + // keep track of captured group so we don't need to do append() on captures captureIdx := 0 filedsCount := len(matchFields) i := 0 @@ -267,6 +274,7 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interfac captureIdx++ } } else if f.needsBacktracking { + // if backtracking is needed, also check for alternative transition altState, present := currentState.transitions["*"] if !present || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength { } else { @@ -275,13 +283,15 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interfac fieldIndex: i, captureIndex: captureIdx, currentCapture: field, } + // if this is not the first time, connect to the previous cursor if backtrackCursor != nil { backtrackCursor.next = &newCursor } backtrackCursor = &newCursor } } - } else { // no more transitions for this state + } else { + // no more transitions for this state break } } // backtrack will resume from here @@ -290,8 +300,7 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interfac if state.result != nil && i == filedsCount-1 { if f.disableOrdering { finalState = state - // do a double break - goto formatLabels + return formatLabels(finalState, captures) } else if finalState == nil || finalState.resultPriority > state.resultPriority { // if we care about ordering, try to find a result with highest prioity finalState = state @@ -327,7 +336,10 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interfac } } -formatLabels: + return formatLabels(finalState, captures) +} + +func formatLabels(finalState *mappingState, captures map[int]string) (interface{}, string, prometheus.Labels, bool) { // format name and labels if finalState != nil { name := finalState.resultNameFormatter.format(captures) From a8dcc589e507d712a8ee8a25314c92f5e03eaa7d Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Mon, 24 Sep 2018 13:36:43 -0700 Subject: [PATCH 11/23] add more perf test Signed-off-by: Wangchong Zhou --- pkg/mapper/mapper_test.go | 293 +++++++++++++++++++++++++++++++++++++- 1 file changed, 286 insertions(+), 7 deletions(-) diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index 1d5f9cd..e976fd8 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -608,24 +608,23 @@ mappings: } } -func TestRPS(t *testing.T) { +func TestPerformance(t *testing.T) { scenarios := []struct { + name string config string configBad bool mappings mappings }{ - // Config with several mapping definitions. { + name: "glob", config: `--- -defaults: - glob_disable_ordering: true mappings: -- match: test.dispatcher.*.*.* +- match: test.dispatcher.*.*.succeeded name: "dispatch_events" labels: processor: "$1" action: "$2" - result: "$3" + result: "succeeded" job: "test_dispatcher" - match: test.my-dispatch-host01.name.dispatcher.*.*.* name: "host_dispatch_events" @@ -650,6 +649,286 @@ mappings: orgid: "${11}" oauthid: "${12}" - match: "*.*" + name: "catchall" + labels: + first: "$1" + second: "$2" + third: "$3" + job: "-" + `, + mappings: mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "host_dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "tyk_http_request", + labels: map[string]string{ + "method_and_path": "get/threads/1/posts", + "response_code": "200", + "apikey": "00000000", + "apiversion": "nonversioned", + "apiname": "discussions", + "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", + "ipv4_t1": "172", + "ipv4_t2": "20", + "ipv4_t3": "0", + "ipv4_t4": "1", + "orgid": "12ba97b7eaa1a50001000001", + "oauthid": "", + }, + }, + "foo.bar": { + name: "catchall", + labels: map[string]string{ + "first": "foo", + "second": "bar", + "third": "", + "job": "-", + }, + }, + "foo.bar.baz": {}, + }, + }, + { + name: "glob no ordering", + config: `--- +defaults: + glob_disable_ordering: true +mappings: +- match: test.dispatcher.*.*.succeeded + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "succeeded" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher.*.*.* + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* + name: "tyk_http_request" + labels: + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" +- match: "*.*" + name: "catchall" + labels: + first: "$1" + second: "$2" + third: "$3" + job: "-" + `, + mappings: mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "host_dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "tyk_http_request", + labels: map[string]string{ + "method_and_path": "get/threads/1/posts", + "response_code": "200", + "apikey": "00000000", + "apiversion": "nonversioned", + "apiname": "discussions", + "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", + "ipv4_t1": "172", + "ipv4_t2": "20", + "ipv4_t3": "0", + "ipv4_t4": "1", + "orgid": "12ba97b7eaa1a50001000001", + "oauthid": "", + }, + }, + "foo.bar": { + name: "catchall", + labels: map[string]string{ + "first": "foo", + "second": "bar", + "third": "", + "job": "-", + }, + }, + "foo.bar.baz": {}, + }, + }, + { + name: "glob with backtracking (no ordering implicated)", + config: `--- +defaults: + glob_disable_ordering: true +mappings: +- match: test.dispatcher.*.*.succeeded + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "succeeded" + job: "test_dispatcher" +- match: test.dispatcher.*.received.* + name: "dispatch_events_wont_match" + labels: + processor: "$1" + action: "received" + result: "$2" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher.*.*.* + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* + name: "tyk_http_request" + labels: + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" +- match: "*.*" + name: "catchall" + labels: + first: "$1" + second: "$2" + third: "$3" + job: "-" + `, + mappings: mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "host_dispatch_events", + labels: map[string]string{ + "processor": "FooProcessor", + "action": "send", + "result": "succeeded", + "job": "test_dispatcher", + }, + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "tyk_http_request", + labels: map[string]string{ + "method_and_path": "get/threads/1/posts", + "response_code": "200", + "apikey": "00000000", + "apiversion": "nonversioned", + "apiname": "discussions", + "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", + "ipv4_t1": "172", + "ipv4_t2": "20", + "ipv4_t3": "0", + "ipv4_t4": "1", + "orgid": "12ba97b7eaa1a50001000001", + "oauthid": "", + }, + }, + "foo.bar": { + name: "catchall", + labels: map[string]string{ + "first": "foo", + "second": "bar", + "third": "", + "job": "-", + }, + }, + "foo.bar.baz": {}, + }, + }, + { + name: "regex", + config: `--- +defaults: + match_type: regex +mappings: +- match: test\.dispatcher\.([^.]*)\.([^.]*)\.([^.]*) + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher\.([^.]*)\.([^.]*)\.([^.]*) + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "tyk_http_request" + labels: + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" +- match: \.([^.]*)\.([^.]*) name: "catchall" labels: first: "$1" @@ -724,7 +1003,7 @@ mappings: mapper.GetMapping(metric, dummyMetricType) } } - fmt.Println("finished 100000 iterations in", time.Now().Sub(start)) + fmt.Printf("finished 100000 iterations for \"%s\" in %v\n", scenario.name, time.Now().Sub(start)) } } From a0681a0cd27176ae19a1062def6cbac23d2e4e67 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Mon, 24 Sep 2018 15:22:43 -0700 Subject: [PATCH 12/23] more perf test Signed-off-by: Wangchong Zhou --- pkg/mapper/fsm/fsm.go | 35 ++- pkg/mapper/mapper_test.go | 459 +++++++++++++++++++++++--------------- 2 files changed, 308 insertions(+), 186 deletions(-) diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go index 28d90dd..f3be09c 100644 --- a/pkg/mapper/fsm/fsm.go +++ b/pkg/mapper/fsm/fsm.go @@ -168,8 +168,8 @@ func (f *FSM) DumpFSM(w io.Writer) { // TestIfNeedBacktracking test if backtrack is needed for current FSM func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { needBacktrack := false - // rule A and B that has same length and - // A one has * in rules but is not a superset of B makes it needed for backtracking + // A has * in rules there's other transisitions at the same state + // this makes A the cause of backtracking ruleByLength := make(map[int][]string) ruleREByLength := make(map[int][]*regexp.Regexp) @@ -193,14 +193,31 @@ func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { continue } rulesRE := ruleREByLength[l] - // for each rule r1 in rules that has * inside, check if r1 is the superset of any rules - // if not then r1 is a rule that leads to backtrack for i1, r1 := range rules { - currentRuleNeedBacktrack := true + currentRuleNeedBacktrack := false re1 := rulesRE[i1] if re1 == nil || strings.Index(r1, "*") == -1 { continue } + // if a rule is A.B.C.*.E.*, is there a rule A.B.C.D.x.x or A.B.C.*.E.F? (x is any string or *) + for index := 0; index < len(r1); index++ { + if r1[index] != '*' { + continue + } + reStr := strings.Replace(r1[:index], ".", "\\.", -1) + reStr = strings.Replace(reStr, "*", "\\*", -1) + re := regexp.MustCompile("^" + reStr) + for i2, r2 := range rules { + if i2 == i1 { + continue + } + if len(re.FindStringSubmatchIndex(r2)) > 0 { + currentRuleNeedBacktrack = true + break + } + } + } + for i2, r2 := range rules { if i2 != i1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { // log if we care about ordering and the superset occurs before @@ -212,13 +229,17 @@ func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { } } for i2, re2 := range rulesRE { - // especially, if r1 is a subset of other rule, we don't need backtrack + if i2 == i1 || re2 == nil { + continue + } + // if r1 is a subset of other rule, we don't need backtrack // because either we turned on ordering // or we disabled ordering and can't match it even with backtrack - if i2 != i1 && re2 != nil && len(re2.FindStringSubmatchIndex(r1)) > 0 { + if len(re2.FindStringSubmatchIndex(r1)) > 0 { currentRuleNeedBacktrack = false } } + if currentRuleNeedBacktrack { log.Warnf("backtracking required because of match \"%s\", "+ "matching performance may be degraded\n", r1) diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index e976fd8..5194b99 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -608,7 +608,44 @@ mappings: } } +func duplicateRules(count int, template string) string { + rules := "" + for i := 0; i < count; i++ { + rules += fmt.Sprintf(template, i) + } + return rules +} + func TestPerformance(t *testing.T) { + sampleRulesAmount := 100 + + ruleTemplateSingleMatchGlob := ` + - match: metric%d.* + name: "metric_single" + labels: + name: "$1" +` + ruleTemplateSingleMatchRegex := ` + - match: metric%d\.([^.]*) + name: "metric_single" + labels: + name: "$1" +` + + ruleTemplateMultipleMatchGlob := ` + - match: metric%d.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" +` + + ruleTemplateMultipleMatchRegex := ` + - match: metric%d\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" +` + scenarios := []struct { name string config string @@ -657,50 +694,10 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "host_dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "tyk_http_request", - labels: map[string]string{ - "method_and_path": "get/threads/1/posts", - "response_code": "200", - "apikey": "00000000", - "apiversion": "nonversioned", - "apiname": "discussions", - "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", - "ipv4_t1": "172", - "ipv4_t2": "20", - "ipv4_t3": "0", - "ipv4_t4": "1", - "orgid": "12ba97b7eaa1a50001000001", - "oauthid": "", - }, - }, - "foo.bar": { - name: "catchall", - labels: map[string]string{ - "first": "foo", - "second": "bar", - "third": "", - "job": "-", - }, - }, + "test.dispatcher.FooProcessor.send.succeeded": {}, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, + "foo.bar": {}, "foo.bar.baz": {}, }, }, @@ -748,50 +745,10 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "host_dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "tyk_http_request", - labels: map[string]string{ - "method_and_path": "get/threads/1/posts", - "response_code": "200", - "apikey": "00000000", - "apiversion": "nonversioned", - "apiname": "discussions", - "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", - "ipv4_t1": "172", - "ipv4_t2": "20", - "ipv4_t3": "0", - "ipv4_t4": "1", - "orgid": "12ba97b7eaa1a50001000001", - "oauthid": "", - }, - }, - "foo.bar": { - name: "catchall", - labels: map[string]string{ - "first": "foo", - "second": "bar", - "third": "", - "job": "-", - }, - }, + "test.dispatcher.FooProcessor.send.succeeded": {}, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, + "foo.bar": {}, "foo.bar.baz": {}, }, }, @@ -846,50 +803,10 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "host_dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "tyk_http_request", - labels: map[string]string{ - "method_and_path": "get/threads/1/posts", - "response_code": "200", - "apikey": "00000000", - "apiversion": "nonversioned", - "apiname": "discussions", - "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", - "ipv4_t1": "172", - "ipv4_t2": "20", - "ipv4_t3": "0", - "ipv4_t4": "1", - "orgid": "12ba97b7eaa1a50001000001", - "oauthid": "", - }, - }, - "foo.bar": { - name: "catchall", - labels: map[string]string{ - "first": "foo", - "second": "bar", - "third": "", - "job": "-", - }, - }, + "test.dispatcher.FooProcessor.send.succeeded": {}, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, + "foo.bar": {}, "foo.bar.baz": {}, }, }, @@ -937,57 +854,241 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "host_dispatch_events", - labels: map[string]string{ - "processor": "FooProcessor", - "action": "send", - "result": "succeeded", - "job": "test_dispatcher", - }, - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "tyk_http_request", - labels: map[string]string{ - "method_and_path": "get/threads/1/posts", - "response_code": "200", - "apikey": "00000000", - "apiversion": "nonversioned", - "apiname": "discussions", - "apiid": "a11bbcdf0ac64ec243658dc64b7100fb", - "ipv4_t1": "172", - "ipv4_t2": "20", - "ipv4_t3": "0", - "ipv4_t4": "1", - "orgid": "12ba97b7eaa1a50001000001", - "oauthid": "", - }, - }, - "foo.bar": { - name: "catchall", - labels: map[string]string{ - "first": "foo", - "second": "bar", - "third": "", - "job": "-", - }, - }, + "test.dispatcher.FooProcessor.send.succeeded": {}, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, + "foo.bar": {}, "foo.bar.baz": {}, }, }, + {}, + { + name: "glob single match ", + config: `--- +mappings: +- match: metric.* + name: "metric_one" + labels: + name: "$1" + `, + mappings: mappings{ + "metric.aaa": {}, + "metric.bbb": {}, + }, + }, + { + name: "regex single match", + config: `--- +mappings: +- match: metric\.([^.]*) + name: "metric_one" + match_type: regex + labels: + name: "$1" + `, + mappings: mappings{ + "metric.aaa": {}, + "metric.bbb": {}, + }, + }, + {}, + { + name: "glob multiple captures", + config: `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" + `, + mappings: mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + }, + }, + { + name: "regex multiple captures", + config: `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" + `, + mappings: mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + }, + }, + { + name: "glob multiple captures no format", + config: `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "$1" + `, + mappings: mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + }, + }, + { + name: "regex multiple captures no expansion", + config: `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + name: "$1" + `, + mappings: mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + }, + }, + {}, + { + name: "glob multiple captures in different labels", + config: `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + label1: "$1" + label2: "$2" + label3: "$3" + label4: "$4" + label5: "$5" + label6: "$6" + label7: "$7" + label8: "$8" + label9: "$9" + label10: "$10" + label11: "$11" + label12: "$12" + `, + mappings: mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + }, + }, + { + name: "regex multiple captures", + config: `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + label1: "$1" + label2: "$2" + label3: "$3" + label4: "$4" + label5: "$5" + label6: "$6" + label7: "$7" + label8: "$8" + label9: "$9" + label10: "$10" + label11: "$11" + label12: "$12" + `, + mappings: mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + }, + }, + {}, + { + name: "glob 100 rules", + config: `--- +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), + mappings: mappings{ + "metric100.a": {}, + }, + }, + { + name: "glob 100 rules, no match", + config: `--- +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), + mappings: mappings{ + "metricnomatchy.a": {}, + }, + }, + { + name: "glob 100 rules without ordering, no match", + config: `--- +defaults: + glob_disable_ordering: true +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), + mappings: mappings{ + "metricnomatchy.a": {}, + }, + }, + { + name: "regex 100 rules, average case", + config: `--- +defaults: + match_type: regex +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchRegex), + mappings: mappings{ + fmt.Sprintf("metric%d.a", sampleRulesAmount/2): {}, // average match position + }, + }, + { + name: "regex 100 rules, worst case", + config: `--- +defaults: + match_type: regex +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchRegex), + mappings: mappings{ + "metric100.a": {}, + }, + }, + {}, + { + name: "glob 100 rules, multiple captures", + config: `--- +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateMultipleMatchGlob), + mappings: mappings{ + "metric50.a.b.c.d.e.f.g.h.i.j.k.l": {}, + }, + }, + { + name: "regex 100 rules, multiple captures, average case", + config: `--- +defaults: + match_type: regex +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateMultipleMatchRegex), + mappings: mappings{ + fmt.Sprintf("metric%d.a.b.c.d.e.f.g.h.i.j.k.l", sampleRulesAmount/2): {}, // average match position + }, + }, + {}, + { + name: "glob 10 rules", + config: `--- +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), + mappings: mappings{ + "metric50.a": {}, + }, + }, + { + name: "regex 10 rules average case", + config: `--- +defaults: + match_type: regex +mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchRegex), + mappings: mappings{ + fmt.Sprintf("metric%d.a", sampleRulesAmount/2): {}, // average match position + }, + }, } - mapper := MetricMapper{} for i, scenario := range scenarios { + if len(scenario.config) == 0 { + fmt.Println("--------------------------------") + continue + } + mapper := MetricMapper{} err := mapper.InitFromYAMLString(scenario.config) if err != nil && !scenario.configBad { t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err) @@ -998,12 +1099,12 @@ mappings: var dummyMetricType MetricType = "" start := time.Now() - for j := 1; j < 100000; j++ { + for j := 1; j < 10000; j++ { for metric, _ := range scenario.mappings { mapper.GetMapping(metric, dummyMetricType) } } - fmt.Printf("finished 100000 iterations for \"%s\" in %v\n", scenario.name, time.Now().Sub(start)) + fmt.Printf("finished 10000 iterations for \"%s\" in %v\n", scenario.name, time.Now().Sub(start)) } } From 6d709d52c1a9c1d1e4b8a6e745891adb919ba2fb Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Mon, 24 Sep 2018 16:42:24 -0700 Subject: [PATCH 13/23] gofmt Signed-off-by: Wangchong Zhou --- pkg/mapper/mapper_test.go | 64 +++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index 5194b99..58e0ec0 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -694,10 +694,18 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": {}, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, - "foo.bar": {}, + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, "foo.bar.baz": {}, }, }, @@ -745,10 +753,18 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": {}, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, - "foo.bar": {}, + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, "foo.bar.baz": {}, }, }, @@ -803,10 +819,18 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": {}, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, - "foo.bar": {}, + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, "foo.bar.baz": {}, }, }, @@ -854,10 +878,18 @@ mappings: job: "-" `, mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": {}, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": {}, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": {}, - "foo.bar": {}, + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, "foo.bar.baz": {}, }, }, From 5262b2904c776d532de6c057e2c0e26e26b45aad Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Fri, 28 Sep 2018 12:55:53 -0700 Subject: [PATCH 14/23] tidy up and use go benchmark Signed-off-by: Wangchong Zhou --- main.go | 4 +- pkg/mapper/fsm/dump.go | 48 ++ pkg/mapper/fsm/formatter.go | 18 +- pkg/mapper/fsm/fsm.go | 260 +++++----- pkg/mapper/mapper.go | 18 +- pkg/mapper/mapper_test.go | 962 +++++++++++++++++++++++------------- 6 files changed, 793 insertions(+), 517 deletions(-) create mode 100644 pkg/mapper/fsm/dump.go diff --git a/main.go b/main.go index 306ee2a..0e7f32a 100644 --- a/main.go +++ b/main.go @@ -142,7 +142,7 @@ func main() { statsdListenTCP = kingpin.Flag("statsd.listen-tcp", "The TCP address on which to receive statsd metric lines. \"\" disables it.").Default(":9125").String() mappingConfig = kingpin.Flag("statsd.mapping-config", "Metric mapping configuration file name.").String() readBuffer = kingpin.Flag("statsd.read-buffer", "Size (in bytes) of the operating system's transmit read buffer associated with the UDP connection. Please make sure the kernel parameters net.core.rmem_max is set to a value greater than the value specified.").Int() - dumpFSMPath = kingpin.Flag("statsd.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String() + dumpFSMPath = kingpin.Flag("debug.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String() ) log.AddFlags(kingpin.CommandLine) @@ -203,7 +203,7 @@ func main() { if *dumpFSMPath != "" { err := dumpFSM(mapper, *dumpFSMPath) if err != nil { - log.Fatal("Error dumpping FSM:", err) + log.Fatal("Error dumping FSM:", err) } } go watchConfig(*mappingConfig, mapper) diff --git a/pkg/mapper/fsm/dump.go b/pkg/mapper/fsm/dump.go new file mode 100644 index 0000000..1bf4675 --- /dev/null +++ b/pkg/mapper/fsm/dump.go @@ -0,0 +1,48 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fsm + +import ( + "fmt" + "io" +) + +// DumpFSM accepts a io.writer and write the current FSM into dot file format +func (f *FSM) DumpFSM(w io.Writer) { + idx := 0 + states := make(map[int]*mappingState) + states[idx] = f.root + + w.Write([]byte("digraph g {\n")) + w.Write([]byte("rankdir=LR\n")) // make it vertical + w.Write([]byte("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n")) // remove label of node + + for idx < len(states) { + for field, transition := range states[idx].transitions { + states[len(states)] = transition + w.Write([]byte(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field))) + if idx == 0 { + // color for metric types + w.Write([]byte(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1))) + } else if transition.transitions == nil || len(transition.transitions) == 0 { + // color for end state + w.Write([]byte(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1))) + } + } + idx++ + } + // color for start state + w.Write([]byte(fmt.Sprintf("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];\n"))) + w.Write([]byte("}")) +} diff --git a/pkg/mapper/fsm/formatter.go b/pkg/mapper/fsm/formatter.go index d6782ba..362c220 100644 --- a/pkg/mapper/fsm/formatter.go +++ b/pkg/mapper/fsm/formatter.go @@ -15,10 +15,15 @@ package fsm import ( "fmt" + "regexp" "strconv" "strings" ) +var ( + templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`) +) + type templateFormatter struct { captureIndexes []int captureCount int @@ -57,12 +62,11 @@ func (formatter *templateFormatter) format(captures map[int]string) string { if formatter.captureCount == 0 { // no label substitution, keep as it is return formatter.fmtString - } else { - indexes := formatter.captureIndexes - vargs := make([]interface{}, formatter.captureCount) - for i, idx := range indexes { - vargs[i] = captures[idx] - } - return fmt.Sprintf(formatter.fmtString, vargs...) } + indexes := formatter.captureIndexes + vargs := make([]interface{}, formatter.captureCount) + for i, idx := range indexes { + vargs[i] = captures[idx] + } + return fmt.Sprintf(formatter.fmtString, vargs...) } diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go index f3be09c..0d7dd7f 100644 --- a/pkg/mapper/fsm/fsm.go +++ b/pkg/mapper/fsm/fsm.go @@ -14,8 +14,6 @@ package fsm import ( - "fmt" - "io" "regexp" "strings" @@ -23,10 +21,6 @@ import ( "github.com/prometheus/common/log" ) -var ( - templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`) -) - type mappingState struct { transitions map[string]*mappingState minRemainingLength int @@ -48,18 +42,18 @@ type fsmBacktrackStackCursor struct { } type FSM struct { - root *mappingState - needsBacktracking bool - metricTypes []string - disableOrdering bool - statesCount int + root *mappingState + metricTypes []string + statesCount int + BacktrackingNeeded bool + OrderingDisabled bool } // NewFSM creates a new FSM instance -func NewFSM(metricTypes []string, maxPossibleTransitions int, disableOrdering bool) *FSM { +func NewFSM(metricTypes []string, maxPossibleTransitions int, orderingDisabled bool) *FSM { fsm := FSM{} root := &mappingState{} - root.transitions = make(map[string]*mappingState, 3) + root.transitions = make(map[string]*mappingState, len(metricTypes)) metricTypes = append(metricTypes, "") for _, field := range metricTypes { @@ -67,7 +61,7 @@ func NewFSM(metricTypes []string, maxPossibleTransitions int, disableOrdering bo (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) root.transitions[string(field)] = state } - fsm.disableOrdering = disableOrdering + fsm.OrderingDisabled = orderingDisabled fsm.metricTypes = metricTypes fsm.statesCount = 0 fsm.root = root @@ -136,38 +130,112 @@ func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matc } -// DumpFSM accepts a io.writer and write the current FSM into dot file format -func (f *FSM) DumpFSM(w io.Writer) { - idx := 0 - states := make(map[int]*mappingState) - states[idx] = f.root +// GetMapping implements a mapping algorithm for Glob pattern +func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interface{}, string, prometheus.Labels, bool) { + matchFields := strings.Split(statsdMetric, ".") + currentState := f.root.transitions[statsdMetricType] - w.Write([]byte("digraph g {\n")) - w.Write([]byte("rankdir=LR\n")) // make it vertical - w.Write([]byte("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n")) // remove label of node + // the cursor/pointer in the backtrack stack implemented as a double-linked list + var backtrackCursor *fsmBacktrackStackCursor + resumeFromBacktrack := false - for idx < len(states) { - for field, transition := range states[idx].transitions { - states[len(states)] = transition - w.Write([]byte(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field))) - if idx == 0 { - // color for metric types - w.Write([]byte(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1))) - } else if transition.transitions == nil || len(transition.transitions) == 0 { - // color for end state - w.Write([]byte(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1))) + // the return variable + var finalState *mappingState + + captures := make(map[int]string, len(matchFields)) + // keep track of captured group so we don't need to do append() on captures + captureIdx := 0 + filedsCount := len(matchFields) + i := 0 + var state *mappingState + for { // the loop for backtracking + for { // the loop for a single "depth only" search + var present bool + // if we resume from backtrack, we should skip this branch in this case + // since the state that were saved at the end of this branch + if !resumeFromBacktrack { + if len(currentState.transitions) > 0 { + field := matchFields[i] + state, present = currentState.transitions[field] + fieldsLeft := filedsCount - i - 1 + // also compare length upfront to avoid unnecessary loop or backtrack + if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + state, present = currentState.transitions["*"] + if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { + break + } else { + captures[captureIdx] = field + captureIdx++ + } + } else if f.BacktrackingNeeded { + // if backtracking is needed, also check for alternative transition + altState, present := currentState.transitions["*"] + if !present || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength { + } else { + // push to backtracking stack + newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: altState, + fieldIndex: i, + captureIndex: captureIdx, currentCapture: field, + } + // if this is not the first time, connect to the previous cursor + if backtrackCursor != nil { + backtrackCursor.next = &newCursor + } + backtrackCursor = &newCursor + } + } + } else { + // no more transitions for this state + break + } + } // backtrack will resume from here + + // do we reach a final state? + if state.result != nil && i == filedsCount-1 { + if f.OrderingDisabled { + finalState = state + return formatLabels(finalState, captures) + } else if finalState == nil || finalState.resultPriority > state.resultPriority { + // if we care about ordering, try to find a result with highest prioity + finalState = state + } + break } + + i++ + if i >= filedsCount { + break + } + + resumeFromBacktrack = false + currentState = state + } + if backtrackCursor == nil { + // if we are not doing backtracking or all path has been travesaled + break + } else { + // pop one from stack + state = backtrackCursor.state + currentState = state + i = backtrackCursor.fieldIndex + captureIdx = backtrackCursor.captureIndex + 1 + // put the * capture back + captures[captureIdx-1] = backtrackCursor.currentCapture + backtrackCursor = backtrackCursor.prev + if backtrackCursor != nil { + // deref for GC + backtrackCursor.next = nil + } + resumeFromBacktrack = true } - idx++ } - // color for start state - w.Write([]byte(fmt.Sprintf("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];\n"))) - w.Write([]byte("}")) + + return formatLabels(finalState, captures) } -// TestIfNeedBacktracking test if backtrack is needed for current FSM -func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { - needBacktrack := false +// TestIfNeedBacktracking test if backtrack is needed for current mappings +func TestIfNeedBacktracking(mappings []string, orderingDisabled bool) bool { + backtrackingNeeded := false // A has * in rules there's other transisitions at the same state // this makes A the cause of backtracking ruleByLength := make(map[int][]string) @@ -221,9 +289,9 @@ func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { for i2, r2 := range rules { if i2 != i1 && len(re1.FindStringSubmatchIndex(r2)) > 0 { // log if we care about ordering and the superset occurs before - if !f.disableOrdering && i1 < i2 { + if !orderingDisabled && i1 < i2 { log.Warnf("match \"%s\" is a super set of match \"%s\" but in a lower order, "+ - "the first will never be matched\n", r1, r2) + "the first will never be matched", r1, r2) } currentRuleNeedBacktrack = false } @@ -242,8 +310,8 @@ func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { if currentRuleNeedBacktrack { log.Warnf("backtracking required because of match \"%s\", "+ - "matching performance may be degraded\n", r1) - needBacktrack = true + "matching performance may be degraded", r1) + backtrackingNeeded = true } } } @@ -252,112 +320,8 @@ func (f *FSM) TestIfNeedBacktracking(mappings []string) bool { // since transistions are stored in (unordered) map // note: don't move this branch to the beginning of this function // since we need logs for superset rules - f.needsBacktracking = !f.disableOrdering || needBacktrack - return f.needsBacktracking -} - -// GetMapping implements a mapping algorithm for Glob pattern -func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interface{}, string, prometheus.Labels, bool) { - matchFields := strings.Split(statsdMetric, ".") - currentState := f.root.transitions[statsdMetricType] - - // the cursor/pointer in the backtrack stack implemented as a double-linked list - var backtrackCursor *fsmBacktrackStackCursor - resumeFromBacktrack := false - - // the return variable - var finalState *mappingState - - captures := make(map[int]string, len(matchFields)) - // keep track of captured group so we don't need to do append() on captures - captureIdx := 0 - filedsCount := len(matchFields) - i := 0 - var state *mappingState - for { // the loop for backtracking - for { // the loop for a single "depth only" search - var present bool - // if we resume from backtrack, we should skip this branch in this case - // since the state that were saved at the end of this branch - if !resumeFromBacktrack { - if len(currentState.transitions) > 0 { - field := matchFields[i] - state, present = currentState.transitions[field] - fieldsLeft := filedsCount - i - 1 - // also compare length upfront to avoid unnecessary loop or backtrack - if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { - state, present = currentState.transitions["*"] - if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength { - break - } else { - captures[captureIdx] = field - captureIdx++ - } - } else if f.needsBacktracking { - // if backtracking is needed, also check for alternative transition - altState, present := currentState.transitions["*"] - if !present || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength { - } else { - // push to backtracking stack - newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: altState, - fieldIndex: i, - captureIndex: captureIdx, currentCapture: field, - } - // if this is not the first time, connect to the previous cursor - if backtrackCursor != nil { - backtrackCursor.next = &newCursor - } - backtrackCursor = &newCursor - } - } - } else { - // no more transitions for this state - break - } - } // backtrack will resume from here - - // do we reach a final state? - if state.result != nil && i == filedsCount-1 { - if f.disableOrdering { - finalState = state - return formatLabels(finalState, captures) - } else if finalState == nil || finalState.resultPriority > state.resultPriority { - // if we care about ordering, try to find a result with highest prioity - finalState = state - } - break - } - - i++ - if i >= filedsCount { - break - } - - resumeFromBacktrack = false - currentState = state - } - if backtrackCursor == nil { - // if we are not doing backtracking or all path has been travesaled - break - } else { - // pop one from stack - state = backtrackCursor.state - currentState = state - i = backtrackCursor.fieldIndex - captureIdx = backtrackCursor.captureIndex + 1 - // put the * capture back - captures[captureIdx-1] = backtrackCursor.currentCapture - backtrackCursor = backtrackCursor.prev - if backtrackCursor != nil { - // deref for GC - backtrackCursor.next = nil - } - resumeFromBacktrack = true - } - } - - return formatLabels(finalState, captures) + return !orderingDisabled || backtrackingNeeded } func formatLabels(finalState *mappingState, captures map[int]string) (interface{}, string, prometheus.Labels, bool) { diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 999c55c..e853aba 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -31,8 +31,6 @@ var ( metricLineRE = regexp.MustCompile(`^(\*\.|` + statsdMetricRE + `\.)+(\*|` + statsdMetricRE + `)$`) metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`) labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`) - - templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`) ) type mapperConfigDefaults struct { @@ -81,20 +79,6 @@ var defaultQuantiles = []metricObjective{ {Quantile: 0.99, Error: 0.001}, } -func min(x, y int) int { - if x < y { - return x - } - return y -} - -func max(x, y int) int { - if x > y { - return x - } - return y -} - func (m *MetricMapper) InitFromYAMLString(fileContents string) error { var n MetricMapper @@ -191,7 +175,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { mappings = append(mappings, mapping.Match) } } - n.FSM.TestIfNeedBacktracking(mappings) + n.FSM.BacktrackingNeeded = fsm.TestIfNeedBacktracking(mappings, n.FSM.OrderingDisabled) m.FSM = n.FSM m.doRegex = n.doRegex diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index 58e0ec0..ace47e4 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -16,7 +16,6 @@ package mapper import ( "fmt" "testing" - "time" ) type mappings map[string]struct { @@ -26,6 +25,35 @@ type mappings map[string]struct { notPresent bool } +var ( + ruleTemplateSingleMatchGlob = ` +- match: metric%d.* + name: "metric_single" + labels: + name: "$1" +` + ruleTemplateSingleMatchRegex = ` +- match: metric%d\.([^.]*) + name: "metric_single" + labels: + name: "$1" +` + + ruleTemplateMultipleMatchGlob = ` +- match: metric%d.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" +` + + ruleTemplateMultipleMatchRegex = ` +- match: metric%d\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" +` +) + func TestMetricMapperYAML(t *testing.T) { scenarios := []struct { config string @@ -615,46 +643,8 @@ func duplicateRules(count int, template string) string { } return rules } - -func TestPerformance(t *testing.T) { - sampleRulesAmount := 100 - - ruleTemplateSingleMatchGlob := ` - - match: metric%d.* - name: "metric_single" - labels: - name: "$1" -` - ruleTemplateSingleMatchRegex := ` - - match: metric%d\.([^.]*) - name: "metric_single" - labels: - name: "$1" -` - - ruleTemplateMultipleMatchGlob := ` - - match: metric%d.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" -` - - ruleTemplateMultipleMatchRegex := ` - - match: metric%d\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" -` - - scenarios := []struct { - name string - config string - configBad bool - mappings mappings - }{ - { - name: "glob", - config: `--- +func BenchmarkGlob(b *testing.B) { + config := `--- mappings: - match: test.dispatcher.*.*.succeeded name: "dispatch_events" @@ -692,26 +682,40 @@ mappings: second: "$2" third: "$3" job: "-" - `, - mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - }, + ` + mappings := mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", }, - { - name: "glob no ordering", - config: `--- + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar.baz": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobNoOrdering(b *testing.B) { + config := `--- defaults: glob_disable_ordering: true mappings: @@ -751,26 +755,40 @@ mappings: second: "$2" third: "$3" job: "-" - `, - mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - }, + ` + mappings := mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", }, - { - name: "glob with backtracking (no ordering implicated)", - config: `--- + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar.baz": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobNoOrderingWithBacktracking(b *testing.B) { + config := `--- defaults: glob_disable_ordering: true mappings: @@ -817,26 +835,40 @@ mappings: second: "$2" third: "$3" job: "-" - `, - mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - }, + ` + mappings := mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", }, - { - name: "regex", - config: `--- + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar.baz": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex(b *testing.B) { + config := `--- defaults: match_type: regex mappings: @@ -876,267 +908,511 @@ mappings: second: "$2" third: "$3" job: "-" - `, - mappings: mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - }, + ` + mappings := mappings{ + "test.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", }, - {}, - { - name: "glob single match ", - config: `--- + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { + name: "ignored-in-test-dont-set-me", + }, + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar": { + name: "ignored-in-test-dont-set-me", + }, + "foo.bar.baz": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobSingleMatch(b *testing.B) { + config := `--- mappings: - match: metric.* name: "metric_one" labels: name: "$1" - `, - mappings: mappings{ - "metric.aaa": {}, - "metric.bbb": {}, - }, - }, - { - name: "regex single match", - config: `--- + ` + mappings := mappings{ + "metric.aaa": {}, + "metric.bbb": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexSingleMatch(b *testing.B) { + config := `--- mappings: - match: metric\.([^.]*) name: "metric_one" match_type: regex labels: name: "$1" - `, - mappings: mappings{ - "metric.aaa": {}, - "metric.bbb": {}, - }, - }, - {}, - { - name: "glob multiple captures", - config: `--- -mappings: -- match: metric.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" - `, - mappings: mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - }, - }, - { - name: "regex multiple captures", - config: `--- -mappings: -- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - match_type: regex - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" - `, - mappings: mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - }, - }, - { - name: "glob multiple captures no format", - config: `--- -mappings: -- match: metric.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - name: "$1" - `, - mappings: mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - }, - }, - { - name: "regex multiple captures no expansion", - config: `--- -mappings: -- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - match_type: regex - labels: - name: "$1" - `, - mappings: mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - }, - }, - {}, - { - name: "glob multiple captures in different labels", - config: `--- -mappings: -- match: metric.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - label1: "$1" - label2: "$2" - label3: "$3" - label4: "$4" - label5: "$5" - label6: "$6" - label7: "$7" - label8: "$8" - label9: "$9" - label10: "$10" - label11: "$11" - label12: "$12" - `, - mappings: mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - }, - }, - { - name: "regex multiple captures", - config: `--- -mappings: -- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - match_type: regex - labels: - label1: "$1" - label2: "$2" - label3: "$3" - label4: "$4" - label5: "$5" - label6: "$6" - label7: "$7" - label8: "$8" - label9: "$9" - label10: "$10" - label11: "$11" - label12: "$12" - `, - mappings: mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - }, - }, - {}, - { - name: "glob 100 rules", - config: `--- -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), - mappings: mappings{ - "metric100.a": {}, - }, - }, - { - name: "glob 100 rules, no match", - config: `--- -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), - mappings: mappings{ - "metricnomatchy.a": {}, - }, - }, - { - name: "glob 100 rules without ordering, no match", - config: `--- -defaults: - glob_disable_ordering: true -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), - mappings: mappings{ - "metricnomatchy.a": {}, - }, - }, - { - name: "regex 100 rules, average case", - config: `--- -defaults: - match_type: regex -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchRegex), - mappings: mappings{ - fmt.Sprintf("metric%d.a", sampleRulesAmount/2): {}, // average match position - }, - }, - { - name: "regex 100 rules, worst case", - config: `--- -defaults: - match_type: regex -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchRegex), - mappings: mappings{ - "metric100.a": {}, - }, - }, - {}, - { - name: "glob 100 rules, multiple captures", - config: `--- -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateMultipleMatchGlob), - mappings: mappings{ - "metric50.a.b.c.d.e.f.g.h.i.j.k.l": {}, - }, - }, - { - name: "regex 100 rules, multiple captures, average case", - config: `--- -defaults: - match_type: regex -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateMultipleMatchRegex), - mappings: mappings{ - fmt.Sprintf("metric%d.a.b.c.d.e.f.g.h.i.j.k.l", sampleRulesAmount/2): {}, // average match position - }, - }, - {}, - { - name: "glob 10 rules", - config: `--- -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchGlob), - mappings: mappings{ - "metric50.a": {}, - }, - }, - { - name: "regex 10 rules average case", - config: `--- -defaults: - match_type: regex -mappings:` + duplicateRules(sampleRulesAmount, ruleTemplateSingleMatchRegex), - mappings: mappings{ - fmt.Sprintf("metric%d.a", sampleRulesAmount/2): {}, // average match position - }, - }, + ` + mappings := mappings{ + "metric.aaa": {}, + "metric.bbb": {}, } - for i, scenario := range scenarios { - if len(scenario.config) == 0 { - fmt.Println("--------------------------------") - continue - } - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(scenario.config) - if err != nil && !scenario.configBad { - t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err) - } - if err == nil && scenario.configBad { - t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config) - } + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } - var dummyMetricType MetricType = "" - start := time.Now() - for j := 1; j < 10000; j++ { - for metric, _ := range scenario.mappings { - mapper.GetMapping(metric, dummyMetricType) - } + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobMultipleCaptures(b *testing.B) { + config := `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" + ` + mappings := mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexMultipleCaptures(b *testing.B) { + config := `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" + ` + mappings := mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobMultipleCapturesNoFormat(b *testing.B) { + config := `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "not_relevant" + ` + mappings := mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexMultipleCapturesNoFormat(b *testing.B) { + config := `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + name: "not_relevant" + ` + mappings := mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobMultipleCapturesDifferentLabels(b *testing.B) { + config := `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + label1: "$1" + label2: "$2" + label3: "$3" + label4: "$4" + label5: "$5" + label6: "$6" + label7: "$7" + label8: "$8" + label9: "$9" + label10: "$10" + label11: "$11" + label12: "$12" + ` + mappings := mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexMultipleCapturesDifferentLabels(b *testing.B) { + config := `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + label1: "$1" + label2: "$2" + label3: "$3" + label4: "$4" + label5: "$5" + label6: "$6" + label7: "$7" + label8: "$8" + label9: "$9" + label10: "$10" + label11: "$11" + label12: "$12" + ` + mappings := mappings{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob10Rules(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := mappings{ + "metric100.a": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex10RulesAverage(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(10, ruleTemplateSingleMatchRegex) + mappings := mappings{ + "metric5.a": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100Rules(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := mappings{ + "metric100.a": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100RulesNoMatch(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := mappings{ + "metricnomatchy.a": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100RulesNoOrderingNoMatch(b *testing.B) { + config := `--- +defaults: + glob_disable_ordering: true +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := mappings{ + "metricnomatchy.a": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesAverage(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) + mappings := mappings{ + "metric50.a": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesWorst(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) + mappings := mappings{ + "metric100.a": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100RulesMultipleCaptures(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateMultipleMatchGlob) + mappings := mappings{ + "metric50.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesMultipleCapturesAverage(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) + mappings := mappings{ + "metric50.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesMultipleCapturesWorst(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) + mappings := mappings{ + "metric100.a.b.c.d.e.f.g.h.i.j.k.l": {}, + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) } - fmt.Printf("finished 10000 iterations for \"%s\" in %v\n", scenario.name, time.Now().Sub(start)) } } From fcf11f02e5c881c4d6d53820892942f0b8f5b820 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Fri, 28 Sep 2018 13:13:06 -0700 Subject: [PATCH 15/23] promote formatLabels to mapper package Signed-off-by: Wangchong Zhou --- pkg/mapper/fsm/formatter.go | 10 ++++---- pkg/mapper/fsm/fsm.go | 50 +++++++++---------------------------- pkg/mapper/mapper.go | 34 ++++++++++++++++++++----- 3 files changed, 45 insertions(+), 49 deletions(-) diff --git a/pkg/mapper/fsm/formatter.go b/pkg/mapper/fsm/formatter.go index 362c220..6acd91b 100644 --- a/pkg/mapper/fsm/formatter.go +++ b/pkg/mapper/fsm/formatter.go @@ -24,17 +24,17 @@ var ( templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`) ) -type templateFormatter struct { +type TemplateFormatter struct { captureIndexes []int captureCount int fmtString string } -func newTemplateFormatter(valueExpr string, captureCount int) *templateFormatter { +func NewTemplateFormatter(valueExpr string, captureCount int) *TemplateFormatter { matches := templateReplaceCaptureRE.FindAllStringSubmatch(valueExpr, -1) if len(matches) == 0 { // if no regex reference found, keep it as it is - return &templateFormatter{captureCount: 0, fmtString: valueExpr} + return &TemplateFormatter{captureCount: 0, fmtString: valueExpr} } var indexes []int @@ -51,14 +51,14 @@ func newTemplateFormatter(valueExpr string, captureCount int) *templateFormatter indexes = append(indexes, idx-1) } } - return &templateFormatter{ + return &TemplateFormatter{ captureIndexes: indexes, captureCount: len(indexes), fmtString: valueFormatter, } } -func (formatter *templateFormatter) format(captures map[int]string) string { +func (formatter *TemplateFormatter) Format(captures []string) string { if formatter.captureCount == 0 { // no label substitution, keep as it is return formatter.fmtString diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go index 0d7dd7f..8e764a9 100644 --- a/pkg/mapper/fsm/fsm.go +++ b/pkg/mapper/fsm/fsm.go @@ -17,7 +17,6 @@ import ( "regexp" "strings" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" ) @@ -26,10 +25,8 @@ type mappingState struct { minRemainingLength int maxRemainingLength int // result* members are nil unless there's a metric ends with this state - result interface{} - resultPriority int - resultNameFormatter *templateFormatter - resultLabelsFormatter map[string]*templateFormatter + Result interface{} + ResultPriority int } type fsmBacktrackStackCursor struct { @@ -69,8 +66,7 @@ func NewFSM(metricTypes []string, maxPossibleTransitions int, orderingDisabled b } // AddState adds a state into the existing FSM -func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matchMetricType string, - maxPossibleTransitions int, result interface{}) { +func (f *FSM) AddState(match string, name string, matchMetricType string, maxPossibleTransitions int, result interface{}) int { // first split by "." matchFields := strings.Split(match, ".") // fill into our FSM @@ -97,7 +93,7 @@ func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matc root.transitions[field] = state // if this is last field, set result to currentMapping instance if i == len(matchFields)-1 { - root.transitions[field].result = result + root.transitions[field].Result = result } } else { (*state).maxRemainingLength = max(len(matchFields)-i-1, (*state).maxRemainingLength) @@ -112,26 +108,18 @@ func (f *FSM) AddState(match string, name string, labels prometheus.Labels, matc } finalStates = append(finalStates, root) } - nameFmt := newTemplateFormatter(name, captureCount) - - currentLabelFormatter := make(map[string]*templateFormatter, captureCount) - for label, valueExpr := range labels { - lblFmt := newTemplateFormatter(valueExpr, captureCount) - currentLabelFormatter[label] = lblFmt - } for _, state := range finalStates { - state.resultNameFormatter = nameFmt - state.resultLabelsFormatter = currentLabelFormatter - state.resultPriority = f.statesCount + state.ResultPriority = f.statesCount } f.statesCount++ + return captureCount } // GetMapping implements a mapping algorithm for Glob pattern -func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interface{}, string, prometheus.Labels, bool) { +func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (*mappingState, []string) { matchFields := strings.Split(statsdMetric, ".") currentState := f.root.transitions[statsdMetricType] @@ -142,7 +130,7 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interfac // the return variable var finalState *mappingState - captures := make(map[int]string, len(matchFields)) + captures := make([]string, len(matchFields)) // keep track of captured group so we don't need to do append() on captures captureIdx := 0 filedsCount := len(matchFields) @@ -191,11 +179,11 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interfac } // backtrack will resume from here // do we reach a final state? - if state.result != nil && i == filedsCount-1 { + if state.Result != nil && i == filedsCount-1 { if f.OrderingDisabled { finalState = state - return formatLabels(finalState, captures) - } else if finalState == nil || finalState.resultPriority > state.resultPriority { + return finalState, captures + } else if finalState == nil || finalState.ResultPriority > state.ResultPriority { // if we care about ordering, try to find a result with highest prioity finalState = state } @@ -230,7 +218,7 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (interfac } } - return formatLabels(finalState, captures) + return finalState, captures } // TestIfNeedBacktracking test if backtrack is needed for current mappings @@ -323,17 +311,3 @@ func TestIfNeedBacktracking(mappings []string, orderingDisabled bool) bool { return !orderingDisabled || backtrackingNeeded } - -func formatLabels(finalState *mappingState, captures map[int]string) (interface{}, string, prometheus.Labels, bool) { - // format name and labels - if finalState != nil { - name := finalState.resultNameFormatter.format(captures) - - labels := prometheus.Labels{} - for key, formatter := range finalState.resultLabelsFormatter { - labels[key] = formatter.format(captures) - } - return finalState.result, name, labels, true - } - return nil, "", nil, false -} diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index e853aba..dc5a735 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -57,8 +57,11 @@ type matchMetricType string type MetricMapping struct { Match string `yaml:"match"` Name string `yaml:"name"` + nameFormatter *fsm.TemplateFormatter regex *regexp.Regexp Labels prometheus.Labels `yaml:"labels"` + labelKeys []string + labelFormatters []*fsm.TemplateFormatter TimerType TimerType `yaml:"timer_type"` Buckets []float64 `yaml:"buckets"` Quantiles []metricObjective `yaml:"quantiles"` @@ -137,9 +140,22 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { return fmt.Errorf("invalid match: %s", currentMapping.Match) } - n.FSM.AddState(currentMapping.Match, currentMapping.Name, currentMapping.Labels, string(currentMapping.MatchMetricType), + captureCount := n.FSM.AddState(currentMapping.Match, currentMapping.Name, string(currentMapping.MatchMetricType), remainingMappingsCount, currentMapping) + currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount) + + labelKeys := make([]string, len(currentMapping.Labels)) + labelFormatters := make([]*fsm.TemplateFormatter, len(currentMapping.Labels)) + labelIndex := 0 + for label, valueExpr := range currentMapping.Labels { + labelKeys[labelIndex] = label + labelFormatters[labelIndex] = fsm.NewTemplateFormatter(valueExpr, captureCount) + labelIndex++ + } + currentMapping.labelFormatters = labelFormatters + currentMapping.labelKeys = labelKeys + } else { if regex, err := regexp.Compile(currentMapping.Match); err != nil { return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err) @@ -200,11 +216,17 @@ func (m *MetricMapper) InitFromFile(fileName string) error { func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) { // glob matching if m.doFSM { - mapping, mappingName, labels, matched := m.FSM.GetMapping(statsdMetric, string(statsdMetricType)) - if matched { - mapping.(*MetricMapping).Name = mappingName - return mapping.(*MetricMapping), labels, matched - } else if !matched && !m.doRegex { + finalState, captures := m.FSM.GetMapping(statsdMetric, string(statsdMetricType)) + if finalState != nil && finalState.Result != nil { + result := finalState.Result.(*MetricMapping) + result.Name = result.nameFormatter.Format(captures) + + labels := prometheus.Labels{} + for index, formatter := range result.labelFormatters { + labels[result.labelKeys[index]] = formatter.Format(captures) + } + return result, labels, true + } else if !m.doRegex { // if there's no regex match type, return immediately return nil, nil, false } From 699fa13c8cd76da444641ca14c0bb75ec244adf2 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Fri, 28 Sep 2018 14:42:09 -0700 Subject: [PATCH 16/23] add benchmark test to ci Signed-off-by: Wangchong Zhou --- Makefile.common | 5 + pkg/mapper/mapper_benchmark_test.go | 797 +++++++++++++++++++++++++++ pkg/mapper/mapper_test.go | 810 ---------------------------- 3 files changed, 802 insertions(+), 810 deletions(-) create mode 100644 pkg/mapper/mapper_benchmark_test.go diff --git a/Makefile.common b/Makefile.common index eaee9f0..528d404 100644 --- a/Makefile.common +++ b/Makefile.common @@ -66,6 +66,11 @@ test: @echo ">> running all tests" $(GO) test -race $(pkgs) +.PHONY: bench +bench: + @echo ">> running all benchmarks" + $(GO) test -bench . -race $(pkgs) + .PHONY: format format: @echo ">> formatting code" diff --git a/pkg/mapper/mapper_benchmark_test.go b/pkg/mapper/mapper_benchmark_test.go new file mode 100644 index 0000000..1511ee2 --- /dev/null +++ b/pkg/mapper/mapper_benchmark_test.go @@ -0,0 +1,797 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper + +import ( + "fmt" + "testing" +) + +var ( + ruleTemplateSingleMatchGlob = ` +- match: metric%d.* + name: "metric_single" + labels: + name: "$1" +` + ruleTemplateSingleMatchRegex = ` +- match: metric%d\.([^.]*) + name: "metric_single" + labels: + name: "$1" +` + + ruleTemplateMultipleMatchGlob = ` +- match: metric%d.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" +` + + ruleTemplateMultipleMatchRegex = ` +- match: metric%d\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" +` +) + +func duplicateRules(count int, template string) string { + rules := "" + for i := 0; i < count; i++ { + rules += fmt.Sprintf(template, i) + } + return rules +} + +func BenchmarkGlob(b *testing.B) { + config := `--- +mappings: +- match: test.dispatcher.*.*.succeeded + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "succeeded" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher.*.*.* + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* + name: "tyk_http_request" + labels: + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" +- match: "*.*" + name: "catchall" + labels: + first: "$1" + second: "$2" + third: "$3" + job: "-" + ` + mappings := []string{ + "test.dispatcher.FooProcessor.send.succeeded", + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded", + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.", + "foo.bar", + "foo.bar.baz", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobNoOrdering(b *testing.B) { + config := `--- +defaults: + glob_disable_ordering: true +mappings: +- match: test.dispatcher.*.*.succeeded + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "succeeded" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher.*.*.* + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* + name: "tyk_http_request" + labels: + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" +- match: "*.*" + name: "catchall" + labels: + first: "$1" + second: "$2" + third: "$3" + job: "-" + ` + mappings := []string{ + "test.dispatcher.FooProcessor.send.succeeded", + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded", + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.", + "foo.bar", + "foo.bar.baz", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobNoOrderingWithBacktracking(b *testing.B) { + config := `--- +defaults: + glob_disable_ordering: true +mappings: +- match: test.dispatcher.*.*.succeeded + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "succeeded" + job: "test_dispatcher" +- match: test.dispatcher.*.received.* + name: "dispatch_events_wont_match" + labels: + processor: "$1" + action: "received" + result: "$2" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher.*.*.* + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* + name: "tyk_http_request" + labels: + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" +- match: "*.*" + name: "catchall" + labels: + first: "$1" + second: "$2" + third: "$3" + job: "-" + ` + mappings := []string{ + "test.dispatcher.FooProcessor.send.succeeded", + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded", + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.", + "foo.bar", + "foo.bar.baz", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings: +- match: test\.dispatcher\.([^.]*)\.([^.]*)\.([^.]*) + name: "dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: test.my-dispatch-host01.name.dispatcher\.([^.]*)\.([^.]*)\.([^.]*) + name: "host_dispatch_events" + labels: + processor: "$1" + action: "$2" + result: "$3" + job: "test_dispatcher" +- match: request_time\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "tyk_http_request" + labels: + method_and_path: "${1}" + response_code: "${2}" + apikey: "${3}" + apiversion: "${4}" + apiname: "${5}" + apiid: "${6}" + ipv4_t1: "${7}" + ipv4_t2: "${8}" + ipv4_t3: "${9}" + ipv4_t4: "${10}" + orgid: "${11}" + oauthid: "${12}" +- match: \.([^.]*)\.([^.]*) + name: "catchall" + labels: + first: "$1" + second: "$2" + third: "$3" + job: "-" + ` + mappings := []string{ + "test.dispatcher.FooProcessor.send.succeeded", + "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded", + "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.", + "foo.bar", + "foo.bar.baz", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobSingleMatch(b *testing.B) { + config := `--- +mappings: +- match: metric.* + name: "metric_one" + labels: + name: "$1" + ` + mappings := []string{ + "metric.aaa", + "metric.bbb", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexSingleMatch(b *testing.B) { + config := `--- +mappings: +- match: metric\.([^.]*) + name: "metric_one" + match_type: regex + labels: + name: "$1" + ` + mappings := []string{ + "metric.aaa", + "metric.bbb", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobMultipleCaptures(b *testing.B) { + config := `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" + ` + mappings := []string{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexMultipleCaptures(b *testing.B) { + config := `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" + ` + mappings := []string{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobMultipleCapturesNoFormat(b *testing.B) { + config := `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + name: "not_relevant" + ` + mappings := []string{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexMultipleCapturesNoFormat(b *testing.B) { + config := `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + name: "not_relevant" + ` + mappings := []string{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlobMultipleCapturesDifferentLabels(b *testing.B) { + config := `--- +mappings: +- match: metric.*.*.*.*.*.*.*.*.*.*.*.* + name: "metric_multi" + labels: + label1: "$1" + label2: "$2" + label3: "$3" + label4: "$4" + label5: "$5" + label6: "$6" + label7: "$7" + label8: "$8" + label9: "$9" + label10: "$10" + label11: "$11" + label12: "$12" + ` + mappings := []string{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegexMultipleCapturesDifferentLabels(b *testing.B) { + config := `--- +mappings: +- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) + name: "metric_multi" + match_type: regex + labels: + label1: "$1" + label2: "$2" + label3: "$3" + label4: "$4" + label5: "$5" + label6: "$6" + label7: "$7" + label8: "$8" + label9: "$9" + label10: "$10" + label11: "$11" + label12: "$12" + ` + mappings := []string{ + "metric.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob10Rules(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := []string{ + "metric100.a", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex10RulesAverage(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(10, ruleTemplateSingleMatchRegex) + mappings := []string{ + "metric5.a", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100Rules(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := []string{ + "metric100.a", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100RulesNoMatch(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := []string{ + "metricnomatchy.a", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100RulesNoOrderingNoMatch(b *testing.B) { + config := `--- +defaults: + glob_disable_ordering: true +mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) + mappings := []string{ + "metricnomatchy.a", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesAverage(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) + mappings := []string{ + "metric50.a", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesWorst(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) + mappings := []string{ + "metric100.a", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkGlob100RulesMultipleCaptures(b *testing.B) { + config := `--- +mappings:` + duplicateRules(100, ruleTemplateMultipleMatchGlob) + mappings := []string{ + "metric50.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesMultipleCapturesAverage(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) + mappings := []string{ + "metric50.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} + +func BenchmarkRegex100RulesMultipleCapturesWorst(b *testing.B) { + config := `--- +defaults: + match_type: regex +mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) + mappings := []string{ + "metric100.a.b.c.d.e.f.g.h.i.j.k.l", + } + + mapper := MetricMapper{} + err := mapper.InitFromYAMLString(config) + if err != nil { + b.Fatalf("Config load error: %s %s", config, err) + } + + var dummyMetricType MetricType + b.ResetTimer() + for j := 0; j < b.N; j++ { + for _, metric := range mappings { + mapper.GetMapping(metric, dummyMetricType) + } + } +} diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index ace47e4..2a66259 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -14,7 +14,6 @@ package mapper import ( - "fmt" "testing" ) @@ -25,35 +24,6 @@ type mappings map[string]struct { notPresent bool } -var ( - ruleTemplateSingleMatchGlob = ` -- match: metric%d.* - name: "metric_single" - labels: - name: "$1" -` - ruleTemplateSingleMatchRegex = ` -- match: metric%d\.([^.]*) - name: "metric_single" - labels: - name: "$1" -` - - ruleTemplateMultipleMatchGlob = ` -- match: metric%d.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" -` - - ruleTemplateMultipleMatchRegex = ` -- match: metric%d\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" -` -) - func TestMetricMapperYAML(t *testing.T) { scenarios := []struct { config string @@ -636,786 +606,6 @@ mappings: } } -func duplicateRules(count int, template string) string { - rules := "" - for i := 0; i < count; i++ { - rules += fmt.Sprintf(template, i) - } - return rules -} -func BenchmarkGlob(b *testing.B) { - config := `--- -mappings: -- match: test.dispatcher.*.*.succeeded - name: "dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "succeeded" - job: "test_dispatcher" -- match: test.my-dispatch-host01.name.dispatcher.*.*.* - name: "host_dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "$3" - job: "test_dispatcher" -- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* - name: "tyk_http_request" - labels: - method_and_path: "${1}" - response_code: "${2}" - apikey: "${3}" - apiversion: "${4}" - apiname: "${5}" - apiid: "${6}" - ipv4_t1: "${7}" - ipv4_t2: "${8}" - ipv4_t3: "${9}" - ipv4_t4: "${10}" - orgid: "${11}" - oauthid: "${12}" -- match: "*.*" - name: "catchall" - labels: - first: "$1" - second: "$2" - third: "$3" - job: "-" - ` - mappings := mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlobNoOrdering(b *testing.B) { - config := `--- -defaults: - glob_disable_ordering: true -mappings: -- match: test.dispatcher.*.*.succeeded - name: "dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "succeeded" - job: "test_dispatcher" -- match: test.my-dispatch-host01.name.dispatcher.*.*.* - name: "host_dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "$3" - job: "test_dispatcher" -- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* - name: "tyk_http_request" - labels: - method_and_path: "${1}" - response_code: "${2}" - apikey: "${3}" - apiversion: "${4}" - apiname: "${5}" - apiid: "${6}" - ipv4_t1: "${7}" - ipv4_t2: "${8}" - ipv4_t3: "${9}" - ipv4_t4: "${10}" - orgid: "${11}" - oauthid: "${12}" -- match: "*.*" - name: "catchall" - labels: - first: "$1" - second: "$2" - third: "$3" - job: "-" - ` - mappings := mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlobNoOrderingWithBacktracking(b *testing.B) { - config := `--- -defaults: - glob_disable_ordering: true -mappings: -- match: test.dispatcher.*.*.succeeded - name: "dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "succeeded" - job: "test_dispatcher" -- match: test.dispatcher.*.received.* - name: "dispatch_events_wont_match" - labels: - processor: "$1" - action: "received" - result: "$2" - job: "test_dispatcher" -- match: test.my-dispatch-host01.name.dispatcher.*.*.* - name: "host_dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "$3" - job: "test_dispatcher" -- match: request_time.*.*.*.*.*.*.*.*.*.*.*.* - name: "tyk_http_request" - labels: - method_and_path: "${1}" - response_code: "${2}" - apikey: "${3}" - apiversion: "${4}" - apiname: "${5}" - apiid: "${6}" - ipv4_t1: "${7}" - ipv4_t2: "${8}" - ipv4_t3: "${9}" - ipv4_t4: "${10}" - orgid: "${11}" - oauthid: "${12}" -- match: "*.*" - name: "catchall" - labels: - first: "$1" - second: "$2" - third: "$3" - job: "-" - ` - mappings := mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegex(b *testing.B) { - config := `--- -defaults: - match_type: regex -mappings: -- match: test\.dispatcher\.([^.]*)\.([^.]*)\.([^.]*) - name: "dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "$3" - job: "test_dispatcher" -- match: test.my-dispatch-host01.name.dispatcher\.([^.]*)\.([^.]*)\.([^.]*) - name: "host_dispatch_events" - labels: - processor: "$1" - action: "$2" - result: "$3" - job: "test_dispatcher" -- match: request_time\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "tyk_http_request" - labels: - method_and_path: "${1}" - response_code: "${2}" - apikey: "${3}" - apiversion: "${4}" - apiname: "${5}" - apiid: "${6}" - ipv4_t1: "${7}" - ipv4_t2: "${8}" - ipv4_t3: "${9}" - ipv4_t4: "${10}" - orgid: "${11}" - oauthid: "${12}" -- match: \.([^.]*)\.([^.]*) - name: "catchall" - labels: - first: "$1" - second: "$2" - third: "$3" - job: "-" - ` - mappings := mappings{ - "test.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": { - name: "ignored-in-test-dont-set-me", - }, - "request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar": { - name: "ignored-in-test-dont-set-me", - }, - "foo.bar.baz": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlobSingleMatch(b *testing.B) { - config := `--- -mappings: -- match: metric.* - name: "metric_one" - labels: - name: "$1" - ` - mappings := mappings{ - "metric.aaa": {}, - "metric.bbb": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegexSingleMatch(b *testing.B) { - config := `--- -mappings: -- match: metric\.([^.]*) - name: "metric_one" - match_type: regex - labels: - name: "$1" - ` - mappings := mappings{ - "metric.aaa": {}, - "metric.bbb": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlobMultipleCaptures(b *testing.B) { - config := `--- -mappings: -- match: metric.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" - ` - mappings := mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegexMultipleCaptures(b *testing.B) { - config := `--- -mappings: -- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - match_type: regex - labels: - name: "$1-$2-$3.$4-$5-$6.$7-$8-$9.$10-$11-$12" - ` - mappings := mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlobMultipleCapturesNoFormat(b *testing.B) { - config := `--- -mappings: -- match: metric.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - name: "not_relevant" - ` - mappings := mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegexMultipleCapturesNoFormat(b *testing.B) { - config := `--- -mappings: -- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - match_type: regex - labels: - name: "not_relevant" - ` - mappings := mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlobMultipleCapturesDifferentLabels(b *testing.B) { - config := `--- -mappings: -- match: metric.*.*.*.*.*.*.*.*.*.*.*.* - name: "metric_multi" - labels: - label1: "$1" - label2: "$2" - label3: "$3" - label4: "$4" - label5: "$5" - label6: "$6" - label7: "$7" - label8: "$8" - label9: "$9" - label10: "$10" - label11: "$11" - label12: "$12" - ` - mappings := mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegexMultipleCapturesDifferentLabels(b *testing.B) { - config := `--- -mappings: -- match: metric\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*)\.([^.]*) - name: "metric_multi" - match_type: regex - labels: - label1: "$1" - label2: "$2" - label3: "$3" - label4: "$4" - label5: "$5" - label6: "$6" - label7: "$7" - label8: "$8" - label9: "$9" - label10: "$10" - label11: "$11" - label12: "$12" - ` - mappings := mappings{ - "metric.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlob10Rules(b *testing.B) { - config := `--- -mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) - mappings := mappings{ - "metric100.a": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegex10RulesAverage(b *testing.B) { - config := `--- -defaults: - match_type: regex -mappings:` + duplicateRules(10, ruleTemplateSingleMatchRegex) - mappings := mappings{ - "metric5.a": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlob100Rules(b *testing.B) { - config := `--- -mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) - mappings := mappings{ - "metric100.a": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlob100RulesNoMatch(b *testing.B) { - config := `--- -mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) - mappings := mappings{ - "metricnomatchy.a": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlob100RulesNoOrderingNoMatch(b *testing.B) { - config := `--- -defaults: - glob_disable_ordering: true -mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) - mappings := mappings{ - "metricnomatchy.a": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegex100RulesAverage(b *testing.B) { - config := `--- -defaults: - match_type: regex -mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) - mappings := mappings{ - "metric50.a": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegex100RulesWorst(b *testing.B) { - config := `--- -defaults: - match_type: regex -mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) - mappings := mappings{ - "metric100.a": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkGlob100RulesMultipleCaptures(b *testing.B) { - config := `--- -mappings:` + duplicateRules(100, ruleTemplateMultipleMatchGlob) - mappings := mappings{ - "metric50.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegex100RulesMultipleCapturesAverage(b *testing.B) { - config := `--- -defaults: - match_type: regex -mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) - mappings := mappings{ - "metric50.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - -func BenchmarkRegex100RulesMultipleCapturesWorst(b *testing.B) { - config := `--- -defaults: - match_type: regex -mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) - mappings := mappings{ - "metric100.a.b.c.d.e.f.g.h.i.j.k.l": {}, - } - - mapper := MetricMapper{} - err := mapper.InitFromYAMLString(config) - if err != nil { - b.Fatalf("Config load error: %s %s", config, err) - } - - var dummyMetricType MetricType - b.ResetTimer() - for j := 0; j < b.N; j++ { - for metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) - } - } -} - func TestAction(t *testing.T) { scenarios := []struct { config string From c10e80c44bba51ddd3afa9e1536c36b9af7b2f13 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Wed, 3 Oct 2018 16:30:01 -0700 Subject: [PATCH 17/23] optimizations Signed-off-by: Wangchong Zhou --- pkg/mapper/fsm/dump.go | 2 +- pkg/mapper/fsm/formatter.go | 12 ++++-- pkg/mapper/fsm/fsm.go | 29 +++++++++---- pkg/mapper/mapper.go | 2 +- pkg/mapper/mapper_benchmark_test.go | 66 ++++++++++------------------- pkg/mapper/mapper_test.go | 5 ++- 6 files changed, 55 insertions(+), 61 deletions(-) diff --git a/pkg/mapper/fsm/dump.go b/pkg/mapper/fsm/dump.go index 1bf4675..d91e2cf 100644 --- a/pkg/mapper/fsm/dump.go +++ b/pkg/mapper/fsm/dump.go @@ -18,7 +18,7 @@ import ( "io" ) -// DumpFSM accepts a io.writer and write the current FSM into dot file format +// DumpFSM accepts a io.writer and write the current FSM into dot file format. func (f *FSM) DumpFSM(w io.Writer) { idx := 0 states := make(map[int]*mappingState) diff --git a/pkg/mapper/fsm/formatter.go b/pkg/mapper/fsm/formatter.go index 6acd91b..8ae0926 100644 --- a/pkg/mapper/fsm/formatter.go +++ b/pkg/mapper/fsm/formatter.go @@ -30,15 +30,17 @@ type TemplateFormatter struct { fmtString string } -func NewTemplateFormatter(valueExpr string, captureCount int) *TemplateFormatter { - matches := templateReplaceCaptureRE.FindAllStringSubmatch(valueExpr, -1) +// NewTemplateFormatter instantialize a TemplateFormatter +// from given template string and the maxium amount of captures. +func NewTemplateFormatter(template string, captureCount int) *TemplateFormatter { + matches := templateReplaceCaptureRE.FindAllStringSubmatch(template, -1) if len(matches) == 0 { // if no regex reference found, keep it as it is - return &TemplateFormatter{captureCount: 0, fmtString: valueExpr} + return &TemplateFormatter{captureCount: 0, fmtString: template} } var indexes []int - valueFormatter := valueExpr + valueFormatter := template for _, match := range matches { idx, err := strconv.Atoi(match[len(match)-1]) if err != nil || idx > captureCount || idx < 1 { @@ -58,6 +60,8 @@ func NewTemplateFormatter(valueExpr string, captureCount int) *TemplateFormatter } } +// Format accepts a list containing captured strings and return the formatted string +// using the template stored in current TemplateFormatter. func (formatter *TemplateFormatter) Format(captures []string) string { if formatter.captureCount == 0 { // no label substitution, keep as it is diff --git a/pkg/mapper/fsm/fsm.go b/pkg/mapper/fsm/fsm.go index 8e764a9..263dba8 100644 --- a/pkg/mapper/fsm/fsm.go +++ b/pkg/mapper/fsm/fsm.go @@ -52,7 +52,6 @@ func NewFSM(metricTypes []string, maxPossibleTransitions int, orderingDisabled b root := &mappingState{} root.transitions = make(map[string]*mappingState, len(metricTypes)) - metricTypes = append(metricTypes, "") for _, field := range metricTypes { state := &mappingState{} (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) @@ -65,14 +64,17 @@ func NewFSM(metricTypes []string, maxPossibleTransitions int, orderingDisabled b return &fsm } -// AddState adds a state into the existing FSM -func (f *FSM) AddState(match string, name string, matchMetricType string, maxPossibleTransitions int, result interface{}) int { +// AddState adds a mapping rule into the existing FSM. +// The maxPossibleTransitions parameter sets the expected count of transitions left. +// The result parameter sets the generic type to be returned when fsm found a match in GetMapping. +func (f *FSM) AddState(match string, matchMetricType string, maxPossibleTransitions int, result interface{}) int { // first split by "." matchFields := strings.Split(match, ".") // fill into our FSM roots := []*mappingState{} + // first state is the metric type if matchMetricType == "" { - // if metricType not specified, connect the state from all three types + // if metricType not specified, connect the start state from all three types for _, metricType := range f.metricTypes { roots = append(roots, f.root.transitions[string(metricType)]) } @@ -81,11 +83,14 @@ func (f *FSM) AddState(match string, name string, matchMetricType string, maxPos } var captureCount int var finalStates []*mappingState + // iterating over different start state (different metric types) for _, root := range roots { captureCount = 0 + // for each start state, connect from start state to end state for i, field := range matchFields { state, prs := root.transitions[field] if !prs { + // create a state if it's not exist in the fsm state = &mappingState{} (*state).transitions = make(map[string]*mappingState, maxPossibleTransitions) (*state).maxRemainingLength = len(matchFields) - i - 1 @@ -118,7 +123,9 @@ func (f *FSM) AddState(match string, name string, matchMetricType string, maxPos return captureCount } -// GetMapping implements a mapping algorithm for Glob pattern +// GetMapping using the fsm to find matching rules according to given statsdMetric and statsdMetricType. +// If it finds a match, the final state and the captured strings are returned; +// if there's no match found, nil and a empty list will be returned. func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (*mappingState, []string) { matchFields := strings.Split(statsdMetric, ".") currentState := f.root.transitions[statsdMetricType] @@ -156,7 +163,7 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (*mapping captureIdx++ } } else if f.BacktrackingNeeded { - // if backtracking is needed, also check for alternative transition + // if backtracking is needed, also check for alternative transition, i.e. * altState, present := currentState.transitions["*"] if !present || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength { } else { @@ -221,10 +228,11 @@ func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (*mapping return finalState, captures } -// TestIfNeedBacktracking test if backtrack is needed for current mappings +// TestIfNeedBacktracking tests if backtrack is needed for given list of mappings +// and whether ordering is disabled. func TestIfNeedBacktracking(mappings []string, orderingDisabled bool) bool { backtrackingNeeded := false - // A has * in rules there's other transisitions at the same state + // A has * in rules, but there's other transisitions at the same state, // this makes A the cause of backtracking ruleByLength := make(map[int][]string) ruleREByLength := make(map[int][]*regexp.Regexp) @@ -255,11 +263,14 @@ func TestIfNeedBacktracking(mappings []string, orderingDisabled bool) bool { if re1 == nil || strings.Index(r1, "*") == -1 { continue } - // if a rule is A.B.C.*.E.*, is there a rule A.B.C.D.x.x or A.B.C.*.E.F? (x is any string or *) + // if rule r1 is A.B.C.*.E.*, is there a rule r2 is A.B.C.D.x.x or A.B.C.*.E.F ? (x is any string or *) + // if such r2 exists, then to match r1 we will need backtracking for index := 0; index < len(r1); index++ { if r1[index] != '*' { continue } + // translate the substring of r1 from 0 to the index of current * into regex + // A.B.C.*.E.* will becomes ^A\.B\.C\. and ^A\.B\.C\.\*\.E\. reStr := strings.Replace(r1[:index], ".", "\\.", -1) reStr = strings.Replace(reStr, "*", "\\*", -1) re := regexp.MustCompile("^" + reStr) diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index dc5a735..46e90e6 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -140,7 +140,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string) error { return fmt.Errorf("invalid match: %s", currentMapping.Match) } - captureCount := n.FSM.AddState(currentMapping.Match, currentMapping.Name, string(currentMapping.MatchMetricType), + captureCount := n.FSM.AddState(currentMapping.Match, string(currentMapping.MatchMetricType), remainingMappingsCount, currentMapping) currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount) diff --git a/pkg/mapper/mapper_benchmark_test.go b/pkg/mapper/mapper_benchmark_test.go index 1511ee2..0388c59 100644 --- a/pkg/mapper/mapper_benchmark_test.go +++ b/pkg/mapper/mapper_benchmark_test.go @@ -109,11 +109,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -174,11 +173,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -246,11 +244,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -311,11 +308,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -339,11 +335,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -368,11 +363,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -395,11 +389,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -423,11 +416,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -450,11 +442,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -478,11 +469,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -516,11 +506,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -555,11 +544,10 @@ mappings: b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -577,11 +565,10 @@ mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -601,11 +588,10 @@ mappings:` + duplicateRules(10, ruleTemplateSingleMatchRegex) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -623,11 +609,10 @@ mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -645,11 +630,10 @@ mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -669,11 +653,10 @@ mappings:` + duplicateRules(100, ruleTemplateSingleMatchGlob) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -693,11 +676,10 @@ mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -717,11 +699,10 @@ mappings:` + duplicateRules(100, ruleTemplateSingleMatchRegex) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -739,11 +720,10 @@ mappings:` + duplicateRules(100, ruleTemplateMultipleMatchGlob) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -763,11 +743,10 @@ mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } @@ -787,11 +766,10 @@ mappings:` + duplicateRules(100, ruleTemplateMultipleMatchRegex) b.Fatalf("Config load error: %s %s", config, err) } - var dummyMetricType MetricType b.ResetTimer() for j := 0; j < b.N; j++ { for _, metric := range mappings { - mapper.GetMapping(metric, dummyMetricType) + mapper.GetMapping(metric, MetricTypeCounter) } } } diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index 2a66259..64fb73d 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -571,9 +571,10 @@ mappings: t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config) } - var dummyMetricType MetricType = "" for metric, mapping := range scenario.mappings { - m, labels, present := mapper.GetMapping(metric, dummyMetricType) + // exporter will call mapper.GetMapping with valid MetricType + // so we also pass a sane MetricType in testing + m, labels, present := mapper.GetMapping(metric, MetricTypeCounter) if present && mapping.name != "" && m.Name != mapping.name { t.Fatalf("%d.%q: Expected name %v, got %v", i, metric, m.Name, mapping.name) } From 4d9ce8c70a055ad299089417474a56d058852578 Mon Sep 17 00:00:00 2001 From: Wangchong Zhou Date: Wed, 3 Oct 2018 16:30:12 -0700 Subject: [PATCH 18/23] add readmes Signed-off-by: Wangchong Zhou --- README.md | 42 ++++++++++++- pkg/mapper/fsm/README.md | 126 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 pkg/mapper/fsm/README.md diff --git a/README.md b/README.md index b541822..a40f738 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ NOTE: Version 0.7.0 switched to the [kingpin](https://github.com/alecthomas/king read buffer associated with the UDP connection. Please make sure the kernel parameters net.core.rmem_max is set to a value greater than the value specified. + --debug.dump-fsm="" The path to dump internal FSM generated for glob + matching as Dot file. --log.level="info" Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal] --log.format="logger:stderr" @@ -181,6 +183,8 @@ mappings: code: "$1" ``` +### StatsD timers + By default, statsd timers are represented as a Prometheus summary with quantiles. You may optionally configure the [quantiles and acceptable error](https://prometheus.io/docs/practices/histograms/#quantiles): @@ -223,6 +227,8 @@ mappings: job: "${1}_server" ``` +### Regular expression matching + Another capability when using YAML configuration is the ability to define matches using raw regular expressions as opposed to the default globbing style of match. This may allow for pulling structured data from otherwise poorly named statsd @@ -249,14 +255,19 @@ automatically. only used when the statsd metric type is a timerand the `timer_type` is set to "histogram." +### Global defaults + One may also set defaults for the timer type, buckets or quantiles, and match_type. These will be used by all mappings that do not define these. +An option that can only be configured in `defaults` is `glob_disable_ordering`, which is `false` if omitted. By setting this to `true`, `glob` match type will not honor the occurance of rules in the mapping rules file and always treat `*` as lower priority than a general string. + ```yaml defaults: timer_type: histogram buckets: [.005, .01, .025, .05, .1, .25, .5, 1, 2.5 ] match_type: glob + glob_disable_ordering: false mappings: # This will be a histogram using the buckets set in `defaults`. - match: test.timing.*.*.* @@ -275,7 +286,34 @@ mappings: job: "${1}_server_other" ``` -You may also drop metrics by specifying a "drop" action on a match. For example: +### Choose between glob or regex match type + +Despite from the missing flexibility of using regular expression in mapping and +formatting labels, `glob` matching is optimized to have better performance than +`regex` in certain use cases. In short, glob will have best performance if the +rules amount is not so less and captures (using of *) is not to much in a +single rule. Whether disabling ordering in glob or not won't have a noticable +effect on performance in general use cases. In edge cases like the below however, +disabling ordering will be beneficial: + + a.*.*.*.* + a.b.*.*.* + a.b.c.*.* + a.b.c.d.* + +The reason is the list assignment of captures (using of *) is the most +expensive operation in glob. Honoring ordering will result fsm to do 10 +times of list assignment at most, while disabling ordering it will need +only 4 at most. + +See also [pkg/mapper/fsm/README.md](pkg/mapper/fsm/README.md). +Also running `go test -bench .` in **pkg/mapper** directory will produce +a detailed comparation between the two match type. + +### `drop` action + +You may also drop metrics by specifying a "drop" action on a match. For +example: ```yaml mappings: @@ -296,6 +334,8 @@ mappings: You can drop any metric using the normal match syntax. The default action is "map" which does the normal metrics mapping. +### Explicit metric type mapping + StatsD allows emitting of different metric types under the same metric name, but the Prometheus client library can't merge those. For this use-case the mapping definition allows you to specify which metric type to match: diff --git a/pkg/mapper/fsm/README.md b/pkg/mapper/fsm/README.md new file mode 100644 index 0000000..156c9f6 --- /dev/null +++ b/pkg/mapper/fsm/README.md @@ -0,0 +1,126 @@ +# FSM Mapping + +## Overview + +This package implements a fast and efficient algorithm for generic glob style +string matching using finite state machine (FSM). + +### Source Hierachy + +``` + '-- fsm + '-- dump.go // functionality to dump the FSM to Dot file + '-- formatter.go // format glob templates using captured * groups + '-- fsm.go // manipulating and searching of FSM + '-- minmax.go // min() max() function for interger +``` + +## FSM Explained + +Per [Wikipedia](https://en.wikipedia.org/wiki/Finite-state_machine): + + A finite-state machine (FSM) or finite-state automaton (FSA, plural: automata), finite automaton, or simply a state machine, is a mathematical model of computation. It is an abstract machine that can be in exactly one of a finite number of states at any given time. The FSM can change from one state to another in response to some external inputs; the change from one state to another is called a transition. An FSM is defined by a list of its states, its initial state, and the conditions for each transition. + +In our use case, each *state* is a substring after the input StatsD metric name is splitted by `.`. + +### Add state to FSM + +`func (f *FSM) AddState(match string, matchMetricType string, +maxPossibleTransitions int, result interface{}) int` + +At first, the fsm only contains three states, representing three possible metric types: + + ____ [gauge] + / + (start)---- [counter] + \ + '--- [ timer ] + + +Adding a rule `client.*.request.count` with type `counter` will make the fsm to be: + + + ____ [gauge] + / + (start)---- [counter] -- [client] -- [*] -- [request] -- [count] -- {R1} + \ + '--- [timer] + +`{R1}` is short for result 1, which is the match result for `client.*.request.count`. + +Adding a rule `client.*.*.size` with type `counter` will make the fsm to be: + + ____ [gauge] __ [request] -- [count] -- {R1} + / / + (start)---- [counter] -- [client] -- [*] + \ \__ [*] -- [size] -- {R2} + '--- [timer] + + +### Finding a result state in FSM + +`func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) +(*mappingState, []string)` + +For example, try to map `client.aaa.request.count` with `counter` type in the +fsm, the `^1` to `^7` symbols indicate how fsm will traversal in its tree: + + + ____ [gauge] __ [request] -- [count] -- {R1} + / / ^5 ^6 ^7 + (start)---- [counter] -- [client] -- [*] + ^1 \ ^2 ^3 \__ [*] -- [size] -- {R2} + '--- [timer] ^4 + + +To map `client.bbb.request.size`, fsm will do a backtracking: + + + ____ [gauge] __ [request] -- [count] -- {R1} + / / ^5 ^6 + (start)---- [counter] -- [client] -- [*] + ^1 \ ^2 ^3 \__ [*] -- [size] -- {R2} + '--- [timer] ^4 + ^7 ^8 ^9 + + +## Debugging + +To see all the states of the current FSM, use `func (f *FSM) DumpFSM(w io.Writer)` +to dump into a Dot file. The Dot file can be further renderer into image using: + +```shell +$ dot -Tpng dump.dot > dump.png +``` + +In StatsD exporter, one could use the following: + +```shell +$ statsd_exporter --statsd.mapping-config=statsd.rules --debug.dump-fsm=dump.dot +$ dot -Tpng dump.dot > dump.png +``` + +For example, the following rules: + +```yaml +mappings: +- match: client.*.request.count + name: request_count + match_metric_type: counter + labels: + client: $1 + +- match: client.*.*.size + name: sizes + match_metric_type: counter + labels: + client: $1 + direction: $2 +``` + +will be rendered as: +![fsm](https://i.imgur.com/Wao4tsI.png) + + +The `dot` program is part of [Graphviz](https://www.graphviz.org/) and is +available in most of popular operating systems. \ No newline at end of file From 761e64df10e6ff32a3a147ea5618dc3f5f546c1f Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Wed, 10 Oct 2018 21:18:00 +0000 Subject: [PATCH 19/23] Copy-edits in README straightening out some wording, and fixing unquoted `*`. Signed-off-by: Matthias Rampke --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a40f738..529fbb1 100644 --- a/README.md +++ b/README.md @@ -286,12 +286,12 @@ mappings: job: "${1}_server_other" ``` -### Choose between glob or regex match type +### Choosing between glob or regex match type Despite from the missing flexibility of using regular expression in mapping and formatting labels, `glob` matching is optimized to have better performance than `regex` in certain use cases. In short, glob will have best performance if the -rules amount is not so less and captures (using of *) is not to much in a +rules amount is not so less and captures (using of `*`) is not to much in a single rule. Whether disabling ordering in glob or not won't have a noticable effect on performance in general use cases. In edge cases like the below however, disabling ordering will be beneficial: @@ -301,14 +301,13 @@ disabling ordering will be beneficial: a.b.c.*.* a.b.c.d.* -The reason is the list assignment of captures (using of *) is the most -expensive operation in glob. Honoring ordering will result fsm to do 10 -times of list assignment at most, while disabling ordering it will need -only 4 at most. +The reason is that the list assignment of captures (using of `*`) is the most +expensive operation in glob. Honoring ordering will result in up to 10 list +assignments, while without ordering it will need only 4 at most. -See also [pkg/mapper/fsm/README.md](pkg/mapper/fsm/README.md). -Also running `go test -bench .` in **pkg/mapper** directory will produce -a detailed comparation between the two match type. +For details, see [pkg/mapper/fsm/README.md](pkg/mapper/fsm/README.md). +Running `go test -bench .` in **pkg/mapper** directory will produce +a detailed comparison between the two match type. ### `drop` action From 9fc976d90682215efb79055b0152238280f83f8f Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Wed, 10 Oct 2018 21:22:53 +0000 Subject: [PATCH 20/23] Copy edits in FSM README Format quote as such, capitalize FSM. Signed-off-by: Matthias Rampke --- pkg/mapper/fsm/README.md | 26 ++++++++++++++++---------- pkg/mapper/fsm/fsm.png | Bin 0 -> 33685 bytes 2 files changed, 16 insertions(+), 10 deletions(-) create mode 100644 pkg/mapper/fsm/fsm.png diff --git a/pkg/mapper/fsm/README.md b/pkg/mapper/fsm/README.md index 156c9f6..02a806d 100644 --- a/pkg/mapper/fsm/README.md +++ b/pkg/mapper/fsm/README.md @@ -3,7 +3,7 @@ ## Overview This package implements a fast and efficient algorithm for generic glob style -string matching using finite state machine (FSM). +string matching using a finite state machine (FSM). ### Source Hierachy @@ -19,7 +19,13 @@ string matching using finite state machine (FSM). Per [Wikipedia](https://en.wikipedia.org/wiki/Finite-state_machine): - A finite-state machine (FSM) or finite-state automaton (FSA, plural: automata), finite automaton, or simply a state machine, is a mathematical model of computation. It is an abstract machine that can be in exactly one of a finite number of states at any given time. The FSM can change from one state to another in response to some external inputs; the change from one state to another is called a transition. An FSM is defined by a list of its states, its initial state, and the conditions for each transition. +> A finite-state machine (FSM) or finite-state automaton (FSA, plural: automata), +> finite automaton, or simply a state machine, is a mathematical model of +> computation. It is an abstract machine that can be in exactly one of a finite +> number of states at any given time. The FSM can change from one state to +> another in response to some external inputs; the change from one state to +> another is called a transition. An FSM is defined by a list of its states, its +> initial state, and the conditions for each transition. In our use case, each *state* is a substring after the input StatsD metric name is splitted by `.`. @@ -28,7 +34,7 @@ In our use case, each *state* is a substring after the input StatsD metric name `func (f *FSM) AddState(match string, matchMetricType string, maxPossibleTransitions int, result interface{}) int` -At first, the fsm only contains three states, representing three possible metric types: +At first, the FSM only contains three states, representing three possible metric types: ____ [gauge] / @@ -37,7 +43,7 @@ At first, the fsm only contains three states, representing three possible metric '--- [ timer ] -Adding a rule `client.*.request.count` with type `counter` will make the fsm to be: +Adding a rule `client.*.request.count` with type `counter` will make the FSM to be: ____ [gauge] @@ -48,7 +54,7 @@ Adding a rule `client.*.request.count` with type `counter` will make the fsm to `{R1}` is short for result 1, which is the match result for `client.*.request.count`. -Adding a rule `client.*.*.size` with type `counter` will make the fsm to be: +Adding a rule `client.*.*.size` with type `counter` will make the FSM to be: ____ [gauge] __ [request] -- [count] -- {R1} / / @@ -62,8 +68,8 @@ Adding a rule `client.*.*.size` with type `counter` will make the fsm to be: `func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (*mappingState, []string)` -For example, try to map `client.aaa.request.count` with `counter` type in the -fsm, the `^1` to `^7` symbols indicate how fsm will traversal in its tree: +For example, when mapping `client.aaa.request.count` with `counter` type in the +FSM, the `^1` to `^7` symbols indicate how FSM will traversal in its tree: ____ [gauge] __ [request] -- [count] -- {R1} @@ -73,7 +79,7 @@ fsm, the `^1` to `^7` symbols indicate how fsm will traversal in its tree: '--- [timer] ^4 -To map `client.bbb.request.size`, fsm will do a backtracking: +To map `client.bbb.request.size`, FSM will do a backtracking: ____ [gauge] __ [request] -- [count] -- {R1} @@ -119,8 +125,8 @@ mappings: ``` will be rendered as: -![fsm](https://i.imgur.com/Wao4tsI.png) +![FSM](fsm.png) The `dot` program is part of [Graphviz](https://www.graphviz.org/) and is -available in most of popular operating systems. \ No newline at end of file +available in most of popular operating systems. diff --git a/pkg/mapper/fsm/fsm.png b/pkg/mapper/fsm/fsm.png new file mode 100644 index 0000000000000000000000000000000000000000..ef87d92a472eda969178cd71c6cff3fc4ac3a19f GIT binary patch literal 33685 zcmZsDbyQVr_q8Ahihzie(nxp1p&RK2=?>{QbV^F2G>4Y%mhKRc?uJ8mcYK?B->d${ z`2JA`Zufp_tu@!2^YM$Ej2QBZw=bSNd4l{wTv*}B6WBH2&wGf^fM2oCk?MfoVC)sd z1fLWS;BGy6!vExhuz->?%x*G*`tUU+2HsN=s9C1K;}j#rc8&!Ff2Z zn^0%)-4}f<0h%EALT_*J16@Kte!|9!h8rzhO~GEWX{K)wD7}@V)A|C}!^d)y0munn z1eMV%bXY-tL_`=OLO4I~SFd0NTeRR*-0IAY6IB2HDe#DoFI)#@ob2bvhyMHv8)g{W z-g$ReL)YJL_~$Q|Xc7Iq!OxY_-_zq{!RJx@lob&%qobk@mQ((SB1`Y(?26DrPWYlp zz;YSuHL7(p;tqa(q2}*Tnoa;8v!2MQxx&*lazQ&jrxjYbU!&-s3VLxi*WQlKd%t{m zj;t{7gPxnGQBFlWAUIedI>s=wp_1SC5GhnZ*TLGa6Geny_+lMFEL{BedrN6xP~~;X zc5BV$kxW>)p$9HK(prV#^)Cd_z97Ch@NOyPxP-5SjW*CuQjU#FCs%L7tMo-YSQLurfJBMa%zGDgYj*GhuLQ{Rs`bdd@C4#AOGhRFQMDb-r+ft?Qr3_GX`{NJ>2g}lS9 zX1Wp(tw)hUrILpAL1CgsE<`965B;Hi-&p^w)Gt48jHY0&i3DK2)48V;aPL|v*LK9LJ>26O_h+Tx)~m&{b|t*s>&xCzVr;1N!Xnj z(igusP=$n0rgnTljrpT>xSfqqW_`WZ{g5_9Z_m|np;qQA&X9zg2vf=*Bi zeQK;3UzR0h?O+J<28jRmcf~{y))?OTrqeoumd2Uu8Xui(T<*o15O^va)iu*9@3x}H8!6tv=1t20K!e?W_a2|UdX>|F&wpHP z@El0*+c4BLiLeUt+HkU|zTHb|OiYQt2RHN;nC`;Y!KQcN^Xv?wgX-$hTLIHayhdj7 znC`y%NxYGvf7Tok<{dDV*_KG1WZ9NJc45jAxb%(;8J7=w*>=6NAcE-rKhIWDJnm2K zsAae>a?qw<#@riwnrxVYTpD_T53QbXeqgH>?ee_e;~<3FCI^D^F4PbS0Ua@v`z<_Y zJsx-dNg(wY)@xBG+>4z$yWBb7B&t9(Q)eqrdRm(G(7{#t43D- zUZ%wHpGnBzerHb@vDd&Mj$$#QsYkr@ckuSIaa+Zm(w#gvp32QH$937#K%4OVffGw& z{->rU;=mjjJ967`kP%0wrbbj&R=)Le+*9)(o1bO;H0H{E*7NJ|l;z?~Z@?!kw~#{2 zbk~-Qm_`!qceYAIdMq7%2@a#S?l*fim(U2lN@%0gf%fW_hUWTM+~NQ4lOLC z;l4d=KRiAb6cG_ge&5^IcXor1kN@Mim-?tfD~Hf+(p|WQ;0b-IH|PSqku-fXkygm@ zt_YN}HDJ^;_xq&rGm|Z)6UXkl(yjBu_Qle+pA7OvBy%~fC0uiA`Gz~;B-FzNnYl9_NlRr zQZZY1b141VNi?H&v84GvKl2yFWGGD_C?rJ4-JLrlBjeOj$NdQYvR&!x{oM_8fa^e5 zO)Xw1JU*OlW46XRRSP;y)O^(S?x+ok{z<;kUAT_5fSekQetl?_xtFT7lDvu^9`&N2 z^DRZ?zGGQIPTbY+o%e|q(5= zhcK3=4NQv5I!-@?6aAp+hHk;@!J`p*@q0Uu%@V&mFc-CE&&GxO;`;p&85W({2A2xY zum`T4!ivLjRUNayn?&>CR^aW$8;y78vt?HEpvL*Ti^UT9SC_naw`QZ+%kc)`0`l_1 z>9&c-NQw%ZtIP^k5%V4w^Ug<|SOOb!?R}`0uA1Sym1jMJQ&xlfbc@CZ=Lbt$ctM*3 z=vurrQWU?Bb0sbiTU?i#%=y2PP|?Yx_??|OHswaU!to05Y(VXN54RQJhBud&X<`xh zn=@6^v2yH{`C_|45}?%`U`RwH+Ib|R}k-$-YCZH-N`RG?A>yAd1~Hjd`J z?!ug5r6wQ%0|IaRK_Ix$EHa+#?Ck9xCvov0x?Pszt^!n$$F;+p`^m3yorysMSM4t= zyL;jb9Pgfs>7)^lo^f5WqO7)H;nd3BxS?g#Cr5Y(?W5p+40!!Je*Ida;NB~EzW+wn z$nVBaEuC}$YHXpRT7?Pm@wo?2G##gn zu2BTejBT$#QL2*ly8@Y-Y$Z_1N-Sfs&WRyCSq4TwG z5;T67BvT@wQaBxcX}VZ&0~;nrd?S+{8KAH)&SMiaSiPRJAME@Lk5pB`^Xh&ubHtK8;fm6XqXEBDgY=z0k~S`PP`4}gyP`f#50?OtKjG@ zlVd1srj1;J@Q%O9^5R5AMP0Ls)>(?!yv&(l>+BFm$}21!yE*jodbndB;JvYEI2quI z2Z8ftLKu&GubPf-fCzJ%x>Ryzw41Nh^5{Z(p5G}q?~nCF6Kh-(o7RMvljNL5bTLmr z#$cKJqV`h%21C-_cZOJRGxJkXinuxtl1Q!Fsao-s4#tSnT{uq?HW7h;wMTZY8EOji zWIuhyU#KU+TbvklxvOK_i7}(sxK};f!h!)RbpI|io3Bjm4va&WN-~Mf^04cj^=20r zP1pBfy@R>hih--}FJE9z2Cwk5P#DUFQLI%!vfH3$I$4~&P+a|R1nY~!r)+?YoRje~^S)oJGAj=?191C}?}(lvRRH&dpe{L2LTI3xfA6 z8zP@^AkW0Jo#X&X`ow9zKV_VA7mkVP+8a!R1)Fx{y3LbV4Xa^FA|1~wQjo)D zTC#0^Omu9l{^{m0h;%*0ZGSkNHFH#}&OxBr^Uk5N;o2}X#q0iR=5mL5{cGVwCKQ!8} zTQGkeu*8`*F6PLbR&v&hhIHNKK5KuI7>>tP3_#0wK#bMb?lVgHbkP2qh)A_pXO|4DU5)+~Fsx zycRhz@%ATu&ic4-55Ot9y}-9;VYh;va{ zME`8I{jaf@No}{iQoT*@RJ&eEkb>3DDM7V-75Nyq*Y#G;8Sn0Z%T7_vN`67{2qoh< z8+dkfhM#q$xdqH3$Vtm+%{75$cTEXybeYh zS$g-%KgiFv7l$BgTxZTvm#OTpAod;~kQ}r0aYin(%@1RENX#P+<)zoCmg>XsJRKH8 zNpEYD1g%TS>#;y2C};{b%k=$Y#SKY_+;HtMA?5Al^w{w)+6OY`CKvW0D4f__2zqFy zyTjjxvTxRUW@}!$+{8g!iaT{*1P9vnc-)aJt-XVjr6<6+J69+VHj>!NR?4ddsE5cbY#ZdqDnw|K9aTEc6T+E?EQ1lVKVJ_Wk)$6l_l{k7r0NzZ;m9P;6#p$jyOz0FijX;f1x@M`s&tb-v$BsDZ&k+pM3wZx3zq)Em9 zYl9&}u=dx~;NTI>EU>m;ey`mc^`PAPzj&=16Df8i z0J;fHWUiTu6m*#2mQr2TRQiF2{03-~N z;D^}dtgyB&^;>&wWyMt{zDRaj4`w#uCL(HuEBHqGCo=)xq=5aZxS+}Rtl!9MJ;8** zeqD;Tmcyv`P4bLjkptszVEq^{_GpoUvi8XlWF_lB+>czXRG18ky1O^ftu_Fla!1zJ6B%iW4W1^{dor3mH;_u>|-S4psAGG$h#uMzfr{wtoB zV1an9e^Q>AquWZB^bEW=gRzn7)ignsBg&b%mMIqDcmbMpiW zGju)4T2x9(${^sI$N$+7DN$fUz9qk!T`QHdpQ$u+K4`png@)$icF>3gRANC>)7(Ad zqJGEhzkCT-hK z^3=j{rK#Hu0E~?s2LMq|+d;%W{N-!bU;EzD`UnUZh5}Gjm6iQ#?Kkufms&ZtzJKHY z*nZIbfFBy0>b75dcp9xZCYN2__}9k@MftgT*^|r4%b6Y#F^}tM3+D#J*Kuz~Z@S!w zA!E%-Jywbu?ym({dIlV^ef>>;e}CtbU+hP~Zt5>KdHgtRLjvzRU+m9oHM~yvn^+KR zp?MtLa6(USZ(ARIb72VERy>g3#?=kD>F5STs3Xn(dOaar|KoutPsI^!2L{bK@AJ*_ zz`Ef`bk1Q^Iw&~U?zo?+;jan!c`H9QN#B`8&fa_qZ3bY0x`F%lSRU~5_wPGCF!z@J zj>N~atU?O(-oAr#*JnFbU6tNr3 z)#J53M$cR4)BX8(<`adj^#@Hzj;n?UGPXT%K$q6>NbfP0yY1nsefSem>9r1T_cVk-?J~e`=$#8>aX@hQP9(Oqw?LR zfx8B|Po$ikoyW@K8fg9=8zEfy`=vRE-FOj`$%f1-8952P5xYcrg}k6GXh+T z*d1K8RoE+1B<$EJG#E0~_W+LS*qyF80@~D_H!-6!T*#8KG_8?DLgDYJOMZCsV@Mo- zl~<1p{(7r)R@*CLB!ZSt)wjI0hAM0A*_E;MZVc0t=;-h~#HKaT@Wo&GgazW=o_u_~ zqW}H&8tI%yJM_+6C9^Q?34Xq^yRHMvtzi2S#!|@$s6qY*AM#xFCO=5R&i=I5ceYw2 z>Hg`fl8nLO#v7JfQ%EHO$ufcaAxsL;q9l{^0$pY@^FmI9%f#k)A@=#wm^G3Am559R z0}`q+dsI}}%gey_S$1(wXw%{8so+N!94aaad6f)0I7y^a+e`Sa+(Es6*=au(bEf&9 zG4TH-0Mk}D?)+%ZmCKWh-PUzk-mYR1Bza|dpWKwjAqjAv#dsC&x21cG6=bFu0DRC zpcD5k2~HDxA1Wk#OHox|U3Am??ypm91LHF;?bsWloYPxn-Q<(Zcki7xVHs&NLeYVO z|1!0uMYM#^%bH5T^8v43xnq8G5{rkv2`;%iHH&0F48IbF3$p<6KFPZ2OHQii+nbAn z$*%!UiyrK+U%!6K5NxrH2)jo5ajMxR)~+kAg}oGmcov1afm$)8%aCuF=7TKJ4+23%O>*J z9Cu?XI)5zg`$6+4ikJ8e$IB+DjIhJjNc7WU`$a*FGOxX=Nr%x?Ac1Sz|6=_C1oUvR zRz2wA8+)=4q4^{%?$j|_gFEaHNDo^uqb~mo?!}e}MYI4!=5^)QJGvxu=#hmA0GQOz|H7AA*0I?n`?g0Saw)B?Y=$ zjaT4GgNNxT^pV>Eou=Fp{}s7duS|a^YDcGp>+IK|XXSHL4319{mKTH^7btJvoL?>K z-l*ziK=TZSy?UjkWlVnl`qe2(-}t@WiplFX;GI=VQO!cIsd6;cIzGV7|6h?{OxcMt zKeEZ3am0yUL^yDGhSSMVX@Q5uNagqDv=GC+U60<#$zI8swN|FWYQlCN{dI`y)Yze! zxE))&{%cZs;hGWK?%`>eU4KI2@k$8&Cld%$pklUE>o4}j(4HRg5J=5>trVs`8NC~} z_Ciz80joe(5nSyp{|QqyA-Inn@e5AqWA}807e@y@_KkY}r>+;gDqW;B5!h&>WTzsH ze)6*JDwYhxjx9ze_LENYN*2~DJ`0X#dEEAHv>IFAn}Mq040I7^bnNpE-yFNz+uQf> z1gq^|`y8)G?P`KBc$;8?c5G$7z2Ho$>u7@*?k+IA)7uLm$)2o|#*;b6g8dyg{2p*B zvoX0EA5{W$4jRR}Un1mHMrBhJjMWgkEtr&q@FI&@H#XVxcwI(5kQ4E2ntRv@IfljgyY?cu3>HW`a9OceQ+ zNIWbbH1NNAdTx1{`sw&vc(@0>DoSlLWh7k9b|@R3tnf^11RD0RbRu%XN?{N2e|48d z2lmX>-z|=Iv@=Fb#}NlMvtRiM4xhVUtllm3C%MCh9F%1k$7{(4p|bdrx}+h)2-#_W z?n|CxPFoDhRhvK3(ULFf+dhS4sYVB}A?YNxFGBaIF%6Y#Ju+9%ExA%(ST{!s74{!W z`i%&qe%b!aQ=p@OTqe<9`u?Nu?^-@2^?YnAT9rcNYqL_Y_-8O`CCE97jroK|% zY2q34qJZVVCD7*wM!uQ^xz{`HwfSRd=c+fV;&z2y?G!gKa9G%V<4Mm~dk z88xnqm&ug>s2%ilmX8^FY$*NbG%yp%r~AJIA&(kfZ6QrqF@JyO@tj6`nC7v&cI z1{~;!V~L8|@(Q16EBJ&0zlpXEcEU#?^Q4>$1!i=|=`3aBPB?E@3~J(Ilr$~q#yIhr zjl7=&;3_tY=_|U%Qw6yE5Ck^T>%(?bd2Q|D`><)|y4n5P&2=Y7aneC-AyW4-xtZ0l6F-@)#*7yq4j;GMwKu-JL3166wjyNeKp? z*2vk{^t@VgEzArDy%^KW8Ev*+E5WphK zHe`WL6_t^v#H^>dyy*7^27sycoWkLbwyNfAi za1hZ=SQu`K=r?c1enYwy$vE_+Q9Hvfk>UcvrK z;NO$Eazc8=rR!}rrn1fTyzD80-bqd>;YZHud8ueC1Bw#tor{wbX3z16e((m{-b@wO zSVOL2j?ByllT)ByrUm%M`}=!ci5Il=^r-e51Mv1^09{w4V{i|;$jXCwk*&g z;Bl>7sPVWtzpll(kE^&L_Y-w;s=rNdqydyYU9S<@p80w;@GbZF!n`5Z@9MyhI3<$S z1v%!9S#Np9vlou1KS^jj)uSkqXn^oicm#DZNslvdLT;Xt?^KkhS;=XkmNRq9C_gu* z8oZ-&=iK}?&+rt&2XMG;0118O=CFkPQzXAB{-?yTV>$;PJRm^+sw@OeuKX4i<3Kgv z*#FcAq9JTpR?ymw3Qb^1Z%Fv=OcA3&on`98-8GTw2Leo^cyj5?G8q?-R|%Oi2CkA| zY8o10b#?W#IO!AahnBa_H_slGaC4_V-~>55tllD`UY{qwQS=$yPD zbHmAEITs8^RKoydk~#1njS$?>!YtxQ7DbCbE@I2+cp#6yf8i zO|{`ZIrK8uScp3348(FN&=U~&b#;>{Vkp+lhPv-xoowukececLlcE103aW zXql-pKikLBa)_cEYq(qqIXgjq$L38%^$t)hbL9~*taIb-&ZkX$@%WJ~PtJBG%}wIN zx$tZys7Ca>Hq!%b0NpGTIKs}C%YjFaI*W$`%0193>aKKr34}QCr2F1?<2ns=8d!|y zW6&uk>v#bV3W0XVbM&^w{71+xkb(;DV@XqP;6*bk3CCZX$B6uqoF|Bu(laUp!I?i7 znvfS|>+sw0U0w=E%nW|2Hn?Y>s@O#hbvy8=p-;|xuOzvZmiQJ&ByK=W$sycYK@5`je!!u} zCA6RRV78{D5BIq4#|TjlW8)gYx3|lUPkm7cwXo=*;Nx;0X=u1R%N)PoSi|$~Ef*}H z)tLCxqQ^tLozBi;3Mvs3Yf@R0$*f7?fAIf=_jKN`9Y>W;;fnN*dZ#lP8Q>00;c;WI zoMGXTG|RPbq;i~XxnW^qYGqW4Wc2?OIbef zTpH5|8r-Ko%9{BhvAksl^#QA_XtM?Y>P2BI`mAC^WD4Dw&PvmsxEUGcDd?50YrXH| zkM;j=M@@a;!X0{*W!S=8rkeyN?0fs7Uw=_PSx#!-7_{jCh;(uY-#8MF3j~7-zGJqp z+b&S^UrjtLiKfQ+W9jdDqN@%o#o&Efleost8dcIcx_)hb>q}Sg33BK#eGI0Nx`IbmC55C`p}Hbn<#lc3#&2NZ?eZ8sZq(hq ze8jL$OF@y2IWDO48NB8FYvHH&ay>^)`kC=(<#%4^mJKnN&&u~K=rtDmV^DcUtdUYsPDOY#*!I%ej~J%^H)0yW zG|`*}>1NL!rSi}D5A`A_&?TmdYD;B1@S156OMNFql8}r{)al?QGP32Nuuqg&CZUOw z?}hyZivP7>#tYyFY|?fYGlrhOOGwMiW2Q5BhAf)5{W_r*9Fi4o^^mWOC@s#Uq$(_mQTW&+T}fL%q== zs-zN4!g9_bS*VIh`MZ3mg|wh;Q+FMscSC~D4jMzg`}1Y&BtXOBeqH6_4lCPlEU}}~ zhaR0VZ@kqi&G$#bTA9@G>J&odS^Yen$R^@^^DVMz&)#tV4oTitD3L9G^rHA54?qFZ zbNa=AjUg@%7bc&7Pbsb_AuB7!BM%siL0xLphiS3cgf~4s-P6}MI_vM}N65{Mn+yUA z)>zFmm<|y>%D8pY^LkHzelwO#E6;~@EB{(5nU#&-GaKS?_D$!j?`sSini2~Ch`5qZ z8Cwy$4-CSA9mJp?w4loEV8`X>hljbW+4JwUt3Yeoo}Qa*dc9xy^J2C=n1F{`n}PVOMG>)`T!&!33Dy3U4JWyTb|?egc!Y>n0d=w zxZ~76Zm?CR+*s57yor^9dO)-w+$7P~rJU^FBO?a~Kxto8-peNdS;-5S4G?s6bWRo9 zFy`x>x=Z!i`NtXB>C@~}5)!^x&Q_as1MIAhn3)-+>)Ez{2z`^v5ktwMGc-!Gu z8NcjfD)s~S%kGBi@5;nOGiFi)+X-EA9Rk%!Rj9@g99orC30dgyJxXdP^uB@Y2(t@s z9UnZXEREj0rfWbP&5`X!qHlO&ZEc;ywb2(x->qTY=nsJC9j~5fcwE*`W9jYGbaY)n zK?~)(zkCW9I5y|&+3!zMAJU3GSyc*ai9a&#bz1`+U(js_I5(C6(zj!Kyx{DcNH7ZoE*3aJsE^%#c8(zROlI_t-FbavdDAC zTUVsdZTe`lMuinQ=3|!_Mj}&X$FzRk5rR8bSnQu(PJj3})kDO*qYlYaoCUd7mEgE8 z=ImP!pwMtG6pK$L5l39EM=PDz_2+<+D_~_+jN&j4kYOAWhOv7BrrVa-7&?t^&oQla zym#9iFs`gZeyGe3fmrYbJkJr5Pn5msXvuNvnQI71yy%I2)uGO7&8JpQp(Gg_1v{b5 zZg~a$_v|vQ3mAP-=aUkOu#1PSH-~b_7Oq;J@We`^nSlhJ*j!Hg9mwo+-jDp&b1FNT zjqx8wYq`G8fHYf0;sIzVD?oGB4JiCJfZt=sYYgJ6y5tbve0Q#Kd(r&R3259Q0P!lv z_3JiJHqdRU#b=4n6Vz5&zh8%>w;c|rlbI=-#Ap!USh_Kg9HJ=03EzBsjP&Bgi$lQC z(h?MmfPk=L`Gd#0)mHlG=tx{bVuW@dss-NaxwpsOw;BO??` zHk%mKfy6FVyiAnS8pAIA2^|*+)-czmRNn3|*@}lM7dN2w; z%{V$syf~bpN7Yjm9mPP}ap53Qpfx|tY=R}p?6OrI&u2*75sf$amvHcS z>Z;(4Tc8UZ8Mrct;K?rWM75mi(Qyvh1$6O_7**{rfQ?3*{Z}m@%q(whePK19*p%PW z9i7fTaj)DC?WJ~Po7Iz<#Zc8uxBhmvISehu066e8Yt}F@XcXE_(7)AuvtM5YmC9)X z?@WNaAWtjJMn%PzuiW(YiA6)P({<_rY5w$JQp+j12~qdG4;y$}>8@wiy*-dX30)8F z&N>FRgHO;TWvh-QERxL^ZNc-sLXB659XO;%DmJE0L5s#hsJ`g|hMutF-x}P43#TU3 ziOA-w1?Uma*es^fo=Qu7G|RVr>6pj1kIO9ARqF}h(rgskoBBxpgt~VRo!d1>%jH;T z-yvJWjr1Z9C?sfe-aXD*eaJ@<9o`+^qU}8?-IWga8?y$Nld38lB&4)%#?Q1HWjLMo zxy+b8D_M>AV(Bo1y_KxHM)+|AjTk!lDme@=LT@9=>g;&|@U{oxJMJNouG?NC@Nv(Q zum`Lgc!_aE>n@}`KvzC)pC=(!Z(v{m?wFa{1bi#}k#)|xC`o8|>YY6;M8pN3+D-QdWNm_0AvQRaEiQ3c4cXKvlWS)Lj&DC0eoZ8)2S_9)FazcX-j(Mh4=F`RUlr#s! z#fZt|H1Jk{2NDYI085P72IQd>r`9U>C>YXZYZ*2`XZdh{H$z!`FZS7GzdM%;wX$e8 zeg^L3_&DFzx}B08D025VM}XqEovNH)`v&KE^06Xk!C$Gd?NPolF=2IP65EZ;9uZP$V8DAM6_zY(*iFTI z33x7-d_fzxj|L9-$mdiWw>}8?L!3+JD8_U626T>KgG$JQ_m+3g_wU~M6xGH`_RMP? zIK~`nqI7tp^7$I+b-5Cfk_GqW8mVptv;0TCIT?lDpVUuyga((x))bPh%v|acZ%Z<8QV=pemiQF z1Ic_=pcE)i-&A;P-dJ{e)E!w{we?}ou|y@y&*EX-ywLRw<9%*%ve3TQ6Z|l#3SCB0 zJX=h^1>0`C>+QG!&Vs@`6@^qrG06FX+rbtKrhyx#9+OUzOnuZlnRy!xgR#Ud3is4- zu3y#O9Y3zu3DN@MmqXue(TFC#c<^Qvw8HK-A~LnJD?j>bhVSvOk7#=$jr^<1prHId zK2~wji>-Rk?iA_J$Oyq%Ud6FFd^(6u_P?expBY33)@T&QPV|@XWi!>_^&P3CbVAse zzWCQv#`v&iwUn36SA8HRPpOsQ2aDtJ%;*MP>g1 zzRWG5fS1Ee{xm-fv)HM-5)Ndb1Oz0#GUIO&i%zT>;-aF`7n$vV7A& z$>H`>7NWg>q71H`2~%9zKfKWS>ooVnN5KjJJZfHn)Agwp)tNT!KzYw^t^oj%h>5g8 zt?hh*xsIkvF5WRi7AtQq)btE_f8oGJcD!FFm1MP;Uoe+B)D4Y-H!anSj zG@ru5fIB()Cff^~7oNv6=)q2nhwj{7ZKR&@MhmZ&F$UfOK3TYA=Ybml&Y1J4h96iN zgmcbLyIsMN7;K>SOgH26HnkoiS!Bho{}@yiaf!0Kjw1x`xR2*C@0E9uR%9}s5>0a1v>M@GtdDBkh%g;uQvF)K(T+vlebwdPz)fT3;=kugjH7P<3Y?gBig7KfoNJF>EhX^K@u^jV zu-?D-g$hoGCn;qf^-VOiqnzA!M@YLhSwK1FKhO)~`QeRyi#N=5ebnoq>P2&74t&pz za=zzxaN6A1lzGN#Nd=Sq1Ycm90aHbsy;`#g=X&uqL4Xq&Q$g5x3c z!M*fE$>LtyfvIvFgy?Fg{o-WAP^&OHljJZND%y@4f~;2tbgDe%YjohQD*=NVDQ%cz z-^n5kc2!z|^{YUBGXW~5`;;8WN#}M96`aU>VsR~>X91n9^!y@2Jf8+T@&V_5*;g^D z|M|d_J>yEK+JeNfb0A0ZpC+?+W_rK;&pO-vS{Fbm2Mvmgq{fCbpRG}9ezR0$K%bWhc_f^7MV1z^4=o?m-HKl%c{^}@%)AjMA6EdLbd3V zqUQ~Uo04zRqt!qvE3K4}S&p^-2rsHIGAV*jG7;4DZ`U3whS6aSmXQTR*zQVMr{=3rzpBZ5EJMFN&Z0fqUoIr zWxad`OKYF9QiI_90;1CFRW1Q&hiLl+9Zgv#9kFFcw9KM^&G$C2fJr)wK{PUOn7j^AZm+U z#Vd@Aj5E{g^B$s=W(`nT^2A_^UAlqFx$C;%lRLnsmJSFeApf8nX81T1zHu9882%gx z3WuV>N9t}e$zmqQUCr|g!SVB^h&(*I}TV`OWn* zm_&bSRO$OP*M^&*Sa3~>q8n@X&Rn02JPZ(>_Rjn3m*N!HLHF;6#_^4b;G2n%ylUD+qYl^0WcGJ0-750eH_93#z$+Hc@@yOETMN%**e8 z?1nZRFpkC9j2IoCj0+Q-`jgE7d^baLFf@AQ0T_9~7gh^;X%<9bRMd~w%l#6;EDXoP z>|)N$4H~H|_-wCwONfqBO=7~_4UILHgk(tUUHQG82iDHYg_(_zFMVx ziu>8yWYUSq|Hs?=is0?UePZF+l0 zi*}28Jc?tBvqZ`T{4?0@zBPf%so2VGka`NRKyaF=;QUs9z=-b)_7|!+BUCtEWEX(A zJUU~aX3hX+qB4&yzz##n>3(?#N>DJE3rb-goa^r1C}rIBXd8HvnJIx`L@9 zRUzkBtFQlGu=VqM*Y;X;?!Ta>ps@K^;A(=4deW#`q@Jb77#~i=2WDQnDjs|(g07x-2)$y29DP zBEh3)4IoP)LYykaS_%FCn;C8L6Ggg@?|TrlsQU*8Q43qk^BK!$(}fW3?kvX*Y}V>N ziuZ!*uS3mP9R}Ll+HB4{jt+Mr{?E||N3eL#_ROm?1`vGE=YhII7kLm$NlZ(@@!Ghcu?RdGlASUwqiB_po z(T^et+F_|R4`>BnPh&_Te>}kVQXNG=*MB9Q5eVPg8Xfg0%JTRFm$feFTcv14FLWnV43iG<+DzgI)f-A}yDG{H7uv$*HnkCmc1?$Vnb*n- z9H85BADDdV?yK|^A2m99BK3OYhcbs#$x1n_v)gq`4{c4dG{jEhE!j;np1`6r6!|?V zsacVIezn|0P+dkqTJz6Xn!bNsHa=d6Ls4nR@L?#Rq}m}O6W6DT7$(0Gv#Js(qx$N< zI|N+44VxK#1Dh5~hcuUa!woQRcPU5B_&g|az6PN8B%9S;JuV*h0I%#egQjmdvbacq znty96x|O#*T<>JfgNq>#5SH>eBY}GjN`p&frJ}(z&;WM8qemM>s+lMQf(FUepY@g#iPGm+DiUe zh7iiVR&EoSZEpcO*o-rdlXA!+(Bu-Cw_jX+|NvDd=9X5KMrpxdPGvvTN5z?uk_sa2lX-3K3Ay`mp_ z`w5S@UjHck;B8$Jpyvx>i~t5->$%9clf+xpton=>Aq|Z^!AAFGcbl0J4z5TicQT-@ z_q0E(_s9FT|3QqjNl$K+OL>F5+6&%|Vj$lDlz_wPxbs51nA^I(!$q>w!thnSLu9JNi-Tc{sxYluca8r=brC1j()C%Ru2ADG*|{fWyVWY>i${H#h;$wn$HGgwiZ1aqg6 z1GvDs7q2K2<#93DlrgS5mJ{Re8i*d(3M<-J=`iZHNgWy`jhe?T}gp*(1Q6`t_F1yZl znnw!uM<9}^C6&p#3;8}lo4Kk{W7P?$d6iw97#jGln?%XoCJ*A}S^w*jVxN$V1C*;z zEXyi(=)FP-M3fanRC()=X-#kB`-imAzBjL-j=ag{pZpJ|%7iWoF99*CU=8E!$-`~L zu)_onV}|EEcybw_+lIqwjo@)CWo2b+%;`WQK<@ooEV}~w_U&60?6YJkss@IdWEcVo z_G99ST&Py;r+k1-ZCz$uI>6&1f^5Dq*J<6qy*{i4(V=??h@(l?w0QE z?(UREka#E8^WOKh_kK1X`0;6Ro@>oD;}~Q7k3gSE_Z$ODrx8EN6f#Z^IPsy-iTuZL zHPH*1=Zq4YVA&(0eq09>otOYmb8JR>I=WeH_>v%LnZ~{U`+l$UK|I{&PL1?);NDEf z914tBa+OcF<_Y29J2<)-DJ(Fsm1JIiHALgH)euRtC@pLFv(m0chLIqL;2Tj}I&oky zohlc>BGRck-Tyvb+w)N|zrosa*N55AusaAPFh*WwT&*RRE7gN-KGT4h(5&iYseTJ& z<{@@)T;6pI_fm4sed)o960|=_OuC#)o{w&UqquHvuSH|GX2WoPs(A9Sk2npUa@Cew zS}pFODo2oHc1!hqzaSh@F9OnE_H^ zQtqZC>?$2Pu_g1XHCJ9D;ED>)vhDjE^gPa(UR6J?bL%w93_LmMW7V)6*s~dVW1P?; zfY>aEb7`buY^OQ(I@jRhehev7d-IFp!@^KaU<0oO`fNUJ&o&AYRrqxmy{iRA%OaPW zAX{;qHm;PH;@OL~NVmVC$m=jxh%l%g<+N<`E+N{IxA?P|7Lz!fob3PP;1rbJX9BiD zn%2|V6{45)r}rRM+xgG&9c7kVNYy49rox|-#7&`5M;^6E>&%g#B)EBIxZp#!df4DZ zB}2$w5>F+(4*P2n(}nGTy`Yb?2Um>uL(@=cB!eAHKZjkh-=ZcQEu)6ZIwUb4`R)5% zYRq$O#26b4BBWa998g_xV8 z%^h2FyN&>Ln|zOk^am~u3}5avznOv13A$!9_nF()g_1A1^XALd@PgdoWkW=BLvddk z7Dm(H=Bjl%AXXM#CbER@BgshiVmgZGT{k+(2pARrewr>%xv~$Yyp^D+4=j0&Bc8yTfRlYNb6e>#N-j?8v(vT%9KegMJ2z{I ztzDsrc|lclpN4rM#c9OP7RQi?+u7guIq-Jzh87qq<&{3vxzJHlf3;aTd$)e#;;g1x zyF%)bi}M5HE(VIHc`zUKlZHmhX=kqcO`XMwP3Vs=Pmj<7-ym^Q@nb0`my5 zYpHstWEB+LFwB_z;io4N9kP|a%S~9x3v>Cjy*D+izYhPnjg!}`j+JrjupO&Cl*b%r zEu-V(Vm!(;#YG_~T{I01eyytTyrkU;qxZJbp%JRX_mY`mX`fmupZH4sKOy6&)H*4T z;J-vgM?yO(^0FDa7CE>ruNEdUA@>W074R+drBaK~0}=woIXTr?0|TV`_i5X{9jsp} z@ipO95s)x4R%Qwg+&m+H5X~E}WcV>k_*@6}yt=yjVSA~Fd?bbG9yf^*6)%M|4`34i&Sgz4{YS9r?A$|-#7pS;YvY+KU93hL}G74m~`QS zI0`df@06m#c8)%w?Nku{3lBDo>}eZrtNycE$?7%p=&@VFPb)k&~>u@I=k)jXIxCy!0zI3V6XU3d*cGTX@EK ztp>+h5yR8`ilZ)gi#KT{1V}}2EDk4CZD!Ci_m|+T)%{{&Z(6RxhRa>YGl@ymH^rn^ z10JOBrS20cBvlaxeU%CHQnkoWyt)cevbJ3vw$mEJg84NBJ356@7r@7m0S@{RIID77 zJHzkT@@^y(NY~bJFi?9Ok8?=8y(v3~88Gs(>oKq{>08UBC zZ#r&$0__uW*1EXa`?I||9(^X$Us&j4HrFS$b z5i=<)X6ig!a6n@}At8yoiqi~12`?T~Qi4YJ9^!ZBIb8rVzCd1Wg(~82`!hVowfY9! zsS+?nm%YZf$#cPy;(%j12)c}t`ZW{o0QH_i=Md2hMa6Sne9NB`-*z^e;V}f^zI#|y zCu(p*s|4>dJ2t{)F2VEqw`T36WKn^g{vlHs&!Gg~nv1z$9(sN=QH|PRU05^N7-+#cvs5jDvRK`{mTM^ATWti~YKyhnLo@+yd z3s#;SJa4J+QZy;Cu{gbE%GRAY^)nztn8uQ>`s+F=7-LN-B($0=C+erdn3Z)8OMg(U zt$B9eNiI-$zP9w_Srq2_Jt|_`YgDrK*!0ujs*m$cnc_uQco9Ep{_+DMja}J~5xf^* z&!hj>1&-CYyz0?{#?lE!8An4_xy6$3j;>Px$YwfkA=)@ANjiyM)kEG`QpVS9~cs&0&DGoP6-WWWGr?ZU-%H10YLDnh>XRM%L zE}(xG|B{udq@fhFlBoV8r4?qu|6qSP4^zaR7p+qF909-R{^q5?NjwI&sKUZwo8w}I zLj)dIvNHJDjVGG7I2V&opG3)Uw@)zT9xdv5$HX;*YuYd~yDFbq6|0G~pcy~g1IOy2 z51n9Bf)kbV*W|ky;G48O!~Dsc!5MFX@6G#{Ll1ZG00EmxXpX6(DP1)Th08G^hh%Wm zNEs(|qa>N zXxW(Ph=*t=B@`Tgaemo!4RIb}1zM`Ei^)*b#awmJ(RVf*hg%6vNho4fNM`*dQV#E{dCc$PfZb3N+zSLTtZviEx|l6N&Ck-5%@% z7EFHJmC`cU$9ez@5SRZhu-wcSOn^qDMk2z_=3AepSg)L#NN+ z8}3E9Gt@-&k&3^Kj&9_x#nvJ4d!Kjsd}6Ecvq3?gkA0^ZGuOdHN98{93_|c>zP~<_eL&WtLWA7PZPkH?ZN?MFcO&n5O1NgnG zzK|;7W0gCMPQ|^k*_C4&6^Hw*r$n!$mas32Tsxyl$v+*Np);lixCzQ?DAL_g!y>99 zhybr6R(^>W6Rra*Q=dZCQN-T=e2rjl8)*!QzSlW4wf=PB&rmFa56(jinS#( z&T}cl&oQ4Ckb)jF`D%cYRf;z1a^QFQJEs)w)RdDyAHM?XZ5}bwj2c#rj$cZzWx>YA zhMLku?6VTBSFtc%rMLsEF}1jW7@hcFsB9kbHXpU#XpnP-Cr9TSVsy-WIDUWyBm>Yl@pWwzD=0{CE?;GoD#>TiV--&!07QhPjS5 zzL=>$$BJcT!z`MBh@`FsHiipWFK%beBJQ1sO=CLF89Ehc6QLL+JdO-{ALpaAddmg) z!T!>n`1f`eRJT2FWO+Ij9-P_E&DkB$sBOboFh=6=O-~b?KqdMaL5r^z7X5u=5lsUuzIip^Q&`vh}i@20d!sHS0<_*fF_SMFVVo`gJsz^QRt#~H4Is!lk&)drJq z1m(rE{VtS|{a@}t{T-h?8{|Spio?Eeq+0KC_Vx7o}QW+3ek`_xO`ZylP}OF>m7&N4>gjW^4!}zVlTAL?6Mf< zr?kB)yWUP!@j=EzF$$jxH-#FVkk67Ld$Gbl6q+;iEriAr?S?hXMGg|QKJ^_;%S%Ju zO&JX*Wa67+L`+6y>vyO>4v(+w*loPWar*EfY35C9t~!rHqaKO=YXy9-Enad%I)0xTc0hlXh8!&;Hll zZ$JHWz1Rt*JcRSL_s^-|7o3*Mtq+uMK%ImBUi2fVG=AfR}d z2r>neH)bbGioMh|C#Ae{iJIhixNBD!Is1s#l1YIRYa%;(v{M5kY!XY;1K^AtYGP&W zaW3l*+_$&4_xDoLZFMw|=sPuZn>jLW7_zs`rz{;w=tZtwF#GE)J5`U2q7J|q^#Zzz zwl*>-J@GeR3he!G`BTQA1zlxFk$t1Y;OK2w#yDpldEU%)Zz0rqXB9MIlP9;3mz}^` zJD8>)V*$P`1C;m7YMx2}ZI&vkj$bT4>y8xpqa!ZF=TRYve0f@%V~|j+;6I?)KGiJY z`kr<;&B{y`gvnBu^egQ9qyYI?DltaEpUEHPtWSnPo5cncde zbzAvhuhGDhgfay!U2z0?+pr)H%|45O+y09e8k4Nb?g}@z>2LuDz8;Rea({@?--Zc# zj&sikiHp%N8hOVgR}PU&s#OQQ2p0zuQLfd8hs!`GQh16K6dNu3^Ge_=n{kT$CvW6U z{zAtT`(v=zi%dDNRwfxsOGJc7kb_WpAc9_J?H$Iuv`kDW_E&YbzbS z(}RI1)#!RCGSYAw>~GWti5A|-^X`AA?d8B@*OUU5(<8%e?u;D6dL zAbCD|LfXEHBl+Q}gU}p{@$FNA+qDlaES(T54&|uiXW@g6!^ePAtKT;ws~(b#b0S>9 zy!udY-yV*rxGS#|91PWD1tb-x*ovGd(zSPr(O@^xUeLYSnb$Dkp|92a!fA)kBcwgW6pNMc7t>G_xp3@xj#;~~_beU)ur8Ca{mlzxhZ5}?m=>4ES!e5b=bFDwWG$|; ztsGHNQE{1?|5dFvj-nOSGgv6v3+kCDQPr`bi}f#H>RxlH_vO6%pF*bc3x;>5+mP?Q z;fai_E!VGECTT1FKZQ5V!kkN~jn+l1<5Vj3mZ@~h2KNI8`RQf36ycrHbnTSWId-~O zcA+Nrzp!o6nG8clNsI*+L8zucU6z@vU;R?yGy@ywvQE~ST6CO=iRp?Y6%$91ZT@3q z5#Cs$yy)nCH$)Npf8ZJ?ZV3lg4aJstMBb}Az3Bc^EC0PWFJBb8_ z*O$8^QwO-$A43QEXxU1s=$l$SRv!7_vRsV4VMW0)wDQTl(M^)6?VqZ2itQczub5Zz z{M3VR>-qkFG`VU8$uHr;giHjA1bd4tki}ogDf+su$293re89JMoX+_PJEL0At;aEH zpHmRgDO#tJUs+5t!>}L3Oc5teZodx>)Q+VtmNi8$xHE@%iBmg(_%t<6ID9{ClV^hm z;9~?|9&8tk}#87@r8 z^a)t;FV466j>fS0*~9+K#TAd*2~}^p=$$jsON8rJOuGE61;*xacbFhjy-VKuJf5no zV~hC)d+J8`BuJ|^WcjknGT~f`=ZNLu)P3j9t421vdABg1DXJ_XUHPy=82g&U$@%AY4x4#GD|E%rqCgw>=2v=) z$jJ;1axgs!ZJCkN*n|u^?I=xmfBx)$<0sjdILMfUqkoT}Z$m|og)5(|6SgrbN{u7i z5y1Pp`yVBV{KE&}qOe9)n?|Kpj7izeRhv_rXl{NZqpGxDl`ASL%DzCfr*}XK7Rvew zM9#hplF|LF7DUe5!V!3i)ItV8C8|Ef8??*?YLicwRCY6sj__70Vjki-R*Cs^WB$3+ zzzY>ZzA!B}sVU3@?1f*-!j3(M6gvG(vvp*lP=%>d`hu8GGVv+0YI*foYKOHaRiH1N&E(2y!i&3|i3TDu1jPMjr$Q8NMrT=BujsWzjMCh(Hw zX`5X`;b4BHRpR~Kd|5-nXS>~x6YA#TKFyKe+Ja91$vG_m9$OCI+b~T!g#!iT;_f4$ z#-N@xWs72}a%o2mZR%f~O0&d(*;P9lzoUG#gPRqj>?4zva?yT<7{{IVvr_`Hwlb}H zM{1mQ+kCEUg-Em+AUH^WDHvPlx8VkHAfWovzAQJ5Nkz#@Ccksk$c= zGg~^HPfef~)-0coCsq$j(=d%A`Q3y-`50xSSgI94`HVRLuBl!34Vn-P|NafE{bZV?ZZtvZ70^m*i*a4V{{lCL_7vg@<_mJUIzt?X!Zsf=qKaK-pZ}j-xR!vJL zdinwg`{C4t<8C5MSzy@ZXu0UYHUqT(xP4M(*HNGgJ5*8>jR7KT z7cRa3@EtJ(z*lgsy>KCQR2G1OBm?X~p&~Zy7!g^se2PB{I=gbTcYUlo$V&~83yRBo z2CDmhmfN?6SRdLK_6x5!juyA&+A zEC%e~nSlhIgwKCOBYB0D0wp9Rd;DO~+Wng1X0gO=4yUQcUtMhHL%x(IUZj4D49yaT zYPVZU8ND;Gb)FL!P#Dt{plNNEwQ9{>@yxs2$qH38a{1^Y=@Op!XHZxP#Ifz2BMU=l zJdIU9g}7eFTyJmw6mFwj81%>1*2_rlXG_0zRg~x1y3%^E*JDO3YCB+SHKp$bin_(DMIlZuhQUm;V_ARPy`L7=>RDa8H6!@&-)71y6!}aI zs(N)(I$?+%^nHi8u8r+i;eXemYZG+DHv5FQY%OGQ?XpAWiMAm&m*H{yY}*CeY6wEW zVbHd}c-YV=KceHBI)^p!PYJ$|a!e!f3?bCjvSooUhwz2T@C*1LcrUKlEf)QV1Oou) ziEq5Ou0d>eW=0px7AlSQRD4f=^fk$%t9;4dtR&B<4X0Y!Er>|}0VbnTgo`?hrNg5h zD5i2JnI9JD393QC7il0OJvWzLd0(RUHSiVY@?>-JtCmgfNUQDu5*itWv~r%c^Dkk? z5jPgm$jgg2heNsO!u>Q#wk`heWMz^-hpTO{4*8xAndr|JlwZjrM}5!>5RrkwWIHS-Q7AfMT%$F*ACC5C1L%j zdlJVFOKVR7aVZ9UG9&^16LId*)vNQpf0 z2!*w}YbZ#{)yG+&6LBH&+BRcH?q7Mt5x-!|wre{$Gdtn5^)~}O_KWZ#EACx-kw?bHyrnuLTs|=ZOO*B{&8k86F%2mDzgKs z@PT?7rIK+}p9nwFvlPn5FZ73PPuKMLMO`Y}U`uR9lDc;?pv#?Ri=V=Yq=lrVOt(n& zh7+LI^R?gbajG^uGZMI;37o*Mj{!t!DYLZTy)BWq={3t!=n8erLAR29(0;&b@BuYX z&mEhH>JI1epjPB^R`xGqD*SBvWC(sjPtj3{Fk8`r0me}>nR2~4CU%~ZX(a0L%3Fm$ z1>LaO>Lwj1Df7=TSxwRc279<+X%S)N(9KWxM}J1FzWahfD*epJh678spMd3fe`Rul zZwYAG6Y@;%4*(Tw8p)E5Cqq7gmS#Gf&QA!!6_9+-4|e&aS#r4E=mHJUTcMd~06GrJ zS23sQzduq<%mpP!%taGf#B6V)6$J%B$4hMiY-R=>PkPdemD>5?yic*fXKFJ@lj}Ba z@eUhZt;&3C?`np>ZKQegHV?(E={jgkhQD}zN%W}`hV!C(I^trO( z=lL<_2DVsT%5rjYVM!r<;X}VPT+vgFSK-|C#>=(&!w7iQa1Y%(x5T6ocuoT<^*_7obs#g6+tI1jdU>@kLStf}4th0xtm)Unwglf)tK}KCPWIfcr z5Cr$Nt|EV7L@y?}hZq-uNJcTv_xPi9=N$bjv3Nh?XTot={pO zIC)jVRRMdk{p^hsOK(}ZW|I=`K74#8JR!n`fbUyfVA$ch_2Oq|-Ce-n+ceQqcD&?GXADo3E+~4P(F_?=Sm=}6Pk1>= zi>BrSf5a(#r2fu$1l6y6r5ACrYhi4;ZZ@U9h}C_Hp`Mlx@H>4DZm&C?3KrW|et6Q4kj!n;ULV9(EC%np%2&^A3ri1mWuH${rqz zq20(IIeWAVt(VomY=V*i=**e&OuPtL=ESLQRwomX7~O8nd5!6)lhlW}ogjW&xMNIM zpfP3)VO?V`|7zP%a9eQShI=2I+HrDVhk}i#4|keMC(B=NXs{p(oj|faM-Gc0_<*$N z=Oh?67&kh0Pe5z)dr&EtsjAx1)5|A%N#WO^=HQ1mO$6g*`f8f7+J>tUXE{@GpmZ-~ zcnj(;#D)(TusmUh{D_J1D^;%#5m<$`#*qqZG2(b<2({+2haayzqS4}7OJD|?C%8@+ z^Tn(t0%kCMfwuE*o?#VjpD zIjD+b@dJ$y%?(?7^x0Djq(RFY2>K!lKXV@PZM)f-q0j96(K#A|h6@pS9?eEz?H`!U zk1a<`g^{DCPf?B)=c>&WJO7|gIlvcy%#mCRVT?#O;qSlbgcmSQ*?Yn@rc$P;C=3^2 zC9FH@Sc51j_CeZy-qk4la)2 zTv%AN<8DJkb~y+-kA$7Cz^~|Vz=Y!%vtf8`OrhaeT#Zh-iNQ!pTlGZ}&T+zo3GbN! zgojST+nvW>^Es8Hn5^sSrVmV)Z2dkzlbd_frYWQQDICETd)hWJjfMTI-NqCS7e6=bxNA;_pp!8j=$gSFSb%xHaAb?JEFF3QejH%>ZW}PFUoSbZ-3s@ z7!`qp6ecF-L$FYUD$OJHQHo2KRpj#gEW2uHr~z!1g9bHUH?w*6upf5EMCuxSUd=G5 z{0c0Bp8lD~g42bWgnFc@rdHHNh(V`OV0e|FFxbg)3fo}~MUnkCr-ZC4TqVW$&)Ei- zZbao>M6dE?V>Nw1!MND~P@Q;h0kIL+_^ObL=evzfWn8p@Go&GzDpBkPhqt%){dYF( z19z*407XIT>5#qOjIrUA-uot@x^x){)5C*Z8ay@A{_ic9bx^-`On6dnx%(z5@cJw{e71IG z^65mK;9`(ln2IB*JT#@2&D@R}45^ZJy-A>c8x!c&PFZvGhj0+9O#~ljos)u^*(TxD zRx>uSfM8Mx7H#Q>kl-K&lVyCBLmbZYpQ)w6ig!=v`A#X~ND3;Mn{Y#AU+oZIGv(E% zFx~-lD3?jxT#znI$-L>v8E9t`!={9t_qhj+FC>5BCXoL@pzmItdzQ1>$%>xkUgcw~ zv=0_+H)hL0vo>grQ$CiuC!Olv>a;)J(Sl}w{!Kwe_va}v!1F3?1PwIPl3wreKVt*F zNqT57%PMbrYxCW1GFLYpDXCDPjBI5{_$?}0c|!&>I@i_|*^^X~nLBFxGke4SdhL2A z7YmC*pg}m$b(e`O*OUs=oZG5)WxFY3qlcnJe|fm8zgu=ZHhUkR+B=LQO^IL^$`**V zQwI_I$m9Dd^&X0rvlfahzdoy3hY-tWQXG5xwZ&ow`v)w()~CbBM*2h3q8Y+6Y6WGC zOp$|_s#?SlVHtZuzaE4*y-#U2oSJNKIP@KE`eKouU?fHpzn4R*#A@a)eO46}&@ z=grML_(wy(Y@SfEZN4DU#S&q(i*psl8RtME|~sY&VZ?S1wFMTDd;s9#I;98 zMwYROuvK{=mbdYuIGSCDYCqr4?avg)W2Tf|=6&(URFYh2_4rHwTSMkz;I*LRx0Fj^ zkr<=x&waS8wJvF41NrCugGp)2e_|qB4>U+Go{>qG0EdXdcKZttNL{*ae7);^=`7*< zVh?O-*G(Wquw)}2zi72fdaVS(lH>DM{6rUTKx2r;&fv?up;3VF( zQmTB>$2|wVf=DY|^i5P~&m9yb=m_P?O|c@ig7;46dBsO14U_v%pdwsz%L9&afp5mK zZf|erJpI0v$Q$#pvepXsopOcNlRPqgtM?BT|20d|$v7`Qv6Acs`k#*Dqg()kjc@Emh) z`IWJ?nILU|iiTGEK$41f)mnXCh=8`5*wp1;8x@YujailyD0SkNZT#?wBwuf8o*)02 zpg~;F>-xFEfkgpl<)NG3q{B9U`TK<-^7?0^urcMGWkW?{+yP# zDzQ0b=~oS&B_$JgpkXVLa{1is-eouM;LeOn-012rFxpUmj`Z2wnc`UPT80*D2RXZW zLosgbKc6*;=ymTtvqWe)C>uT42EgxSVWx^S9;61>fDFm(-|?xzZjL zYB&xW(BtdIz$6hLA)QA`&bvpOW>ChMm&ka&8bkky0RAzfewi|RdHaNX4R3MD!bdYfhM>q@4_b|fz051I>FN~kbGESWsT*K`5 zggfHci=BTA*l>I#ZdeDGN+>qg`BRuf!(>4nNkp1AS8#z6X2^(ow#QG04Fj2p6=!q0 z$9O)3N+SraFkZ`5jG_w+ICc6w6?Lyw``$m7X*$E6T|F^{(r&A&$Dv{^&E>}?ET$x} zFQL=s!n?b-ABaicCPKX%ky592O#-|Be{rsE(BmA&?p$3&Sn1|wP*T0AQ!+2=e%$63 zu~!KBYF?FKcDyunfukN%(8br2N)8^Lie4W1f)~fIJl-^k$-p$)SM3mGPD8=&SBe_&;oMyF7{s?maCnNGtUyURY zuT|x*j9as=c@v@yy$>R_wr0dgLxM_T!eVD5*p(!WM@eG(ad(a2`!IJ# zlr%a`Uso7x>3f@`u zbT-Z2c1WwQTD`D~B9-ZpDcuB3ypXgqay;LEo>`3!1VZ>lD1x;spUFC7<+Or#3{3V_ z43oV-Oz72xtG@m>XhpFc)^M)@4&&UlouPPrP`Ktgh$#}-qF{{7B>3K1n#>j?Ao}j*k zi=R>an3`Un7GJz;@l&&AwvCE0fry3nOONt5A>UOE46wVV3XEQs6hEutOpoo!p*>7~43CQX{hxaS#)gjnE5l1X6A9uR z(yV&6fHIlNZd`}iaGmd<4U8IITu)g-bi4l|C@0!$+yQcH=-{F)&-2Keg3@PIwJE)K z1oy%Y7PV=rW^HWo!!ltPUw`!f+wZa+8e(E%LM>t}s4|t2oV8Q6Ml?wuQsR)t9f%U} zmMzyxrTU(g4_T2`71Zrhu(67U5eRx`vgqQ76JvdZdGi;c`6MEDJuKVC|GCkh+ZiBA z;FAh#MvIGzzRRLAnlm{fN|0e6zR;fLU2={sFG4P1;1JHi0b6al@YZA}^eh89M#0rM zLEfZCRq;JU-FtE{K!n zB%c%s#C2BzKl|4~UNQiP8&7#hW!i8kYkiw)O{<$~xP%Hq3}@&52o`X?sPO-+Eq-22 zFjvP5^2#|HO^&5=*gGlA#Xp$xG?FYJL4%>J;^4))TmP1yBytr*`aAvzS{L%Xhf&%| zq2Ghsn7H)`tJMv&|2u7dfLxaFs_X_V19YWQwVmClWInt-9`pGM?IJn0mcHFJ!Ox$4 z1SuA%sL|1>^By!@yaoGk9mW@`cogOovh)p#7N~+`EippP|LtNF_5pN3WA{SGhl6XO z;PGaP03l$=J-U%=$j#25$cyB73;ux`{0V5H{d62bphOu28_c>?jJ0D_*)Q_%!+?f} zX}i3=?E^*gFu;0c1eAgz-fhq|$adeEvtJ9MbUeT7LkcBF24YG5t&ZSLKHuFyXV<_S&$?M)ZTSAtG|saGbJgwl&5A5Q>p^pF$es*>EGufF2> zR29jAgW+b{>tqgb$0q@W8HEh1zzz&Ia5Z9_yU+i9rVKvqz*S${{DF5oZS`g0(mV^Y zi`(UcfPZs*xN>Qk~k9<`a`-9Z?! zwSUJjvUN=R_CWMGkO}mG?g84N`ev_iT>k>?Os+s?-S;g*C(sm+r(1KG|H6Q6On?(R zk1qktn<$*QwI@Uk`|dO}Tx(J*%u@+ZXhsX%7AL0=+0!gIqLqIiyCj8-hDI2u#qUS7 z{ypW^Jg+`z;aYtoXbPmiqF7xvP^hn{VCdtHCP@`p4=i!`V|)Ae%aQ#@E|F_%wYCVy z7@K>F<6*VX^T#}zJCRFt9^9)I`A%gV7x34>(GJlIZ)3%S(i04bX;D zB$5Du?mY(uV?RRE`*`@2I37H~rtqG~#5H2|qj>GA&?PoJ-fB_o?IVV&zVJGnZC6XZ zn5GldflvQlngmiM!sGgIhO6+W)@4{2ylA7zBqL~HIz#Xm@Qc@%tlESDr6Kx0+Ba1i zJb*35&pVX(eIEPMNTY=90R*?jIivWA#f-$)uB`ElYquCJ){h?1cD8IO_MZ!)Um~Zgb zfXVxQ4wBY1*;XUiT>F7WEr8+I`v0fl;9jdbNK z*PTCPa7<@r_13|~jl#jA4mV(^WbqO-h}Q}`5G3f{5j!xtH8am9b!8Lz()w@b5T1y3 z(2_S8kZcaHE?<)O09neLl~r=|X~`8e3}5?~HPK5EOh`y*{e|xO%Vx`*SL5aq8Dg*a z>-ovk&U{qM>xoSK7_P6L*Qd=D+0C1@$72ftK9d?63pQB$xmG8b`}+%__aWKa+Z_MS zTq+Rg=;+QH{+MTg_HZyFWjKV4g%$L& zrN{J)+iiiA!ocrIdh@Ih%a4?yY3)WTeDg_92v&{8!_G}z*nu+wlR(Jv>HALbnCM)> zkhS8VbGJVkz(sXE_E1`V5#sx|!={JK8DpNMVMoGfKG2is5-@FvHpK1^?7wvZE z!K-MI+Au?uX3Pbm(Xp04gN(YeFHY^)-5_d8F-I}Nna^Z2hq|Mh zOV?6{@JV{Qh~V_~ zFZVCaFxz(>1G@jdOPI(UPNykxG6oZbuKf!I>!<_-h}4Ziv%fGkgfOKm39c8>w3J*B zV_DI~D|2C=2KA*Poxdbo`@It>z^<`9r~M$(Pz>|+`QHn^jOPp#2)T)BlusA`CH4Hz zFtFx){Z}qplMf&J-=+rt4F4J)2q3~Xo6#BnzuJ?n>q0&ueHMj^deSOcc?EvNgr$Va IKI{1WA3s+3)c^nh literal 0 HcmV?d00001 From 97f71db21ba398659be6166b32da51b1aa558dff Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Wed, 10 Oct 2018 21:24:08 +0000 Subject: [PATCH 21/23] Put in-line image on its own paragraph Signed-off-by: Matthias Rampke --- pkg/mapper/fsm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mapper/fsm/README.md b/pkg/mapper/fsm/README.md index 02a806d..722ee21 100644 --- a/pkg/mapper/fsm/README.md +++ b/pkg/mapper/fsm/README.md @@ -125,8 +125,8 @@ mappings: ``` will be rendered as: -![FSM](fsm.png) +![FSM](fsm.png) The `dot` program is part of [Graphviz](https://www.graphviz.org/) and is available in most of popular operating systems. From e5734e34e91b6febaae16dc86535d101a2f91cf6 Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Wed, 10 Oct 2018 21:31:51 +0000 Subject: [PATCH 22/23] Copy edit comments in fsm/formatter Fixing a typo and language. Signed-off-by: Matthias Rampke --- pkg/mapper/fsm/formatter.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/mapper/fsm/formatter.go b/pkg/mapper/fsm/formatter.go index 8ae0926..567bbc2 100644 --- a/pkg/mapper/fsm/formatter.go +++ b/pkg/mapper/fsm/formatter.go @@ -30,8 +30,8 @@ type TemplateFormatter struct { fmtString string } -// NewTemplateFormatter instantialize a TemplateFormatter -// from given template string and the maxium amount of captures. +// NewTemplateFormatter instantiates a TemplateFormatter +// from given template string and the maximum amount of captures. func NewTemplateFormatter(template string, captureCount int) *TemplateFormatter { matches := templateReplaceCaptureRE.FindAllStringSubmatch(template, -1) if len(matches) == 0 { @@ -60,8 +60,8 @@ func NewTemplateFormatter(template string, captureCount int) *TemplateFormatter } } -// Format accepts a list containing captured strings and return the formatted string -// using the template stored in current TemplateFormatter. +// Format accepts a list containing captured strings and returns the formatted +// string using the template stored in current TemplateFormatter. func (formatter *TemplateFormatter) Format(captures []string) string { if formatter.captureCount == 0 { // no label substitution, keep as it is From 4e534403160ce1391069a757d1c4026a175b202f Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Wed, 10 Oct 2018 21:37:26 +0000 Subject: [PATCH 23/23] Move `bench` target out of common Makefile we should not change the vendored Makefile.common here. Signed-off-by: Matthias Rampke --- Makefile | 7 +++++++ Makefile.common | 5 ----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 36af97c..57df555 100644 --- a/Makefile +++ b/Makefile @@ -17,3 +17,10 @@ STATICCHECK_IGNORE = \ github.com/prometheus/statsd_exporter/main.go:SA1019 \ DOCKER_IMAGE_NAME ?= statsd-exporter + +.PHONY: bench +bench: + @echo ">> running all benchmarks" + $(GO) test -bench . -race $(pkgs) + +all: bench diff --git a/Makefile.common b/Makefile.common index 528d404..eaee9f0 100644 --- a/Makefile.common +++ b/Makefile.common @@ -66,11 +66,6 @@ test: @echo ">> running all tests" $(GO) test -race $(pkgs) -.PHONY: bench -bench: - @echo ">> running all benchmarks" - $(GO) test -bench . -race $(pkgs) - .PHONY: format format: @echo ">> formatting code"