Merge pull request #361 from glightfoot/defaults2

Allow setting defaults for histogram and summary options

README.md

@@ -341,9 +341,9 @@ mappings:
         error: 0.05
       - quantile: 0.5
         error: 0.005
-    max_summary_age: 30s
+    max_age: 30s
-    summary_age_buckets: 3
+    age_buckets: 3
-    stream_buffer_size: 1000
+    buf_cap: 1000
 ```
 
 The default quantiles are 0.99, 0.9, and 0.5.
@@ -414,16 +414,34 @@ mappings:
 
 ### Global defaults
 
-One may also set defaults for the observer type, buckets or quantiles, and match type.
+One may also set defaults for the observer type, histogram options, summary options, and match type.
 These will be used by all mappings that do not define them.
 
 An option that can only be configured in `defaults` is `glob_disable_ordering`, which is `false` if omitted.
 By setting this to `true`, `glob` match type will not honor the occurance of rules in the mapping rules file and always treat `*` as lower priority than a concrete string.
 
+Setting `buckets` or `quantiles` in the defaults is deprecated in favor of `histogram_options` and `summary_options`, which will override the deprecated values.
+
+If `summary_options` is present in a mapping config, it will only override the fields set in the mapping. Unset fields in the mapping will take the values from the defaults. 
+
 ```yaml
 defaults:
   observer_type: histogram
-  buckets: [.005, .01, .025, .05, .1, .25, .5, 1, 2.5 ]
+  histogram_options:
+    buckets: [.005, .01, .025, .05, .1, .25, .5, 1, 2.5 ]
+  summary_options:
+    quantiles:
+      - quantile: 0.99
+        error: 0.001
+      - quantile: 0.95
+        error: 0.01
+      - quantile: 0.9
+        error: 0.05
+      - quantile: 0.5
+        error: 0.005
+    max_age: 5m
+    age_buckets: 2
+    buf_cap: 1000
   match_type: glob
   glob_disable_ordering: false
   ttl: 0 # metrics do not expire
@@ -435,7 +453,7 @@ mappings:
     provider: "$2"
     outcome: "$3"
     job: "${1}_server"
-# This will be a summary timer.
+# This will be a summary using the summary_options set in `defaults`
 - match: "other.distribution.*.*.*"
   observer_type: summary
   name: "other_distribution"

pkg/exporter/exporter.go

@@ -20,6 +20,7 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/prometheus/statsd_exporter/pkg/clock"
 	"github.com/prometheus/statsd_exporter/pkg/event"
 	"github.com/prometheus/statsd_exporter/pkg/mapper"

pkg/mapper/mapper.go

@@ -22,8 +22,9 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
-	"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
 	yaml "gopkg.in/yaml.v2"
+
+	"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
 )
 
 var (
@@ -77,12 +78,12 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op
 		return err
 	}
 
-	if n.Defaults.Buckets == nil || len(n.Defaults.Buckets) == 0 {
+	if len(n.Defaults.HistogramOptions.Buckets) == 0 {
-		n.Defaults.Buckets = prometheus.DefBuckets
+		n.Defaults.HistogramOptions.Buckets = prometheus.DefBuckets
 	}
 
-	if n.Defaults.Quantiles == nil || len(n.Defaults.Quantiles) == 0 {
+	if len(n.Defaults.SummaryOptions.Quantiles) == 0 {
-		n.Defaults.Quantiles = defaultQuantiles
+		n.Defaults.SummaryOptions.Quantiles = defaultQuantiles
 	}
 
 	if n.Defaults.MatchType == MatchTypeDefault {
@@ -190,7 +191,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op
 				currentMapping.HistogramOptions.Buckets = currentMapping.LegacyBuckets
 			}
 			if currentMapping.HistogramOptions.Buckets == nil || len(currentMapping.HistogramOptions.Buckets) == 0 {
-				currentMapping.HistogramOptions.Buckets = n.Defaults.Buckets
+				currentMapping.HistogramOptions.Buckets = n.Defaults.HistogramOptions.Buckets
 			}
 		}
 
@@ -205,7 +206,16 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op
 				currentMapping.SummaryOptions.Quantiles = currentMapping.LegacyQuantiles
 			}
 			if currentMapping.SummaryOptions.Quantiles == nil || len(currentMapping.SummaryOptions.Quantiles) == 0 {
-				currentMapping.SummaryOptions.Quantiles = n.Defaults.Quantiles
+				currentMapping.SummaryOptions.Quantiles = n.Defaults.SummaryOptions.Quantiles
+			}
+			if currentMapping.SummaryOptions.MaxAge == 0 {
+				currentMapping.SummaryOptions.MaxAge = n.Defaults.SummaryOptions.MaxAge
+			}
+			if currentMapping.SummaryOptions.AgeBuckets == 0 {
+				currentMapping.SummaryOptions.AgeBuckets = n.Defaults.SummaryOptions.AgeBuckets
+			}
+			if currentMapping.SummaryOptions.BufCap == 0 {
+				currentMapping.SummaryOptions.BufCap = n.Defaults.SummaryOptions.BufCap
 			}
 		}
 

pkg/mapper/mapper_defaults.go

@@ -16,19 +16,30 @@ package mapper
 import "time"
 
 type mapperConfigDefaults struct {
+	ObserverType        ObserverType     `yaml:"observer_type"`
+	MatchType           MatchType        `yaml:"match_type"`
+	GlobDisableOrdering bool             `yaml:"glob_disable_ordering"`
+	Ttl                 time.Duration    `yaml:"ttl"`
+	SummaryOptions      SummaryOptions   `yaml:"summary_options"`
+	HistogramOptions    HistogramOptions `yaml:"histogram_options"`
+}
+
+// mapperConfigDefaultsAlias is used to unmarshal the yaml config into mapperConfigDefaults and allows deprecated fields
+type mapperConfigDefaultsAlias struct {
 	ObserverType        ObserverType      `yaml:"observer_type"`
-	TimerType           ObserverType      `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty
+	TimerType           ObserverType      `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs
-	Buckets             []float64         `yaml:"buckets"`
+	Buckets             []float64         `yaml:"buckets"`              // DEPRECATED - field only present to preserve backwards compatibility in configs
-	Quantiles           []metricObjective `yaml:"quantiles"`
+	Quantiles           []metricObjective `yaml:"quantiles"`            // DEPRECATED - field only present to preserve backwards compatibility in configs
 	MatchType           MatchType         `yaml:"match_type"`
 	GlobDisableOrdering bool              `yaml:"glob_disable_ordering"`
 	Ttl                 time.Duration     `yaml:"ttl"`
+	SummaryOptions      SummaryOptions    `yaml:"summary_options"`
+	HistogramOptions    HistogramOptions  `yaml:"histogram_options"`
 }
 
 // UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys
 // observer_type will override timer_type
 func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	type mapperConfigDefaultsAlias mapperConfigDefaults
 	var tmp mapperConfigDefaultsAlias
 	if err := unmarshal(&tmp); err != nil {
 		return err
@@ -36,16 +47,26 @@ func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error)
 
 	// Copy defaults
 	d.ObserverType = tmp.ObserverType
-	d.Buckets = tmp.Buckets
-	d.Quantiles = tmp.Quantiles
 	d.MatchType = tmp.MatchType
 	d.GlobDisableOrdering = tmp.GlobDisableOrdering
 	d.Ttl = tmp.Ttl
+	d.SummaryOptions = tmp.SummaryOptions
+	d.HistogramOptions = tmp.HistogramOptions
 
 	// Use deprecated TimerType if necessary
 	if tmp.ObserverType == "" {
 		d.ObserverType = tmp.TimerType
 	}
 
+	// Use deprecated quantiles if necessary
+	if len(tmp.SummaryOptions.Quantiles) == 0 && len(tmp.Quantiles) > 0 {
+		d.SummaryOptions = SummaryOptions{Quantiles: tmp.Quantiles}
+	}
+
+	// Use deprecated buckets if necessary
+	if len(tmp.HistogramOptions.Buckets) == 0 && len(tmp.Buckets) > 0 {
+		d.HistogramOptions = HistogramOptions{Buckets: tmp.Buckets}
+	}
+
 	return nil
 }

pkg/registry/registry.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/statsd_exporter/pkg/clock"
 	"github.com/prometheus/statsd_exporter/pkg/mapper"
 	"github.com/prometheus/statsd_exporter/pkg/metrics"
@@ -248,7 +249,7 @@ func (r *Registry) GetHistogram(metricName string, labels prometheus.Labels, hel
 	var histogramVec *prometheus.HistogramVec
 	if vh == nil {
 		metricsCount.WithLabelValues("histogram").Inc()
-		buckets := r.Mapper.Defaults.Buckets
+		buckets := r.Mapper.Defaults.HistogramOptions.Buckets
 		if mapping.HistogramOptions != nil && len(mapping.HistogramOptions.Buckets) > 0 {
 			buckets = mapping.HistogramOptions.Buckets
 		}
@@ -295,14 +296,21 @@ func (r *Registry) GetSummary(metricName string, labels prometheus.Labels, help
 	var summaryVec *prometheus.SummaryVec
 	if vh == nil {
 		metricsCount.WithLabelValues("summary").Inc()
-		quantiles := r.Mapper.Defaults.Quantiles
+		quantiles := r.Mapper.Defaults.SummaryOptions.Quantiles
 		if mapping != nil && mapping.SummaryOptions != nil && len(mapping.SummaryOptions.Quantiles) > 0 {
 			quantiles = mapping.SummaryOptions.Quantiles
 		}
-		summaryOptions := mapper.SummaryOptions{}
+
+		summaryOptions := mapper.SummaryOptions{
+			MaxAge:     r.Mapper.Defaults.SummaryOptions.MaxAge,
+			AgeBuckets: r.Mapper.Defaults.SummaryOptions.AgeBuckets,
+			BufCap:     r.Mapper.Defaults.SummaryOptions.BufCap,
+		}
+
 		if mapping != nil && mapping.SummaryOptions != nil {
 			summaryOptions = *mapping.SummaryOptions
 		}
+
 		objectives := make(map[float64]float64)
 		for _, q := range quantiles {
 			objectives[q.Quantile] = q.Error