Add YAML config file and ability to set histograms

exporter.go

@@ -141,6 +141,36 @@ func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels) prom
 	return summary
 }
 
+type HistogramContainer struct {
+	Elements map[uint64]prometheus.Histogram
+}
+
+func NewHistogramContainer() *HistogramContainer {
+	return &HistogramContainer{
+		Elements: make(map[uint64]prometheus.Histogram),
+	}
+}
+
+func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, mapping *metricMapping) prometheus.Histogram {
+	hash := hashNameAndLabels(metricName, labels)
+	histogram, ok := c.Elements[hash]
+	if !ok {
+		// TODO: buckets
+		histogram = prometheus.NewHistogram(
+			prometheus.HistogramOpts{
+				Name:        metricName,
+				Help:        defaultHelp,
+				ConstLabels: labels,
+				Buckets:     mapping.Buckets,
+			})
+		c.Elements[hash] = histogram
+		if err := prometheus.Register(histogram); err != nil {
+			log.Fatalf(regErrF, metricName, err)
+		}
+	}
+	return histogram
+}
+
 type Event interface {
 	MetricName() string
 	Value() float64
@@ -181,11 +211,12 @@ func (c *TimerEvent) Labels() map[string]string { return c.labels }
 type Events []Event
 
 type Exporter struct {
-	Counters  *CounterContainer
+	Counters   *CounterContainer
-	Gauges    *GaugeContainer
+	Gauges     *GaugeContainer
-	Summaries *SummaryContainer
+	Summaries  *SummaryContainer
-	mapper    *metricMapper
+	Histograms *HistogramContainer
-	addSuffix bool
+	mapper     *metricMapper
+	addSuffix  bool
 }
 
 func escapeMetricName(metricName string) string {
@@ -218,7 +249,7 @@ func (b *Exporter) Listen(e <-chan Events) {
 			metricName := ""
 			prometheusLabels := event.Labels()
 
-			labels, present := b.mapper.getMapping(event.MetricName())
+			mapping, labels, present := b.mapper.getMapping(event.MetricName())
 			if present {
 				metricName = labels["name"]
 				for label, value := range labels {
@@ -263,12 +294,20 @@ func (b *Exporter) Listen(e <-chan Events) {
 				eventStats.WithLabelValues("gauge").Inc()
 
 			case *TimerEvent:
-				summary := b.Summaries.Get(
+				if mapping.TimerType == "histogram" {
-					b.suffix(metricName, "timer"),
+					histogram := b.Histograms.Get(
-					prometheusLabels,
+						b.suffix(metricName, "timer"),
-				)
+						prometheusLabels,
-				summary.Observe(event.Value())
+						mapping,
-
+					)
+					histogram.Observe(event.Value())
+				} else {
+					summary := b.Summaries.Get(
+						b.suffix(metricName, "timer"),
+						prometheusLabels,
+					)
+					summary.Observe(event.Value())
+				}
 				eventStats.WithLabelValues("timer").Inc()
 
 			default:
@@ -281,11 +320,12 @@ func (b *Exporter) Listen(e <-chan Events) {
 
 func NewExporter(mapper *metricMapper, addSuffix bool) *Exporter {
 	return &Exporter{
-		addSuffix: addSuffix,
+		addSuffix:  addSuffix,
-		Counters:  NewCounterContainer(),
+		Counters:   NewCounterContainer(),
-		Gauges:    NewGaugeContainer(),
+		Gauges:     NewGaugeContainer(),
-		Summaries: NewSummaryContainer(),
+		Summaries:  NewSummaryContainer(),
-		mapper:    mapper,
+		Histograms: NewHistogramContainer(),
+		mapper:     mapper,
 	}
 }
 
@@ -373,7 +413,8 @@ func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
 		} else {
 			samples = strings.Split(elements[1], ":")
 		}
-		samples: for _, sample := range samples {
+	samples:
+		for _, sample := range samples {
 			components := strings.Split(sample, "|")
 			samplingFactor := 1.0
 			if len(components) < 2 || len(components) > 4 {

mapper.go

@@ -16,11 +16,13 @@ package main
 import (
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
+	yaml "gopkg.in/yaml.v2"
 )
 
 var (
@@ -32,16 +34,25 @@ var (
 	metricNameRE = regexp.MustCompile(`^` + identifierRE + `$`)
 )
 
-type metricMapping struct {
+type mapperConfigDefaults struct {
-	regex  *regexp.Regexp
+	TimerType string    `yaml:"timer_type"`
-	labels prometheus.Labels
+	Buckets   []float64 `yaml:"buckets"`
 }
 
 type metricMapper struct {
-	mappings []metricMapping
+	Defaults mapperConfigDefaults `yaml:"defaults"`
+	Mappings []metricMapping      `yaml:"mappings"`
 	mutex    sync.Mutex
 }
 
+type metricMapping struct {
+	Match     string `yaml:"match"`
+	regex     *regexp.Regexp
+	Labels    prometheus.Labels
+	TimerType string    `yaml:"timer_type"`
+	Buckets   []float64 `yaml:"buckets"`
+}
+
 type configLoadStates int
 
 const (
@@ -55,9 +66,9 @@ func (m *metricMapper) initFromString(fileContents string) error {
 	state := SEARCHING
 
 	parsedMappings := []metricMapping{}
-	currentMapping := metricMapping{labels: prometheus.Labels{}}
+	currentMapping := metricMapping{Labels: prometheus.Labels{}}
 	for i, line := range lines {
-		line := strings.TrimSpace(line)
+		line = strings.TrimSpace(line)
 
 		switch state {
 		case SEARCHING:
@@ -81,15 +92,15 @@ func (m *metricMapper) initFromString(fileContents string) error {
 				return fmt.Errorf("Line %d: missing terminating newline", i)
 			}
 			if line == "" {
-				if len(currentMapping.labels) == 0 {
+				if len(currentMapping.Labels) == 0 {
 					return fmt.Errorf("Line %d: metric mapping didn't set any labels", i)
 				}
-				if _, ok := currentMapping.labels["name"]; !ok {
+				if _, ok := currentMapping.Labels["name"]; !ok {
 					return fmt.Errorf("Line %d: metric mapping didn't set a metric name", i)
 				}
 				parsedMappings = append(parsedMappings, currentMapping)
 				state = SEARCHING
-				currentMapping = metricMapping{labels: prometheus.Labels{}}
+				currentMapping = metricMapping{Labels: prometheus.Labels{}}
 				continue
 			}
 			if err := m.updateMapping(line, i, &currentMapping); err != nil {
@@ -102,40 +113,90 @@ func (m *metricMapper) initFromString(fileContents string) error {
 
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
-	m.mappings = parsedMappings
+	m.Mappings = parsedMappings
 
 	mappingsCount.Set(float64(len(parsedMappings)))
 
 	return nil
 }
 
+func (m *metricMapper) initFromYamlString(fileContents string) error {
+
+	var n metricMapper
+
+	if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
+		return err
+	}
+
+	if n.Defaults.Buckets == nil || len(n.Defaults.Buckets) == 0 {
+		n.Defaults.Buckets = prometheus.DefBuckets
+	}
+
+	for i := range n.Mappings {
+		currentMapping := &n.Mappings[i]
+
+		if !metricLineRE.MatchString(currentMapping.Match) {
+			return fmt.Errorf("invalid match: %s", currentMapping.Match)
+		}
+
+		// Translate the glob-style metric match line into a proper regex that we
+		// can use to match metrics later on.
+		metricRe := strings.Replace(currentMapping.Match, ".", "\\.", -1)
+		metricRe = strings.Replace(metricRe, "*", "([^.]+)", -1)
+		currentMapping.regex = regexp.MustCompile("^" + metricRe + "$")
+
+		if currentMapping.TimerType == "" {
+			currentMapping.TimerType = n.Defaults.TimerType
+		}
+
+		if currentMapping.Buckets == nil || len(currentMapping.Buckets) == 0 {
+			currentMapping.Buckets = n.Defaults.Buckets
+		}
+	}
+
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	m.Defaults = n.Defaults
+	m.Mappings = n.Mappings
+
+	mappingsCount.Set(float64(len(n.Mappings)))
+
+	return nil
+}
+
 func (m *metricMapper) initFromFile(fileName string) error {
 	mappingStr, err := ioutil.ReadFile(fileName)
 	if err != nil {
 		return err
 	}
-	return m.initFromString(string(mappingStr))
+	switch strings.ToLower(filepath.Ext(fileName)) {
+	case ".yaml", ".yml":
+		return m.initFromYamlString(string(mappingStr))
+	default:
+		return m.initFromString(string(mappingStr))
+	}
 }
 
-func (m *metricMapper) getMapping(statsdMetric string) (labels prometheus.Labels, present bool) {
+func (m *metricMapper) getMapping(statsdMetric string) (*metricMapping, prometheus.Labels, bool) {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	for _, mapping := range m.mappings {
+	for _, mapping := range m.Mappings {
 		matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
 		if len(matches) == 0 {
 			continue
 		}
 
 		labels := prometheus.Labels{}
-		for label, valueExpr := range mapping.labels {
+		for label, valueExpr := range mapping.Labels {
 			value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
 			labels[label] = string(value)
 		}
-		return labels, true
+		return &mapping, labels, true
 	}
 
-	return nil, false
+	return nil, nil, false
 }
 
 func (m *metricMapper) updateMapping(line string, i int, mapping *metricMapping) error {
@@ -147,6 +208,6 @@ func (m *metricMapper) updateMapping(line string, i int, mapping *metricMapping)
 	if label == "name" && !metricNameRE.MatchString(value) {
 		return fmt.Errorf("Line %d: metric name '%s' doesn't match regex '%s'", i, value, metricNameRE)
 	}
-	(*mapping).labels[label] = value
+	(*mapping).Labels[label] = value
 	return nil
 }