diff options
Diffstat (limited to 'src/go/collectors/go.d.plugin/pkg/metrics')
11 files changed, 1206 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/counter.go b/src/go/collectors/go.d.plugin/pkg/metrics/counter.go new file mode 100644 index 000000000..7231fc7a4 --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/counter.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "errors" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" +) + +type ( + // Counter is a Metric that represents a single numerical bits that only ever + // goes up. That implies that it cannot be used to count items whose number can + // also go down, e.g. the number of currently running goroutines. Those + // "counters" are represented by Gauges. + // + // A Counter is typically used to count requests served, tasks completed, errors + // occurred, etc. + Counter struct { + valInt int64 + valFloat float64 + } + + // CounterVec is a Collector that bundles a set of Counters which have different values for their names. + // This is used if you want to count the same thing partitioned by various dimensions + // (e.g. number of HTTP requests, partitioned by response code and method). + // + // Create instances with NewCounterVec. + CounterVec map[string]*Counter +) + +var ( + _ stm.Value = Counter{} + _ stm.Value = CounterVec{} +) + +// WriteTo writes its value into given map. +func (c Counter) WriteTo(rv map[string]int64, key string, mul, div int) { + rv[key] = int64(c.Value() * float64(mul) / float64(div)) +} + +// Value gets current counter. +func (c Counter) Value() float64 { + return float64(c.valInt) + c.valFloat +} + +// Inc increments the counter by 1. Use Add to increment it by arbitrary +// non-negative values. +func (c *Counter) Inc() { + c.valInt++ +} + +// Add adds the given bits to the counter. It panics if the value is < 0. +func (c *Counter) Add(v float64) { + if v < 0 { + panic(errors.New("counter cannot decrease in value")) + } + val := int64(v) + if float64(val) == v { + c.valInt += val + return + } + c.valFloat += v +} + +// NewCounterVec creates a new CounterVec +func NewCounterVec() CounterVec { + return CounterVec{} +} + +// WriteTo writes its value into given map. +func (c CounterVec) WriteTo(rv map[string]int64, key string, mul, div int) { + for name, value := range c { + rv[key+"_"+name] = int64(value.Value() * float64(mul) / float64(div)) + } +} + +// Get gets counter instance by name +func (c CounterVec) Get(name string) *Counter { + item, _ := c.GetP(name) + return item +} + +// GetP gets counter instance by name +func (c CounterVec) GetP(name string) (counter *Counter, ok bool) { + counter, ok = c[name] + if ok { + return + } + counter = &Counter{} + c[name] = counter + return +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/counter_test.go b/src/go/collectors/go.d.plugin/pkg/metrics/counter_test.go new file mode 100644 index 000000000..61f50501a --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/counter_test.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCounter_WriteTo(t *testing.T) { + c := Counter{} + c.Inc() + c.Inc() + c.Inc() + c.Add(0.14) + m := map[string]int64{} + c.WriteTo(m, "pi", 100, 1) + assert.Len(t, m, 1) + assert.EqualValues(t, 314, m["pi"]) +} + +func TestCounterVec_WriteTo(t *testing.T) { + c := NewCounterVec() + c.Get("foo").Inc() + c.Get("foo").Inc() + c.Get("bar").Inc() + c.Get("bar").Add(0.14) + + m := map[string]int64{} + c.WriteTo(m, "pi", 100, 1) + assert.Len(t, m, 2) + assert.EqualValues(t, 200, m["pi_foo"]) + assert.EqualValues(t, 114, m["pi_bar"]) +} + +func TestCounter_Inc(t *testing.T) { + c := Counter{} + c.Inc() + assert.Equal(t, 1.0, c.Value()) + c.Inc() + assert.Equal(t, 2.0, c.Value()) +} + +func TestCounter_Add(t *testing.T) { + c := Counter{} + c.Add(3.14) + assert.InDelta(t, 3.14, c.Value(), 0.0001) + c.Add(2) + assert.InDelta(t, 5.14, c.Value(), 0.0001) + assert.Panics(t, func() { + c.Add(-1) + }) +} + +func BenchmarkCounter_Add(b *testing.B) { + benchmarks := []struct { + name string + value float64 + }{ + {"int", 1}, + {"float", 3.14}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + var c Counter + for i := 0; i < b.N; i++ { + c.Add(bm.value) + } + }) + } +} + +func BenchmarkCounter_Inc(b *testing.B) { + var c Counter + for i := 0; i < b.N; i++ { + c.Inc() + } +} + +func BenchmarkCounterVec_Inc(b *testing.B) { + c := NewCounterVec() + for i := 0; i < b.N; i++ { + c.Get("foo").Inc() + } +} + +func BenchmarkCounter_Value(b *testing.B) { + var c Counter + c.Inc() + c.Add(3.14) + for i := 0; i < b.N; i++ { + c.Value() + } +} + +func BenchmarkCounter_WriteTo(b *testing.B) { + var c Counter + c.Inc() + c.Add(3.14) + m := map[string]int64{} + for i := 0; i < b.N; i++ { + c.WriteTo(m, "pi", 100, 1) + } +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/gauge.go b/src/go/collectors/go.d.plugin/pkg/metrics/gauge.go new file mode 100644 index 000000000..6f0930f66 --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/gauge.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "time" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" +) + +type ( + // Gauge is a Metric that represents a single numerical value that can + // arbitrarily go up and down. + // + // A Gauge is typically used for measured values like temperatures or current + // memory usage, but also "counts" that can go up and down, like the number of + // running goroutines. + Gauge float64 + + // GaugeVec is a Collector that bundles a set of Gauges which have different values for their names. + // This is used if you want to count the same thing partitioned by various dimensions + // + // Create instances with NewGaugeVec. + GaugeVec map[string]*Gauge +) + +var ( + _ stm.Value = Gauge(0) + _ stm.Value = GaugeVec{} +) + +// WriteTo writes its value into given map. +func (g Gauge) WriteTo(rv map[string]int64, key string, mul, div int) { + rv[key] = int64(float64(g) * float64(mul) / float64(div)) +} + +// Value gets current counter. +func (g Gauge) Value() float64 { + return float64(g) +} + +// Set sets the atomicGauge to an arbitrary bits. +func (g *Gauge) Set(v float64) { + *g = Gauge(v) +} + +// Inc increments the atomicGauge by 1. Use Add to increment it by arbitrary +// values. +func (g *Gauge) Inc() { + *g++ +} + +// Dec decrements the atomicGauge by 1. Use Sub to decrement it by arbitrary +// values. +func (g *Gauge) Dec() { + *g-- +} + +// Add adds the given bits to the atomicGauge. (The bits can be negative, +// resulting in a decrease of the atomicGauge.) +func (g *Gauge) Add(delta float64) { + *g += Gauge(delta) +} + +// Sub subtracts the given bits from the atomicGauge. (The bits can be +// negative, resulting in an increase of the atomicGauge.) +func (g *Gauge) Sub(delta float64) { + *g -= Gauge(delta) +} + +// SetToCurrentTime sets the atomicGauge to the current Unix time in second. +func (g *Gauge) SetToCurrentTime() { + *g = Gauge(time.Now().UnixNano()) / 1e9 +} + +// NewGaugeVec creates a new GaugeVec +func NewGaugeVec() GaugeVec { + return GaugeVec{} +} + +// WriteTo writes its value into given map. +func (g GaugeVec) WriteTo(rv map[string]int64, key string, mul, div int) { + for name, value := range g { + rv[key+"_"+name] = int64(value.Value() * float64(mul) / float64(div)) + } +} + +// Get gets counter instance by name +func (g GaugeVec) Get(name string) *Gauge { + item, _ := g.GetP(name) + return item +} + +// GetP gets counter instance by name +func (g GaugeVec) GetP(name string) (gauge *Gauge, ok bool) { + gauge, ok = g[name] + if ok { + return + } + gauge = new(Gauge) + g[name] = gauge + return +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/gauge_test.go b/src/go/collectors/go.d.plugin/pkg/metrics/gauge_test.go new file mode 100644 index 000000000..8940e330e --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/gauge_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGauge_Set(t *testing.T) { + var g Gauge + assert.Equal(t, 0.0, g.Value()) + g.Set(100) + assert.Equal(t, 100.0, g.Value()) + g.Set(200) + assert.Equal(t, 200.0, g.Value()) +} + +func TestGauge_Add(t *testing.T) { + var g Gauge + assert.Equal(t, 0.0, g.Value()) + g.Add(100) + assert.Equal(t, 100.0, g.Value()) + g.Add(200) + assert.Equal(t, 300.0, g.Value()) +} +func TestGauge_Sub(t *testing.T) { + var g Gauge + assert.Equal(t, 0.0, g.Value()) + g.Sub(100) + assert.Equal(t, -100.0, g.Value()) + g.Sub(200) + assert.Equal(t, -300.0, g.Value()) +} + +func TestGauge_Inc(t *testing.T) { + var g Gauge + assert.Equal(t, 0.0, g.Value()) + g.Inc() + assert.Equal(t, 1.0, g.Value()) +} + +func TestGauge_Dec(t *testing.T) { + var g Gauge + assert.Equal(t, 0.0, g.Value()) + g.Dec() + assert.Equal(t, -1.0, g.Value()) +} + +func TestGauge_SetToCurrentTime(t *testing.T) { + var g Gauge + g.SetToCurrentTime() + assert.InDelta(t, time.Now().Unix(), g.Value(), 1) +} + +func TestGauge_WriteTo(t *testing.T) { + g := Gauge(3.14) + m := map[string]int64{} + g.WriteTo(m, "pi", 100, 1) + assert.Len(t, m, 1) + assert.EqualValues(t, 314, m["pi"]) +} + +func TestGaugeVec_WriteTo(t *testing.T) { + g := NewGaugeVec() + g.Get("foo").Inc() + g.Get("foo").Inc() + g.Get("bar").Inc() + g.Get("bar").Add(0.14) + + m := map[string]int64{} + g.WriteTo(m, "pi", 100, 1) + assert.Len(t, m, 2) + assert.EqualValues(t, 200, m["pi_foo"]) + assert.EqualValues(t, 114, m["pi_bar"]) +} + +func BenchmarkGauge_Add(b *testing.B) { + benchmarks := []struct { + name string + value float64 + }{ + {"int", 1}, + {"float", 3.14}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + var c Gauge + for i := 0; i < b.N; i++ { + c.Add(bm.value) + } + }) + } +} + +func BenchmarkGauge_Inc(b *testing.B) { + var c Gauge + for i := 0; i < b.N; i++ { + c.Inc() + } +} + +func BenchmarkGauge_Set(b *testing.B) { + var c Gauge + for i := 0; i < b.N; i++ { + c.Set(3.14) + } +} + +func BenchmarkGauge_Value(b *testing.B) { + var c Gauge + c.Inc() + c.Add(3.14) + for i := 0; i < b.N; i++ { + c.Value() + } +} + +func BenchmarkGauge_WriteTo(b *testing.B) { + var c Gauge + c.Inc() + c.Add(3.14) + m := map[string]int64{} + for i := 0; i < b.N; i++ { + c.WriteTo(m, "pi", 100, 1) + } +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/histogram.go b/src/go/collectors/go.d.plugin/pkg/metrics/histogram.go new file mode 100644 index 000000000..caabf09af --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/histogram.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "fmt" + "sort" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" +) + +type ( + // A Histogram counts individual observations from an event or sample stream in + // configurable buckets. Similar to a summary, it also provides a sum of + // observations and an observation count. + // + // Note that Histograms, in contrast to Summaries, can be aggregated. + // However, Histograms require the user to pre-define suitable + // buckets, and they are in general less accurate. The Observe method of a + // histogram has a very low performance overhead in comparison with the Observe + // method of a summary. + // + // To create histogram instances, use NewHistogram. + Histogram interface { + Observer + } + + histogram struct { + buckets []int64 + upperBounds []float64 + sum float64 + count int64 + rangeBuckets bool + } +) + +var ( + _ stm.Value = histogram{} +) + +// DefBuckets are the default histogram buckets. The default buckets are +// tailored to broadly measure the response time (in seconds) of a network +// service. Most likely, however, you will be required to define buckets +// customized to your use case. +var DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} + +// LinearBuckets creates 'count' buckets, each 'width' wide, where the lowest +// bucket has an upper bound of 'start'. The final +Inf bucket is not counted +// and not included in the returned slice. The returned slice is meant to be +// used for the Buckets field of HistogramOpts. +// +// The function panics if 'count' is zero or negative. +func LinearBuckets(start, width float64, count int) []float64 { + if count < 1 { + panic("LinearBuckets needs a positive count") + } + buckets := make([]float64, count) + for i := range buckets { + buckets[i] = start + start += width + } + return buckets +} + +// ExponentialBuckets creates 'count' buckets, where the lowest bucket has an +// upper bound of 'start' and each following bucket's upper bound is 'factor' +// times the previous bucket's upper bound. The final +Inf bucket is not counted +// and not included in the returned slice. The returned slice is meant to be +// used for the Buckets field of HistogramOpts. +// +// The function panics if 'count' is 0 or negative, if 'start' is 0 or negative, +// or if 'factor' is less than or equal 1. +func ExponentialBuckets(start, factor float64, count int) []float64 { + if count < 1 { + panic("ExponentialBuckets needs a positive count") + } + if start <= 0 { + panic("ExponentialBuckets needs a positive start value") + } + if factor <= 1 { + panic("ExponentialBuckets needs a factor greater than 1") + } + buckets := make([]float64, count) + for i := range buckets { + buckets[i] = start + start *= factor + } + return buckets +} + +// NewHistogram creates a new Histogram. +func NewHistogram(buckets []float64) Histogram { + if len(buckets) == 0 { + buckets = DefBuckets + } else { + sort.Slice(buckets, func(i, j int) bool { return buckets[i] < buckets[j] }) + } + + return &histogram{ + buckets: make([]int64, len(buckets)), + upperBounds: buckets, + count: 0, + sum: 0, + } +} + +func NewHistogramWithRangeBuckets(buckets []float64) Histogram { + if len(buckets) == 0 { + buckets = DefBuckets + } else { + sort.Slice(buckets, func(i, j int) bool { return buckets[i] < buckets[j] }) + } + + return &histogram{ + buckets: make([]int64, len(buckets)), + upperBounds: buckets, + count: 0, + sum: 0, + rangeBuckets: true, + } +} + +// WriteTo writes its values into given map. +// It adds those key-value pairs: +// +// ${key}_sum gauge, for sum of it's observed values +// ${key}_count counter, for count of it's observed values (equals to +Inf bucket) +// ${key}_bucket_1 counter, for 1st bucket count +// ${key}_bucket_2 counter, for 2nd bucket count +// ... +// ${key}_bucket_N counter, for Nth bucket count +func (h histogram) WriteTo(rv map[string]int64, key string, mul, div int) { + rv[key+"_sum"] = int64(h.sum * float64(mul) / float64(div)) + rv[key+"_count"] = h.count + var conn int64 + for i, bucket := range h.buckets { + name := fmt.Sprintf("%s_bucket_%d", key, i+1) + conn += bucket + if h.rangeBuckets { + rv[name] = bucket + } else { + rv[name] = conn + } + } + if h.rangeBuckets { + name := fmt.Sprintf("%s_bucket_inf", key) + rv[name] = h.count - conn + } +} + +// Observe observes a value +func (h *histogram) Observe(v float64) { + hotIdx := h.searchBucketIndex(v) + if hotIdx < len(h.buckets) { + h.buckets[hotIdx]++ + } + h.sum += v + h.count++ +} + +func (h *histogram) searchBucketIndex(v float64) int { + if len(h.upperBounds) < 30 { + for i, upper := range h.upperBounds { + if upper >= v { + return i + } + } + return len(h.upperBounds) + } + return sort.SearchFloat64s(h.upperBounds, v) +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/histogram_test.go b/src/go/collectors/go.d.plugin/pkg/metrics/histogram_test.go new file mode 100644 index 000000000..91266915c --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/histogram_test.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLinearBuckets(t *testing.T) { + buckets := LinearBuckets(0, 1, 10) + assert.Len(t, buckets, 10) + assert.EqualValues(t, 0, buckets[0]) + assert.EqualValues(t, 5.0, buckets[5]) + assert.EqualValues(t, 9.0, buckets[9]) + + assert.Panics(t, func() { + LinearBuckets(0, 1, 0) + }) +} + +func TestExponentialBuckets(t *testing.T) { + buckets := ExponentialBuckets(1, 2, 10) + assert.Len(t, buckets, 10) + assert.EqualValues(t, 1, buckets[0]) + assert.EqualValues(t, 32.0, buckets[5]) + assert.EqualValues(t, 512.0, buckets[9]) + + assert.Panics(t, func() { + ExponentialBuckets(1, 2, 0) + }) + assert.Panics(t, func() { + ExponentialBuckets(0, 2, 2) + }) + + assert.Panics(t, func() { + ExponentialBuckets(1, 1, 2) + }) +} + +func TestNewHistogram(t *testing.T) { + h := NewHistogram(nil).(*histogram) + assert.EqualValues(t, 0, h.count) + assert.EqualValues(t, 0.0, h.sum) + assert.Equal(t, DefBuckets, h.upperBounds) + + h = NewHistogram([]float64{1, 10, 5}).(*histogram) + assert.Equal(t, []float64{1, 5, 10}, h.upperBounds) + assert.Len(t, h.buckets, 3) +} + +func TestHistogram_WriteTo(t *testing.T) { + h := NewHistogram([]float64{1, 2, 3}) + m := map[string]int64{} + h.WriteTo(m, "pi", 100, 1) + assert.Len(t, m, 5) + assert.EqualValues(t, 0, m["pi_count"]) + assert.EqualValues(t, 0, m["pi_sum"]) + assert.EqualValues(t, 0, m["pi_bucket_1"]) + assert.EqualValues(t, 0, m["pi_bucket_2"]) + assert.EqualValues(t, 0, m["pi_bucket_3"]) + + h.Observe(0) + h.Observe(1.5) + h.Observe(3.5) + h.WriteTo(m, "pi", 100, 1) + assert.Len(t, m, 5) + assert.EqualValues(t, 3, m["pi_count"]) + assert.EqualValues(t, 500, m["pi_sum"]) + assert.EqualValues(t, 1, m["pi_bucket_1"]) + assert.EqualValues(t, 2, m["pi_bucket_2"]) + assert.EqualValues(t, 2, m["pi_bucket_3"]) +} + +func TestHistogram_searchBucketIndex(t *testing.T) { + h := NewHistogram(LinearBuckets(1, 1, 5)).(*histogram) // [1, 2, ..., 5] + assert.Equal(t, 0, h.searchBucketIndex(0.1)) + assert.Equal(t, 1, h.searchBucketIndex(1.1)) + assert.Equal(t, 5, h.searchBucketIndex(8.1)) + + h = NewHistogram(LinearBuckets(1, 1, 40)).(*histogram) // [1, 2, ..., 5] + assert.Equal(t, 0, h.searchBucketIndex(0.1)) + assert.Equal(t, 1, h.searchBucketIndex(1.1)) + assert.Equal(t, 5, h.searchBucketIndex(5.1)) + assert.Equal(t, 7, h.searchBucketIndex(8)) + assert.Equal(t, 39, h.searchBucketIndex(39.5)) + assert.Equal(t, 40, h.searchBucketIndex(40.5)) +} + +func BenchmarkHistogram_Observe(b *testing.B) { + benchmarks := []struct { + name string + buckets []float64 + }{ + {"default", nil}, + {"len_10", LinearBuckets(0, 0.1, 10)}, + {"len_20", LinearBuckets(0, 0.1, 20)}, + {"len_30", LinearBuckets(0, 0.1, 30)}, + {"len_40", LinearBuckets(0, 0.1, 40)}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + h := NewHistogram(bm.buckets) + for i := 0; i < b.N; i++ { + h.Observe(2.5) + } + }) + } +} + +func BenchmarkHistogram_WriteTo(b *testing.B) { + benchmarks := []struct { + name string + buckets []float64 + }{ + {"default", nil}, + {"len_10", LinearBuckets(0, 0.1, 10)}, + {"len_20", LinearBuckets(0, 0.1, 20)}, + {"len_30", LinearBuckets(0, 0.1, 30)}, + {"len_40", LinearBuckets(0, 0.1, 40)}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + h := NewHistogram(bm.buckets) + h.Observe(0.1) + h.Observe(0.01) + h.Observe(0.5) + h.Observe(10) + m := map[string]int64{} + for i := 0; i < b.N; i++ { + h.WriteTo(m, "pi", 100, 1) + } + }) + } +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/metrics.go b/src/go/collectors/go.d.plugin/pkg/metrics/metrics.go new file mode 100644 index 000000000..44a24056f --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/metrics.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" + +// Observer is an interface that wraps the Observe method, which is used by +// Histogram and Summary to add observations. +type Observer interface { + stm.Value + Observe(v float64) +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/summary.go b/src/go/collectors/go.d.plugin/pkg/metrics/summary.go new file mode 100644 index 000000000..01b85f65e --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/summary.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "math" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" +) + +type ( + // A Summary captures individual observations from an event or sample stream and + // summarizes them in a manner similar to traditional summary statistics: + // sum of observations + // observation count + // observation average. + // + // To create summary instances, use NewSummary. + Summary interface { + Observer + Reset() + } + + // SummaryVec is a Collector that bundles a set of Summary which have different values for their names. + // This is used if you want to count the same thing partitioned by various dimensions + // (e.g. number of HTTP response time, partitioned by response code and method). + // + // Create instances with NewSummaryVec. + SummaryVec map[string]Summary + + summary struct { + min float64 + max float64 + sum float64 + count int64 + } +) + +var ( + _ stm.Value = summary{} + _ stm.Value = SummaryVec{} +) + +// NewSummary creates a new Summary. +func NewSummary() Summary { + return &summary{ + min: math.MaxFloat64, + max: -math.MaxFloat64, + } +} + +// WriteTo writes its values into given map. +// It adds those key-value pairs: +// +// ${key}_sum gauge, for sum of it's observed values from last Reset calls +// ${key}_count counter, for count of it's observed values from last Reset calls +// ${key}_min gauge, for min of it's observed values from last Reset calls (only exists if count > 0) +// ${key}_max gauge, for max of it's observed values from last Reset calls (only exists if count > 0) +// ${key}_avg gauge, for avg of it's observed values from last Reset calls (only exists if count > 0) +func (s summary) WriteTo(rv map[string]int64, key string, mul, div int) { + if s.count > 0 { + rv[key+"_min"] = int64(s.min * float64(mul) / float64(div)) + rv[key+"_max"] = int64(s.max * float64(mul) / float64(div)) + rv[key+"_sum"] = int64(s.sum * float64(mul) / float64(div)) + rv[key+"_count"] = s.count + rv[key+"_avg"] = int64(s.sum / float64(s.count) * float64(mul) / float64(div)) + } else { + rv[key+"_count"] = 0 + rv[key+"_sum"] = 0 + delete(rv, key+"_min") + delete(rv, key+"_max") + delete(rv, key+"_avg") + } +} + +// Reset resets all of its counters. +// Call it before every scrape loop. +func (s *summary) Reset() { + s.min = math.MaxFloat64 + s.max = -math.MaxFloat64 + s.sum = 0 + s.count = 0 +} + +// Observe observes a value +func (s *summary) Observe(v float64) { + if v > s.max { + s.max = v + } + if v < s.min { + s.min = v + } + s.sum += v + s.count++ +} + +// NewSummaryVec creates a new SummaryVec instance. +func NewSummaryVec() SummaryVec { + return SummaryVec{} +} + +// WriteTo writes its value into given map. +func (c SummaryVec) WriteTo(rv map[string]int64, key string, mul, div int) { + for name, value := range c { + value.WriteTo(rv, key+"_"+name, mul, div) + } +} + +// Get gets counter instance by name. +func (c SummaryVec) Get(name string) Summary { + item, ok := c[name] + if ok { + return item + } + item = NewSummary() + c[name] = item + return item +} + +// Reset resets its all summaries. +func (c SummaryVec) Reset() { + for _, value := range c { + value.Reset() + } +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/summary_test.go b/src/go/collectors/go.d.plugin/pkg/metrics/summary_test.go new file mode 100644 index 000000000..b98218369 --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/summary_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewSummary(t *testing.T) { + s := NewSummary().(*summary) + assert.EqualValues(t, 0, s.count) + assert.Equal(t, 0.0, s.sum) + s.Observe(3.14) + assert.Equal(t, 3.14, s.min) + assert.Equal(t, 3.14, s.max) +} + +func TestSummary_WriteTo(t *testing.T) { + s := NewSummary() + + m1 := map[string]int64{} + s.WriteTo(m1, "pi", 100, 1) + assert.Len(t, m1, 2) + assert.Contains(t, m1, "pi_count") + assert.Contains(t, m1, "pi_sum") + assert.EqualValues(t, 0, m1["pi_count"]) + assert.EqualValues(t, 0, m1["pi_sum"]) + + s.Observe(3.14) + s.Observe(2.71) + s.Observe(-10) + + m2 := map[string]int64{} + s.WriteTo(m1, "pi", 100, 1) + s.WriteTo(m2, "pi", 100, 1) + assert.Equal(t, m1, m2) + assert.Len(t, m1, 5) + assert.EqualValues(t, 3, m1["pi_count"]) + assert.EqualValues(t, -415, m1["pi_sum"]) + assert.EqualValues(t, -1000, m1["pi_min"]) + assert.EqualValues(t, 314, m1["pi_max"]) + assert.EqualValues(t, -138, m1["pi_avg"]) + + s.Reset() + s.WriteTo(m1, "pi", 100, 1) + assert.Len(t, m1, 2) + assert.Contains(t, m1, "pi_count") + assert.Contains(t, m1, "pi_sum") + assert.EqualValues(t, 0, m1["pi_count"]) + assert.EqualValues(t, 0, m1["pi_sum"]) +} + +func TestSummary_Reset(t *testing.T) { + s := NewSummary().(*summary) + s.Observe(1) + s.Reset() + assert.EqualValues(t, 0, s.count) +} + +func BenchmarkSummary_Observe(b *testing.B) { + s := NewSummary() + for i := 0; i < b.N; i++ { + s.Observe(2.5) + } +} + +func BenchmarkSummary_WriteTo(b *testing.B) { + s := NewSummary() + s.Observe(2.5) + s.Observe(3.5) + s.Observe(4.5) + m := map[string]int64{} + for i := 0; i < b.N; i++ { + s.WriteTo(m, "pi", 100, 1) + } +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/unique_counter.go b/src/go/collectors/go.d.plugin/pkg/metrics/unique_counter.go new file mode 100644 index 000000000..dfc96126a --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/unique_counter.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "github.com/axiomhq/hyperloglog" + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" +) + +type ( + UniqueCounter interface { + stm.Value + Insert(s string) + Value() int + Reset() + } + + mapUniqueCounter struct { + m map[string]bool + } + + hyperLogLogUniqueCounter struct { + sketch *hyperloglog.Sketch + } + + UniqueCounterVec struct { + useHyperLogLog bool + Items map[string]UniqueCounter + } +) + +var ( + _ stm.Value = mapUniqueCounter{} + _ stm.Value = hyperLogLogUniqueCounter{} + _ stm.Value = UniqueCounterVec{} +) + +func NewUniqueCounter(useHyperLogLog bool) UniqueCounter { + if useHyperLogLog { + return &hyperLogLogUniqueCounter{hyperloglog.New()} + } + return mapUniqueCounter{map[string]bool{}} +} + +func (c mapUniqueCounter) WriteTo(rv map[string]int64, key string, mul, div int) { + rv[key] = int64(float64(c.Value()*mul) / float64(div)) +} + +func (c mapUniqueCounter) Insert(s string) { + c.m[s] = true +} + +func (c mapUniqueCounter) Value() int { + return len(c.m) +} + +func (c mapUniqueCounter) Reset() { + for key := range c.m { + delete(c.m, key) + } +} + +// WriteTo writes its value into given map. +func (c hyperLogLogUniqueCounter) WriteTo(rv map[string]int64, key string, mul, div int) { + rv[key] = int64(float64(c.Value()*mul) / float64(div)) +} + +func (c *hyperLogLogUniqueCounter) Insert(s string) { + c.sketch.Insert([]byte(s)) +} + +func (c *hyperLogLogUniqueCounter) Value() int { + return int(c.sketch.Estimate()) +} + +func (c *hyperLogLogUniqueCounter) Reset() { + c.sketch = hyperloglog.New() +} + +func NewUniqueCounterVec(useHyperLogLog bool) UniqueCounterVec { + return UniqueCounterVec{ + Items: map[string]UniqueCounter{}, + useHyperLogLog: useHyperLogLog, + } +} + +// WriteTo writes its value into given map. +func (c UniqueCounterVec) WriteTo(rv map[string]int64, key string, mul, div int) { + for name, value := range c.Items { + value.WriteTo(rv, key+"_"+name, mul, div) + } +} + +// Get gets UniqueCounter instance by name +func (c UniqueCounterVec) Get(name string) UniqueCounter { + item, ok := c.Items[name] + if ok { + return item + } + item = NewUniqueCounter(c.useHyperLogLog) + c.Items[name] = item + return item +} + +func (c UniqueCounterVec) Reset() { + for _, value := range c.Items { + value.Reset() + } +} diff --git a/src/go/collectors/go.d.plugin/pkg/metrics/unique_counter_test.go b/src/go/collectors/go.d.plugin/pkg/metrics/unique_counter_test.go new file mode 100644 index 000000000..b9439c9a3 --- /dev/null +++ b/src/go/collectors/go.d.plugin/pkg/metrics/unique_counter_test.go @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package metrics + +import ( + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHyperLogLogUniqueCounter_Value(t *testing.T) { + for _, useHLL := range []bool{true, false} { + t.Run(fmt.Sprintf("HLL=%v", useHLL), func(t *testing.T) { + c := NewUniqueCounter(useHLL) + assert.Equal(t, 0, c.Value()) + + c.Insert("foo") + assert.Equal(t, 1, c.Value()) + + c.Insert("foo") + assert.Equal(t, 1, c.Value()) + + c.Insert("bar") + assert.Equal(t, 2, c.Value()) + + c.Insert("baz") + assert.Equal(t, 3, c.Value()) + + c.Reset() + assert.Equal(t, 0, c.Value()) + + c.Insert("foo") + assert.Equal(t, 1, c.Value()) + }) + } +} + +func TestHyperLogLogUniqueCounter_WriteTo(t *testing.T) { + for _, useHLL := range []bool{true, false} { + t.Run(fmt.Sprintf("HLL=%v", useHLL), func(t *testing.T) { + c := NewUniqueCounterVec(useHLL) + c.Get("a").Insert("foo") + c.Get("a").Insert("bar") + c.Get("b").Insert("foo") + + m := map[string]int64{} + c.WriteTo(m, "pi", 100, 1) + assert.Len(t, m, 2) + assert.EqualValues(t, 200, m["pi_a"]) + assert.EqualValues(t, 100, m["pi_b"]) + }) + } +} + +func TestUniqueCounterVec_Reset(t *testing.T) { + for _, useHLL := range []bool{true, false} { + t.Run(fmt.Sprintf("HLL=%v", useHLL), func(t *testing.T) { + c := NewUniqueCounterVec(useHLL) + c.Get("a").Insert("foo") + c.Get("a").Insert("bar") + c.Get("b").Insert("foo") + + assert.Equal(t, 2, len(c.Items)) + assert.Equal(t, 2, c.Get("a").Value()) + assert.Equal(t, 1, c.Get("b").Value()) + + c.Reset() + assert.Equal(t, 2, len(c.Items)) + assert.Equal(t, 0, c.Get("a").Value()) + assert.Equal(t, 0, c.Get("b").Value()) + }) + } +} + +func BenchmarkUniqueCounter_Insert(b *testing.B) { + benchmarks := []struct { + name string + same bool + hyperloglog bool + nop bool + }{ + + {"map-same", true, false, false}, + {"hll-same", true, true, false}, + + {"nop", false, false, true}, + {"map-diff", false, false, false}, + {"hll-diff", false, true, false}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + c := NewUniqueCounter(bm.hyperloglog) + if bm.same { + for i := 0; i < b.N; i++ { + c.Insert("foo") + } + } else if bm.nop { + for i := 0; i < b.N; i++ { + strconv.Itoa(i) + } + } else { + for i := 0; i < b.N; i++ { + c.Insert(strconv.Itoa(i)) + } + } + }) + } +} + +func BenchmarkUniqueCounterVec_Insert(b *testing.B) { + benchmarks := []struct { + name string + same bool + hyperloglog bool + nop bool + }{ + + {"map-same", true, false, false}, + {"hll-same", true, true, false}, + + {"nop", false, false, true}, + {"map-diff", false, false, false}, + {"hll-diff", false, true, false}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + c := NewUniqueCounterVec(bm.hyperloglog) + if bm.same { + for i := 0; i < b.N; i++ { + c.Get("a").Insert("foo") + } + } else if bm.nop { + for i := 0; i < b.N; i++ { + strconv.Itoa(i) + } + } else { + for i := 0; i < b.N; i++ { + c.Get("a").Insert(strconv.Itoa(i)) + } + } + }) + } +} |