diff options
Diffstat (limited to 'src/go/collectors/go.d.plugin/modules/example')
9 files changed, 916 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/modules/example/README.md b/src/go/collectors/go.d.plugin/modules/example/README.md new file mode 100644 index 000000000..70441aaa6 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/README.md @@ -0,0 +1,80 @@ +<!-- +title: "Example module" +description: "Use this example data collection module, which produces example charts with random values, to better understand how to build your own collector in Go." +custom_edit_url: "https://github.com/netdata/go.d.plugin/edit/master/modules/example/README.md" +sidebar_label: "Example module in Go" +learn_status: "Published" +learn_topic_type: "References" +learn_rel_path: "Integrations/Monitor/Mock Collectors" +--> + +# Example module + +An example data collection module. Use it as an example writing a new module. + +## Charts + +This module produces example charts with random values. Number of charts, dimensions and chart type is configurable. + +## Configuration + +Edit the `go.d/example.conf` configuration file using `edit-config` from the +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md), which is typically at `/etc/netdata`. + +```bash +cd /etc/netdata # Replace this path with your Netdata config directory +sudo ./edit-config go.d/example.conf +``` + +Disabled by default. Should be explicitly enabled +in [go.d.conf](https://github.com/netdata/netdata/blob/master/src/go/collectors/go.d.plugin/config/go.d.conf). + +```yaml +# go.d.conf +modules: + example: yes +``` + +Here is an example configuration with several jobs: + +```yaml +jobs: + - name: example + charts: + num: 3 + dimensions: 5 + + - name: hidden_example + hidden_charts: + num: 3 + dimensions: 5 +``` + +--- + +For all available options, see the Example +collector's [configuration file](https://github.com/netdata/netdata/blob/master/src/go/collectors/go.d.plugin/config/go.d/example.conf). + +## Troubleshooting + +To troubleshoot issues with the `example` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m example + ``` diff --git a/src/go/collectors/go.d.plugin/modules/example/charts.go b/src/go/collectors/go.d.plugin/modules/example/charts.go new file mode 100644 index 000000000..d3973a99d --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/charts.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package example + +import ( + "fmt" + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +var chartTemplate = module.Chart{ + ID: "random_%d", + Title: "A Random Number", + Units: "random", + Fam: "random", + Ctx: "example.random", +} + +var hiddenChartTemplate = module.Chart{ + ID: "hidden_random_%d", + Title: "A Random Number", + Units: "random", + Fam: "random", + Ctx: "example.random", + Opts: module.Opts{ + Hidden: true, + }, +} + +func newChart(num, ctx, labels int, typ module.ChartType) *module.Chart { + chart := chartTemplate.Copy() + chart.ID = fmt.Sprintf(chart.ID, num) + chart.Type = typ + if ctx > 0 { + chart.Ctx += fmt.Sprintf("_%d", ctx) + } + for i := 0; i < labels; i++ { + chart.Labels = append(chart.Labels, module.Label{ + Key: fmt.Sprintf("example_name_%d", i), + Value: fmt.Sprintf("example_value_%d_%d", num, i), + }) + } + return chart +} + +func newHiddenChart(num, ctx, labels int, typ module.ChartType) *module.Chart { + chart := hiddenChartTemplate.Copy() + chart.ID = fmt.Sprintf(chart.ID, num) + chart.Type = typ + if ctx > 0 { + chart.Ctx += fmt.Sprintf("_%d", ctx) + } + for i := 0; i < labels; i++ { + chart.Labels = append(chart.Labels, module.Label{ + Key: fmt.Sprintf("example_name_%d", i), + Value: fmt.Sprintf("example_value_%d_%d", num, i), + }) + } + return chart +} diff --git a/src/go/collectors/go.d.plugin/modules/example/collect.go b/src/go/collectors/go.d.plugin/modules/example/collect.go new file mode 100644 index 000000000..588d605df --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/collect.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package example + +import ( + "fmt" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +func (e *Example) collect() (map[string]int64, error) { + collected := make(map[string]int64) + + for _, chart := range *e.Charts() { + e.collectChart(collected, chart) + } + return collected, nil +} + +func (e *Example) collectChart(collected map[string]int64, chart *module.Chart) { + var num int + if chart.Opts.Hidden { + num = e.Config.HiddenCharts.Dims + } else { + num = e.Config.Charts.Dims + } + + for i := 0; i < num; i++ { + name := fmt.Sprintf("random%d", i) + id := fmt.Sprintf("%s_%s", chart.ID, name) + + if !e.collectedDims[id] { + e.collectedDims[id] = true + + dim := &module.Dim{ID: id, Name: name} + if err := chart.AddDim(dim); err != nil { + e.Warning(err) + } + chart.MarkNotCreated() + } + if i%2 == 0 { + collected[id] = e.randInt() + } else { + collected[id] = -e.randInt() + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/example/config_schema.json b/src/go/collectors/go.d.plugin/modules/example/config_schema.json new file mode 100644 index 000000000..328773f6d --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/config_schema.json @@ -0,0 +1,177 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Example collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "charts": { + "title": "Charts configuration", + "type": [ + "object", + "null" + ], + "properties": { + "type": { + "title": "Chart type", + "description": "The type of all charts.", + "type": "string", + "enum": [ + "line", + "area", + "stacked" + ], + "default": "line" + }, + "num": { + "title": "Number of charts", + "description": "The total number of charts to create.", + "type": "integer", + "minimum": 0, + "default": 1 + }, + "contexts": { + "title": "Number of contexts", + "description": "The total number of unique contexts.", + "type": "integer", + "minimum": 0, + "default": 0 + }, + "dimensions": { + "title": "Number of dimensions", + "description": "The number of dimensions each chart will have.", + "type": "integer", + "minimum": 1, + "default": 4 + }, + "labels": { + "title": "Number of labels", + "description": "The number of labels each chart will have.", + "type": "integer", + "minimum": 0, + "default": 0 + } + }, + "required": [ + "type", + "num", + "contexts", + "dimensions", + "labels" + ] + }, + "hidden_charts": { + "title": "Hidden charts configuration", + "type": [ + "object", + "null" + ], + "properties": { + "type": { + "title": "Chart type", + "description": "The type of all charts.", + "type": "string", + "enum": [ + "line", + "area", + "stacked" + ], + "default": "line" + }, + "num": { + "title": "Number of charts", + "description": "The total number of charts to create.", + "type": "integer", + "minimum": 0, + "default": 0 + }, + "contexts": { + "title": "Number of contexts", + "description": "The total number of unique contexts.", + "type": "integer", + "minimum": 0, + "default": 0 + }, + "dimensions": { + "title": "Number of dimensions", + "description": "The number of dimensions each chart will have.", + "type": "integer", + "minimum": 1, + "default": 4 + }, + "labels": { + "title": "Number of labels", + "description": "The number of labels each chart will have.", + "type": "integer", + "minimum": 0, + "default": 0 + } + }, + "required": [ + "type", + "num", + "contexts", + "dimensions", + "labels" + ] + } + }, + "required": [ + "charts" + ], + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "uiOptions": { + "fullPage": true + }, + "charts": { + "type": { + "ui:widget": "radio", + "ui:options": { + "inline": true + } + } + }, + "hidden_charts": { + "type": { + "ui:widget": "radio", + "ui:options": { + "inline": true + } + } + }, + "ui:flavour": "tabs", + "ui:options": { + "tabs": [ + { + "title": "Base", + "fields": [ + "update_every" + ] + }, + { + "title": "Charts", + "fields": [ + "charts" + ] + }, + { + "title": "Hidden charts", + "fields": [ + "hidden_charts" + ] + } + ] + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/example/example.go b/src/go/collectors/go.d.plugin/modules/example/example.go new file mode 100644 index 000000000..2b1da8a51 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/example.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package example + +import ( + _ "embed" + "math/rand" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("example", module.Creator{ + JobConfigSchema: configSchema, + Defaults: module.Defaults{ + UpdateEvery: module.UpdateEvery, + Priority: module.Priority, + Disabled: true, + }, + Create: func() module.Module { return New() }, + }) +} + +func New() *Example { + return &Example{ + Config: Config{ + Charts: ConfigCharts{ + Num: 1, + Dims: 4, + }, + HiddenCharts: ConfigCharts{ + Num: 0, + Dims: 4, + }, + }, + + randInt: func() int64 { return rand.Int63n(100) }, + collectedDims: make(map[string]bool), + } +} + +type ( + Config struct { + UpdateEvery int `yaml:"update_every" json:"update_every"` + Charts ConfigCharts `yaml:"charts" json:"charts"` + HiddenCharts ConfigCharts `yaml:"hidden_charts" json:"hidden_charts"` + } + ConfigCharts struct { + Type string `yaml:"type" json:"type"` + Num int `yaml:"num" json:"num"` + Contexts int `yaml:"contexts" json:"contexts"` + Dims int `yaml:"dimensions" json:"dimensions"` + Labels int `yaml:"labels" json:"labels"` + } +) + +type Example struct { + module.Base // should be embedded by every module + Config `yaml:",inline"` + + randInt func() int64 + charts *module.Charts + collectedDims map[string]bool +} + +func (e *Example) Configuration() any { + return e.Config +} + +func (e *Example) Init() error { + err := e.validateConfig() + if err != nil { + e.Errorf("config validation: %v", err) + return err + } + + charts, err := e.initCharts() + if err != nil { + e.Errorf("charts init: %v", err) + return err + } + e.charts = charts + return nil +} + +func (e *Example) Check() error { + return nil +} + +func (e *Example) Charts() *module.Charts { + return e.charts +} + +func (e *Example) Collect() map[string]int64 { + mx, err := e.collect() + if err != nil { + e.Error(err) + } + + if len(mx) == 0 { + return nil + } + return mx +} + +func (e *Example) Cleanup() {} diff --git a/src/go/collectors/go.d.plugin/modules/example/example_test.go b/src/go/collectors/go.d.plugin/modules/example/example_test.go new file mode 100644 index 000000000..6fde9b649 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/example_test.go @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package example + +import ( + "os" + "testing" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + } { + require.NotNil(t, data, name) + } +} + +func TestExample_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Example{}, dataConfigJSON, dataConfigYAML) +} + +func TestNew(t *testing.T) { + // We want to ensure that module is a reference type, nothing more. + + assert.IsType(t, (*Example)(nil), New()) +} + +func TestExample_Init(t *testing.T) { + // 'Init() bool' initializes the module with an appropriate config, so to test it we need: + // - provide the config. + // - set module.Config field with the config. + // - call Init() and compare its return value with the expected value. + + // 'test' map contains different test cases. + tests := map[string]struct { + config Config + wantFail bool + }{ + "success on default config": { + config: New().Config, + }, + "success when only 'charts' set": { + config: Config{ + Charts: ConfigCharts{ + Num: 1, + Dims: 2, + }, + }, + }, + "success when only 'hidden_charts' set": { + config: Config{ + HiddenCharts: ConfigCharts{ + Num: 1, + Dims: 2, + }, + }, + }, + "success when 'charts' and 'hidden_charts' set": { + config: Config{ + Charts: ConfigCharts{ + Num: 1, + Dims: 2, + }, + HiddenCharts: ConfigCharts{ + Num: 1, + Dims: 2, + }, + }, + }, + "fails when 'charts' and 'hidden_charts' set, but 'num' == 0": { + wantFail: true, + config: Config{ + Charts: ConfigCharts{ + Num: 0, + Dims: 2, + }, + HiddenCharts: ConfigCharts{ + Num: 0, + Dims: 2, + }, + }, + }, + "fails when only 'charts' set, 'num' > 0, but 'dimensions' == 0": { + wantFail: true, + config: Config{ + Charts: ConfigCharts{ + Num: 1, + Dims: 0, + }, + }, + }, + "fails when only 'hidden_charts' set, 'num' > 0, but 'dimensions' == 0": { + wantFail: true, + config: Config{ + HiddenCharts: ConfigCharts{ + Num: 1, + Dims: 0, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + example := New() + example.Config = test.config + + if test.wantFail { + assert.Error(t, example.Init()) + } else { + assert.NoError(t, example.Init()) + } + }) + } +} + +func TestExample_Check(t *testing.T) { + // 'Check() bool' reports whether the module is able to collect any data, so to test it we need: + // - provide the module with a specific config. + // - initialize the module (call Init()). + // - call Check() and compare its return value with the expected value. + + // 'test' map contains different test cases. + tests := map[string]struct { + prepare func() *Example + wantFail bool + }{ + "success on default": {prepare: prepareExampleDefault}, + "success when only 'charts' set": {prepare: prepareExampleOnlyCharts}, + "success when only 'hidden_charts' set": {prepare: prepareExampleOnlyHiddenCharts}, + "success when 'charts' and 'hidden_charts' set": {prepare: prepareExampleChartsAndHiddenCharts}, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + example := test.prepare() + require.NoError(t, example.Init()) + + if test.wantFail { + assert.Error(t, example.Check()) + } else { + assert.NoError(t, example.Check()) + } + }) + } +} + +func TestExample_Charts(t *testing.T) { + // We want to ensure that initialized module does not return 'nil'. + // If it is not 'nil' we are ok. + + // 'test' map contains different test cases. + tests := map[string]struct { + prepare func(t *testing.T) *Example + wantNil bool + }{ + "not initialized collector": { + wantNil: true, + prepare: func(t *testing.T) *Example { + return New() + }, + }, + "initialized collector": { + prepare: func(t *testing.T) *Example { + example := New() + require.NoError(t, example.Init()) + return example + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + example := test.prepare(t) + + if test.wantNil { + assert.Nil(t, example.Charts()) + } else { + assert.NotNil(t, example.Charts()) + } + }) + } +} + +func TestExample_Cleanup(t *testing.T) { + // Since this module has nothing to clean up, + // we want just to ensure that Cleanup() not panics. + + assert.NotPanics(t, New().Cleanup) +} + +func TestExample_Collect(t *testing.T) { + // 'Collect() map[string]int64' returns collected data, so to test it we need: + // - provide the module with a specific config. + // - initialize the module (call Init()). + // - call Collect() and compare its return value with the expected value. + + // 'test' map contains different test cases. + tests := map[string]struct { + prepare func() *Example + wantCollected map[string]int64 + }{ + "default config": { + prepare: prepareExampleDefault, + wantCollected: map[string]int64{ + "random_0_random0": 1, + "random_0_random1": -1, + "random_0_random2": 1, + "random_0_random3": -1, + }, + }, + "only 'charts' set": { + prepare: prepareExampleOnlyCharts, + wantCollected: map[string]int64{ + "random_0_random0": 1, + "random_0_random1": -1, + "random_0_random2": 1, + "random_0_random3": -1, + "random_0_random4": 1, + "random_1_random0": 1, + "random_1_random1": -1, + "random_1_random2": 1, + "random_1_random3": -1, + "random_1_random4": 1, + }, + }, + "only 'hidden_charts' set": { + prepare: prepareExampleOnlyHiddenCharts, + wantCollected: map[string]int64{ + "hidden_random_0_random0": 1, + "hidden_random_0_random1": -1, + "hidden_random_0_random2": 1, + "hidden_random_0_random3": -1, + "hidden_random_0_random4": 1, + "hidden_random_1_random0": 1, + "hidden_random_1_random1": -1, + "hidden_random_1_random2": 1, + "hidden_random_1_random3": -1, + "hidden_random_1_random4": 1, + }, + }, + "'charts' and 'hidden_charts' set": { + prepare: prepareExampleChartsAndHiddenCharts, + wantCollected: map[string]int64{ + "hidden_random_0_random0": 1, + "hidden_random_0_random1": -1, + "hidden_random_0_random2": 1, + "hidden_random_0_random3": -1, + "hidden_random_0_random4": 1, + "hidden_random_1_random0": 1, + "hidden_random_1_random1": -1, + "hidden_random_1_random2": 1, + "hidden_random_1_random3": -1, + "hidden_random_1_random4": 1, + "random_0_random0": 1, + "random_0_random1": -1, + "random_0_random2": 1, + "random_0_random3": -1, + "random_0_random4": 1, + "random_1_random0": 1, + "random_1_random1": -1, + "random_1_random2": 1, + "random_1_random3": -1, + "random_1_random4": 1, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + example := test.prepare() + require.NoError(t, example.Init()) + + collected := example.Collect() + + assert.Equal(t, test.wantCollected, collected) + ensureCollectedHasAllChartsDimsVarsIDs(t, example, collected) + }) + } +} + +func ensureCollectedHasAllChartsDimsVarsIDs(t *testing.T, e *Example, collected map[string]int64) { + for _, chart := range *e.Charts() { + if chart.Obsolete { + continue + } + for _, dim := range chart.Dims { + _, ok := collected[dim.ID] + assert.Truef(t, ok, + "collected metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID) + } + for _, v := range chart.Vars { + _, ok := collected[v.ID] + assert.Truef(t, ok, + "collected metrics has no data for var '%s' chart '%s'", v.ID, chart.ID) + } + } +} + +func prepareExampleDefault() *Example { + return prepareExample(New().Config) +} + +func prepareExampleOnlyCharts() *Example { + return prepareExample(Config{ + Charts: ConfigCharts{ + Num: 2, + Dims: 5, + }, + }) +} + +func prepareExampleOnlyHiddenCharts() *Example { + return prepareExample(Config{ + HiddenCharts: ConfigCharts{ + Num: 2, + Dims: 5, + }, + }) +} + +func prepareExampleChartsAndHiddenCharts() *Example { + return prepareExample(Config{ + Charts: ConfigCharts{ + Num: 2, + Dims: 5, + }, + HiddenCharts: ConfigCharts{ + Num: 2, + Dims: 5, + }, + }) +} + +func prepareExample(cfg Config) *Example { + example := New() + example.Config = cfg + example.randInt = func() int64 { return 1 } + return example +} diff --git a/src/go/collectors/go.d.plugin/modules/example/init.go b/src/go/collectors/go.d.plugin/modules/example/init.go new file mode 100644 index 000000000..6ee39ef4f --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/init.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package example + +import ( + "errors" + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +func (e *Example) validateConfig() error { + if e.Config.Charts.Num <= 0 && e.Config.HiddenCharts.Num <= 0 { + return errors.New("'charts->num' or `hidden_charts->num` must be > 0") + } + if e.Config.Charts.Num > 0 && e.Config.Charts.Dims <= 0 { + return errors.New("'charts->dimensions' must be > 0") + } + if e.Config.HiddenCharts.Num > 0 && e.Config.HiddenCharts.Dims <= 0 { + return errors.New("'hidden_charts->dimensions' must be > 0") + } + return nil +} + +func (e *Example) initCharts() (*module.Charts, error) { + charts := &module.Charts{} + + var ctx int + v := calcContextEvery(e.Config.Charts.Num, e.Config.Charts.Contexts) + for i := 0; i < e.Config.Charts.Num; i++ { + if i != 0 && v != 0 && ctx < (e.Config.Charts.Contexts-1) && i%v == 0 { + ctx++ + } + chart := newChart(i, ctx, e.Config.Charts.Labels, module.ChartType(e.Config.Charts.Type)) + + if err := charts.Add(chart); err != nil { + return nil, err + } + } + + ctx = 0 + v = calcContextEvery(e.Config.HiddenCharts.Num, e.Config.HiddenCharts.Contexts) + for i := 0; i < e.Config.HiddenCharts.Num; i++ { + if i != 0 && v != 0 && ctx < (e.Config.HiddenCharts.Contexts-1) && i%v == 0 { + ctx++ + } + chart := newHiddenChart(i, ctx, e.Config.HiddenCharts.Labels, module.ChartType(e.Config.HiddenCharts.Type)) + + if err := charts.Add(chart); err != nil { + return nil, err + } + } + + return charts, nil +} + +func calcContextEvery(charts, contexts int) int { + if contexts <= 1 { + return 0 + } + if contexts > charts { + return 1 + } + return charts / contexts +} diff --git a/src/go/collectors/go.d.plugin/modules/example/testdata/config.json b/src/go/collectors/go.d.plugin/modules/example/testdata/config.json new file mode 100644 index 000000000..af06e85ac --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/testdata/config.json @@ -0,0 +1,17 @@ +{ + "update_every": 123, + "charts": { + "type": "ok", + "num": 123, + "contexts": 123, + "dimensions": 123, + "labels": 123 + }, + "hidden_charts": { + "type": "ok", + "num": 123, + "contexts": 123, + "dimensions": 123, + "labels": 123 + } +} diff --git a/src/go/collectors/go.d.plugin/modules/example/testdata/config.yaml b/src/go/collectors/go.d.plugin/modules/example/testdata/config.yaml new file mode 100644 index 000000000..a5f6556fd --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/example/testdata/config.yaml @@ -0,0 +1,13 @@ +update_every: 123 +charts: + type: "ok" + num: 123 + contexts: 123 + dimensions: 123 + labels: 123 +hidden_charts: + type: "ok" + num: 123 + contexts: 123 + dimensions: 123 + labels: 123 |