diff options
Diffstat (limited to '')
15 files changed, 1877 insertions, 0 deletions
diff --git a/collectors/python.d.plugin/beanstalk/README.md b/src/go/plugin/go.d/modules/beanstalk/README.md index 4efe13889..4efe13889 120000 --- a/collectors/python.d.plugin/beanstalk/README.md +++ b/src/go/plugin/go.d/modules/beanstalk/README.md diff --git a/src/go/plugin/go.d/modules/beanstalk/beanstalk.go b/src/go/plugin/go.d/modules/beanstalk/beanstalk.go new file mode 100644 index 000000000..f37cbeda4 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/beanstalk.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package beanstalk + +import ( + _ "embed" + "errors" + "fmt" + "time" + + "github.com/netdata/netdata/go/plugins/logger" + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/matcher" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("beanstalk", module.Creator{ + JobConfigSchema: configSchema, + Create: func() module.Module { return New() }, + Config: func() any { return &Config{} }, + }) +} + +func New() *Beanstalk { + return &Beanstalk{ + Config: Config{ + Address: "127.0.0.1:11300", + Timeout: web.Duration(time.Second * 1), + TubeSelector: "*", + }, + + charts: statsCharts.Copy(), + newConn: newBeanstalkConn, + discoverTubesEvery: time.Minute * 1, + tubeSr: matcher.TRUE(), + seenTubes: make(map[string]bool), + } +} + +type Config struct { + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"` + Address string `yaml:"address" json:"address"` + Timeout web.Duration `yaml:"timeout,omitempty" json:"timeout"` + TubeSelector string `yaml:"tube_selector,omitempty" json:"tube_selector"` +} + +type Beanstalk struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + + newConn func(Config, *logger.Logger) beanstalkConn + conn beanstalkConn + + discoverTubesEvery time.Duration + lastDiscoverTubesTime time.Time + discoveredTubes []string + tubeSr matcher.Matcher + seenTubes map[string]bool +} + +func (b *Beanstalk) Configuration() any { + return b.Config +} + +func (b *Beanstalk) Init() error { + if err := b.validateConfig(); err != nil { + return fmt.Errorf("config validation: %v", err) + } + + sr, err := b.initTubeSelector() + if err != nil { + return fmt.Errorf("failed to init tube selector: %v", err) + } + b.tubeSr = sr + + return nil +} + +func (b *Beanstalk) Check() error { + mx, err := b.collect() + if err != nil { + b.Error(err) + return err + } + + if len(mx) == 0 { + return errors.New("no metrics collected") + } + + return nil +} + +func (b *Beanstalk) Charts() *module.Charts { + return b.charts +} + +func (b *Beanstalk) Collect() map[string]int64 { + mx, err := b.collect() + if err != nil { + b.Error(err) + } + + if len(mx) == 0 { + return nil + } + + return mx +} + +func (b *Beanstalk) Cleanup() { + if b.conn != nil { + if err := b.conn.disconnect(); err != nil { + b.Warningf("error on disconnect: %s", err) + } + b.conn = nil + } +} diff --git a/src/go/plugin/go.d/modules/beanstalk/beanstalk_test.go b/src/go/plugin/go.d/modules/beanstalk/beanstalk_test.go new file mode 100644 index 000000000..da1fcaf08 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/beanstalk_test.go @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package beanstalk + +import ( + "bufio" + "errors" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") + + dataStats, _ = os.ReadFile("testdata/stats.txt") + dataListTubes, _ = os.ReadFile("testdata/list-tubes.txt") + dataStatsTubeDefault, _ = os.ReadFile("testdata/stats-tube-default.txt") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + "dataStats": dataStats, + "dataListTubes": dataListTubes, + "dataStatsTubeDefault": dataStatsTubeDefault, + } { + require.NotNil(t, data, name) + } +} + +func TestBeanstalk_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Beanstalk{}, dataConfigJSON, dataConfigYAML) +} + +func TestBeanstalk_Init(t *testing.T) { + tests := map[string]struct { + config Config + wantFail bool + }{ + "success with default config": { + wantFail: false, + config: New().Config, + }, + "fails if address not set": { + wantFail: true, + config: func() Config { + conf := New().Config + conf.Address = "" + return conf + }(), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + beans := New() + beans.Config = test.config + + if test.wantFail { + assert.Error(t, beans.Init()) + } else { + assert.NoError(t, beans.Init()) + } + }) + } +} + +func TestBeanstalk_Charts(t *testing.T) { + assert.NotNil(t, New().Charts()) +} + +func TestBeanstalk_Check(t *testing.T) { + tests := map[string]struct { + prepare func() (*Beanstalk, *mockBeanstalkDaemon) + wantFail bool + }{ + "success on valid response": { + wantFail: false, + prepare: prepareCaseOk, + }, + "fails on unexpected response": { + wantFail: true, + prepare: prepareCaseUnexpectedResponse, + }, + "fails on connection refused": { + wantFail: true, + prepare: prepareCaseConnectionRefused, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + beanstalk, daemon := test.prepare() + + defer func() { + assert.NoError(t, daemon.Close(), "daemon.Close()") + }() + go func() { + assert.NoError(t, daemon.Run(), "daemon.Run()") + }() + + select { + case <-daemon.started: + case <-time.After(time.Second * 3): + t.Errorf("mock beanstalk daemon start timed out") + } + + require.NoError(t, beanstalk.Init()) + + if test.wantFail { + assert.Error(t, beanstalk.Check()) + } else { + assert.NoError(t, beanstalk.Check()) + } + + beanstalk.Cleanup() + + select { + case <-daemon.stopped: + case <-time.After(time.Second * 3): + t.Errorf("mock beanstalk daemon stop timed out") + } + }) + } +} + +func TestBeanstalk_Collect(t *testing.T) { + tests := map[string]struct { + prepare func() (*Beanstalk, *mockBeanstalkDaemon) + wantMetrics map[string]int64 + wantCharts int + }{ + "success on valid response": { + prepare: prepareCaseOk, + wantMetrics: map[string]int64{ + "binlog-records-migrated": 0, + "binlog-records-written": 0, + "cmd-bury": 0, + "cmd-delete": 0, + "cmd-ignore": 0, + "cmd-kick": 0, + "cmd-list-tube-used": 0, + "cmd-list-tubes": 317, + "cmd-list-tubes-watched": 0, + "cmd-pause-tube": 0, + "cmd-peek": 0, + "cmd-peek-buried": 0, + "cmd-peek-delayed": 0, + "cmd-peek-ready": 0, + "cmd-put": 0, + "cmd-release": 0, + "cmd-reserve": 0, + "cmd-reserve-with-timeout": 0, + "cmd-stats": 23619, + "cmd-stats-job": 0, + "cmd-stats-tube": 18964, + "cmd-touch": 0, + "cmd-use": 0, + "cmd-watch": 0, + "current-connections": 2, + "current-jobs-buried": 0, + "current-jobs-delayed": 0, + "current-jobs-ready": 0, + "current-jobs-reserved": 0, + "current-jobs-urgent": 0, + "current-producers": 0, + "current-tubes": 1, + "current-waiting": 0, + "current-workers": 0, + "job-timeouts": 0, + "rusage-stime": 3922, + "rusage-utime": 1602, + "total-connections": 72, + "total-jobs": 0, + "tube_default_cmd-delete": 0, + "tube_default_cmd-pause-tube": 0, + "tube_default_current-jobs-buried": 0, + "tube_default_current-jobs-delayed": 0, + "tube_default_current-jobs-ready": 0, + "tube_default_current-jobs-reserved": 0, + "tube_default_current-jobs-urgent": 0, + "tube_default_current-using": 2, + "tube_default_current-waiting": 0, + "tube_default_current-watching": 2, + "tube_default_pause": 0, + "tube_default_pause-time-left": 0, + "tube_default_total-jobs": 0, + "uptime": 105881, + }, + wantCharts: len(statsCharts) + len(tubeChartsTmpl)*1, + }, + "fails on unexpected response": { + prepare: prepareCaseUnexpectedResponse, + wantCharts: len(statsCharts), + }, + "fails on connection refused": { + prepare: prepareCaseConnectionRefused, + wantCharts: len(statsCharts), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + beanstalk, daemon := test.prepare() + + defer func() { + assert.NoError(t, daemon.Close(), "daemon.Close()") + }() + go func() { + assert.NoError(t, daemon.Run(), "daemon.Run()") + }() + + select { + case <-daemon.started: + case <-time.After(time.Second * 3): + t.Errorf("mock beanstalk daemon start timed out") + } + + require.NoError(t, beanstalk.Init()) + + mx := beanstalk.Collect() + + require.Equal(t, test.wantMetrics, mx) + + assert.Equal(t, test.wantCharts, len(*beanstalk.Charts()), "want charts") + + if len(test.wantMetrics) > 0 { + module.TestMetricsHasAllChartsDims(t, beanstalk.Charts(), mx) + } + + beanstalk.Cleanup() + + select { + case <-daemon.stopped: + case <-time.After(time.Second * 3): + t.Errorf("mock beanstalk daemon stop timed out") + } + }) + } +} + +func prepareCaseOk() (*Beanstalk, *mockBeanstalkDaemon) { + daemon := &mockBeanstalkDaemon{ + addr: "127.0.0.1:65001", + started: make(chan struct{}), + stopped: make(chan struct{}), + dataStats: dataStats, + dataListTubes: dataListTubes, + dataStatsTube: dataStatsTubeDefault, + } + + beanstalk := New() + beanstalk.Address = daemon.addr + + return beanstalk, daemon +} + +func prepareCaseUnexpectedResponse() (*Beanstalk, *mockBeanstalkDaemon) { + daemon := &mockBeanstalkDaemon{ + addr: "127.0.0.1:65001", + started: make(chan struct{}), + stopped: make(chan struct{}), + dataStats: []byte("INTERNAL_ERROR\n"), + dataListTubes: []byte("INTERNAL_ERROR\n"), + dataStatsTube: []byte("INTERNAL_ERROR\n"), + } + + beanstalk := New() + beanstalk.Address = daemon.addr + + return beanstalk, daemon +} + +func prepareCaseConnectionRefused() (*Beanstalk, *mockBeanstalkDaemon) { + ch := make(chan struct{}) + close(ch) + daemon := &mockBeanstalkDaemon{ + addr: "127.0.0.1:65001", + dontStart: true, + started: ch, + stopped: ch, + } + + beanstalk := New() + beanstalk.Address = daemon.addr + + return beanstalk, daemon +} + +type mockBeanstalkDaemon struct { + addr string + srv net.Listener + started chan struct{} + stopped chan struct{} + dontStart bool + + dataStats []byte + dataListTubes []byte + dataStatsTube []byte +} + +func (m *mockBeanstalkDaemon) Run() error { + if m.dontStart { + return nil + } + + srv, err := net.Listen("tcp", m.addr) + if err != nil { + return err + } + + m.srv = srv + + close(m.started) + defer close(m.stopped) + + return m.handleConnections() +} + +func (m *mockBeanstalkDaemon) Close() error { + if m.srv != nil { + err := m.srv.Close() + m.srv = nil + return err + } + return nil +} + +func (m *mockBeanstalkDaemon) handleConnections() error { + conn, err := m.srv.Accept() + if err != nil || conn == nil { + return errors.New("could not accept connection") + } + return m.handleConnection(conn) +} + +func (m *mockBeanstalkDaemon) handleConnection(conn net.Conn) error { + defer func() { _ = conn.Close() }() + + rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) + var line string + var err error + + for { + if line, err = rw.ReadString('\n'); err != nil { + return fmt.Errorf("error reading from connection: %v", err) + } + + line = strings.TrimSpace(line) + + cmd, param, _ := strings.Cut(line, " ") + + switch cmd { + case cmdQuit: + return nil + case cmdStats: + _, err = rw.Write(m.dataStats) + case cmdListTubes: + _, err = rw.Write(m.dataListTubes) + case cmdStatsTube: + if param == "default" { + _, err = rw.Write(m.dataStatsTube) + } else { + _, err = rw.WriteString("NOT_FOUND\n") + } + default: + return fmt.Errorf("unexpected command: %s", line) + } + _ = rw.Flush() + if err != nil { + return err + } + } +} diff --git a/src/go/plugin/go.d/modules/beanstalk/charts.go b/src/go/plugin/go.d/modules/beanstalk/charts.go new file mode 100644 index 000000000..fb2f22628 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/charts.go @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package beanstalk + +import ( + "fmt" + "strings" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" +) + +const ( + prioCurrentJobs = module.Priority + iota + prioJobsRate + prioJobsTimeouts + + prioCurrentTubes + + prioCommandsRate + + prioCurrentConnections + prioConnectionsRate + + prioBinlogRecords + + prioCpuUsage + + prioUptime + + prioTubeCurrentJobs + prioTubeJobsRate + + prioTubeCommands + + prioTubeCurrentConnections + + prioTubePauseTime +) + +var ( + statsCharts = module.Charts{ + currentJobs.Copy(), + jobsRateChart.Copy(), + jobsTimeoutsChart.Copy(), + + currentTubesChart.Copy(), + + commandsRateChart.Copy(), + + currentConnectionsChart.Copy(), + connectionsRateChart.Copy(), + + binlogRecordsChart.Copy(), + + cpuUsageChart.Copy(), + + uptimeChart.Copy(), + } + + currentJobs = module.Chart{ + ID: "current_jobs", + Title: "Current Jobs", + Units: "jobs", + Fam: "jobs", + Ctx: "beanstalk.current_jobs", + Type: module.Stacked, + Priority: prioCurrentJobs, + Dims: module.Dims{ + {ID: "current-jobs-ready", Name: "ready"}, + {ID: "current-jobs-buried", Name: "buried"}, + {ID: "current-jobs-urgent", Name: "urgent"}, + {ID: "current-jobs-delayed", Name: "delayed"}, + {ID: "current-jobs-reserved", Name: "reserved"}, + }, + } + jobsRateChart = module.Chart{ + ID: "jobs_rate", + Title: "Jobs Rate", + Units: "jobs/s", + Fam: "jobs", + Ctx: "beanstalk.jobs_rate", + Type: module.Line, + Priority: prioJobsRate, + Dims: module.Dims{ + {ID: "total-jobs", Name: "created", Algo: module.Incremental}, + }, + } + jobsTimeoutsChart = module.Chart{ + ID: "jobs_timeouts", + Title: "Timed Out Jobs", + Units: "jobs/s", + Fam: "jobs", + Ctx: "beanstalk.jobs_timeouts", + Type: module.Line, + Priority: prioJobsTimeouts, + Dims: module.Dims{ + {ID: "job-timeouts", Name: "timeouts", Algo: module.Incremental}, + }, + } + + currentTubesChart = module.Chart{ + ID: "current_tubes", + Title: "Current Tubes", + Units: "tubes", + Fam: "tubes", + Ctx: "beanstalk.current_tubes", + Type: module.Line, + Priority: prioCurrentTubes, + Dims: module.Dims{ + {ID: "current-tubes", Name: "tubes"}, + }, + } + + commandsRateChart = module.Chart{ + ID: "commands_rate", + Title: "Commands Rate", + Units: "commands/s", + Fam: "commands", + Ctx: "beanstalk.commands_rate", + Type: module.Stacked, + Priority: prioCommandsRate, + Dims: module.Dims{ + {ID: "cmd-put", Name: "put", Algo: module.Incremental}, + {ID: "cmd-peek", Name: "peek", Algo: module.Incremental}, + {ID: "cmd-peek-ready", Name: "peek-ready", Algo: module.Incremental}, + {ID: "cmd-peek-delayed", Name: "peek-delayed", Algo: module.Incremental}, + {ID: "cmd-peek-buried", Name: "peek-buried", Algo: module.Incremental}, + {ID: "cmd-reserve", Name: "reserve", Algo: module.Incremental}, + {ID: "cmd-reserve-with-timeout", Name: "reserve-with-timeout", Algo: module.Incremental}, + {ID: "cmd-touch", Name: "touch", Algo: module.Incremental}, + {ID: "cmd-use", Name: "use", Algo: module.Incremental}, + {ID: "cmd-watch", Name: "watch", Algo: module.Incremental}, + {ID: "cmd-ignore", Name: "ignore", Algo: module.Incremental}, + {ID: "cmd-delete", Name: "delete", Algo: module.Incremental}, + {ID: "cmd-release", Name: "release", Algo: module.Incremental}, + {ID: "cmd-bury", Name: "bury", Algo: module.Incremental}, + {ID: "cmd-kick", Name: "kick", Algo: module.Incremental}, + {ID: "cmd-stats", Name: "stats", Algo: module.Incremental}, + {ID: "cmd-stats-job", Name: "stats-job", Algo: module.Incremental}, + {ID: "cmd-stats-tube", Name: "stats-tube", Algo: module.Incremental}, + {ID: "cmd-list-tubes", Name: "list-tubes", Algo: module.Incremental}, + {ID: "cmd-list-tube-used", Name: "list-tube-used", Algo: module.Incremental}, + {ID: "cmd-list-tubes-watched", Name: "list-tubes-watched", Algo: module.Incremental}, + {ID: "cmd-pause-tube", Name: "pause-tube", Algo: module.Incremental}, + }, + } + + currentConnectionsChart = module.Chart{ + ID: "current_connections", + Title: "Current Connections", + Units: "connections", + Fam: "connections", + Ctx: "beanstalk.current_connections", + Type: module.Line, + Priority: prioCurrentConnections, + Dims: module.Dims{ + {ID: "current-connections", Name: "open"}, + {ID: "current-producers", Name: "producers"}, + {ID: "current-workers", Name: "workers"}, + {ID: "current-waiting", Name: "waiting"}, + }, + } + connectionsRateChart = module.Chart{ + ID: "connections_rate", + Title: "Connections Rate", + Units: "connections/s", + Fam: "connections", + Ctx: "beanstalk.connections_rate", + Type: module.Line, + Priority: prioConnectionsRate, + Dims: module.Dims{ + {ID: "total-connections", Name: "created", Algo: module.Incremental}, + }, + } + + binlogRecordsChart = module.Chart{ + ID: "binlog_records", + Title: "Binlog Records", + Units: "records/s", + Fam: "binlog", + Ctx: "beanstalk.binlog_records", + Type: module.Line, + Priority: prioBinlogRecords, + Dims: module.Dims{ + {ID: "binlog-records-written", Name: "written", Algo: module.Incremental}, + {ID: "binlog-records-migrated", Name: "migrated", Algo: module.Incremental}, + }, + } + + cpuUsageChart = module.Chart{ + ID: "cpu_usage", + Title: "CPU Usage", + Units: "percent", + Fam: "cpu usage", + Ctx: "beanstalk.cpu_usage", + Type: module.Stacked, + Priority: prioCpuUsage, + Dims: module.Dims{ + {ID: "rusage-utime", Name: "user", Algo: module.Incremental, Mul: 100, Div: 1000}, + {ID: "rusage-stime", Name: "system", Algo: module.Incremental, Mul: 100, Div: 1000}, + }, + } + + uptimeChart = module.Chart{ + ID: "uptime", + Title: "Uptime", + Units: "seconds", + Fam: "uptime", + Ctx: "beanstalk.uptime", + Type: module.Line, + Priority: prioUptime, + Dims: module.Dims{ + {ID: "uptime"}, + }, + } +) + +var ( + tubeChartsTmpl = module.Charts{ + tubeCurrentJobsChartTmpl.Copy(), + tubeJobsRateChartTmpl.Copy(), + + tubeCommandsRateChartTmpl.Copy(), + + tubeCurrentConnectionsChartTmpl.Copy(), + + tubePauseTimeChartTmpl.Copy(), + } + + tubeCurrentJobsChartTmpl = module.Chart{ + ID: "tube_%s_current_jobs", + Title: "Tube Current Jobs", + Units: "jobs", + Fam: "tube jobs", + Ctx: "beanstalk.tube_current_jobs", + Type: module.Stacked, + Priority: prioTubeCurrentJobs, + Dims: module.Dims{ + {ID: "tube_%s_current-jobs-ready", Name: "ready"}, + {ID: "tube_%s_current-jobs-buried", Name: "buried"}, + {ID: "tube_%s_current-jobs-urgent", Name: "urgent"}, + {ID: "tube_%s_current-jobs-delayed", Name: "delayed"}, + {ID: "tube_%s_current-jobs-reserved", Name: "reserved"}, + }, + } + tubeJobsRateChartTmpl = module.Chart{ + ID: "tube_%s_jobs_rate", + Title: "Tube Jobs Rate", + Units: "jobs/s", + Fam: "tube jobs", + Ctx: "beanstalk.tube_jobs_rate", + Type: module.Line, + Priority: prioTubeJobsRate, + Dims: module.Dims{ + {ID: "tube_%s_total-jobs", Name: "created", Algo: module.Incremental}, + }, + } + tubeCommandsRateChartTmpl = module.Chart{ + ID: "tube_%s_commands_rate", + Title: "Tube Commands", + Units: "commands/s", + Fam: "tube commands", + Ctx: "beanstalk.tube_commands_rate", + Type: module.Stacked, + Priority: prioTubeCommands, + Dims: module.Dims{ + {ID: "tube_%s_cmd-delete", Name: "delete", Algo: module.Incremental}, + {ID: "tube_%s_cmd-pause-tube", Name: "pause-tube", Algo: module.Incremental}, + }, + } + tubeCurrentConnectionsChartTmpl = module.Chart{ + ID: "tube_%s_current_connections", + Title: "Tube Current Connections", + Units: "connections", + Fam: "tube connections", + Ctx: "beanstalk.tube_current_connections", + Type: module.Stacked, + Priority: prioTubeCurrentConnections, + Dims: module.Dims{ + {ID: "tube_%s_current-using", Name: "using"}, + {ID: "tube_%s_current-waiting", Name: "waiting"}, + {ID: "tube_%s_current-watching", Name: "watching"}, + }, + } + tubePauseTimeChartTmpl = module.Chart{ + ID: "tube_%s_pause_time", + Title: "Tube Pause Time", + Units: "seconds", + Fam: "tube pause", + Ctx: "beanstalk.tube_pause", + Type: module.Line, + Priority: prioTubePauseTime, + Dims: module.Dims{ + {ID: "tube_%s_pause", Name: "since"}, + {ID: "tube_%s_pause-time-left", Name: "left"}, + }, + } +) + +func (b *Beanstalk) addTubeCharts(name string) { + charts := tubeChartsTmpl.Copy() + + for _, chart := range *charts { + chart.ID = fmt.Sprintf(chart.ID, cleanTubeName(name)) + chart.Labels = []module.Label{ + {Key: "tube_name", Value: name}, + } + + for _, dim := range chart.Dims { + dim.ID = fmt.Sprintf(dim.ID, name) + } + } + + if err := b.Charts().Add(*charts...); err != nil { + b.Warning(err) + } +} + +func (b *Beanstalk) removeTubeCharts(name string) { + px := fmt.Sprintf("tube_%s_", cleanTubeName(name)) + + for _, chart := range *b.Charts() { + if strings.HasPrefix(chart.ID, px) { + chart.MarkRemove() + chart.MarkNotCreated() + } + } +} + +func cleanTubeName(name string) string { + r := strings.NewReplacer(" ", "_", ".", "_", ",", "_") + return r.Replace(name) +} diff --git a/src/go/plugin/go.d/modules/beanstalk/client.go b/src/go/plugin/go.d/modules/beanstalk/client.go new file mode 100644 index 000000000..66a8b1cef --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/client.go @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package beanstalk + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/netdata/netdata/go/plugins/logger" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/socket" + + "gopkg.in/yaml.v2" +) + +type beanstalkConn interface { + connect() error + disconnect() error + queryStats() (*beanstalkdStats, error) + queryListTubes() ([]string, error) + queryStatsTube(string) (*tubeStats, error) +} + +// https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L553 +type beanstalkdStats struct { + CurrentJobsUrgent int64 `yaml:"current-jobs-urgent" stm:"current-jobs-urgent"` + CurrentJobsReady int64 `yaml:"current-jobs-ready" stm:"current-jobs-ready"` + CurrentJobsReserved int64 `yaml:"current-jobs-reserved" stm:"current-jobs-reserved"` + CurrentJobsDelayed int64 `yaml:"current-jobs-delayed" stm:"current-jobs-delayed"` + CurrentJobsBuried int64 `yaml:"current-jobs-buried" stm:"current-jobs-buried"` + CmdPut int64 `yaml:"cmd-put" stm:"cmd-put"` + CmdPeek int64 `yaml:"cmd-peek" stm:"cmd-peek"` + CmdPeekReady int64 `yaml:"cmd-peek-ready" stm:"cmd-peek-ready"` + CmdPeekDelayed int64 `yaml:"cmd-peek-delayed" stm:"cmd-peek-delayed"` + CmdPeekBuried int64 `yaml:"cmd-peek-buried" stm:"cmd-peek-buried"` + CmdReserve int64 `yaml:"cmd-reserve" stm:"cmd-reserve"` + CmdReserveWithTimeout int64 `yaml:"cmd-reserve-with-timeout" stm:"cmd-reserve-with-timeout"` + CmdTouch int64 `yaml:"cmd-touch" stm:"cmd-touch"` + CmdUse int64 `yaml:"cmd-use" stm:"cmd-use"` + CmdWatch int64 `yaml:"cmd-watch" stm:"cmd-watch"` + CmdIgnore int64 `yaml:"cmd-ignore" stm:"cmd-ignore"` + CmdDelete int64 `yaml:"cmd-delete" stm:"cmd-delete"` + CmdRelease int64 `yaml:"cmd-release" stm:"cmd-release"` + CmdBury int64 `yaml:"cmd-bury" stm:"cmd-bury"` + CmdKick int64 `yaml:"cmd-kick" stm:"cmd-kick"` + CmdStats int64 `yaml:"cmd-stats" stm:"cmd-stats"` + CmdStatsJob int64 `yaml:"cmd-stats-job" stm:"cmd-stats-job"` + CmdStatsTube int64 `yaml:"cmd-stats-tube" stm:"cmd-stats-tube"` + CmdListTubes int64 `yaml:"cmd-list-tubes" stm:"cmd-list-tubes"` + CmdListTubeUsed int64 `yaml:"cmd-list-tube-used" stm:"cmd-list-tube-used"` + CmdListTubesWatched int64 `yaml:"cmd-list-tubes-watched" stm:"cmd-list-tubes-watched"` + CmdPauseTube int64 `yaml:"cmd-pause-tube" stm:"cmd-pause-tube"` + JobTimeouts int64 `yaml:"job-timeouts" stm:"job-timeouts"` + TotalJobs int64 `yaml:"total-jobs" stm:"total-jobs"` + CurrentTubes int64 `yaml:"current-tubes" stm:"current-tubes"` + CurrentConnections int64 `yaml:"current-connections" stm:"current-connections"` + CurrentProducers int64 `yaml:"current-producers" stm:"current-producers"` + CurrentWorkers int64 `yaml:"current-workers" stm:"current-workers"` + CurrentWaiting int64 `yaml:"current-waiting" stm:"current-waiting"` + TotalConnections int64 `yaml:"total-connections" stm:"total-connections"` + RusageUtime float64 `yaml:"rusage-utime" stm:"rusage-utime,1000,1"` + RusageStime float64 `yaml:"rusage-stime" stm:"rusage-stime,1000,1"` + Uptime int64 `yaml:"uptime" stm:"uptime"` + BinlogRecordsWritten int64 `yaml:"binlog-records-written" stm:"binlog-records-written"` + BinlogRecordsMigrated int64 `yaml:"binlog-records-migrated" stm:"binlog-records-migrated"` +} + +// https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L497 +type tubeStats struct { + Name string `yaml:"name"` + CurrentJobsUrgent int64 `yaml:"current-jobs-urgent" stm:"current-jobs-urgent"` + CurrentJobsReady int64 `yaml:"current-jobs-ready" stm:"current-jobs-ready"` + CurrentJobsReserved int64 `yaml:"current-jobs-reserved" stm:"current-jobs-reserved"` + CurrentJobsDelayed int64 `yaml:"current-jobs-delayed" stm:"current-jobs-delayed"` + CurrentJobsBuried int64 `yaml:"current-jobs-buried" stm:"current-jobs-buried"` + TotalJobs int64 `yaml:"total-jobs" stm:"total-jobs"` + CurrentUsing int64 `yaml:"current-using" stm:"current-using"` + CurrentWaiting int64 `yaml:"current-waiting" stm:"current-waiting"` + CurrentWatching int64 `yaml:"current-watching" stm:"current-watching"` + Pause float64 `yaml:"pause" stm:"pause"` + CmdDelete int64 `yaml:"cmd-delete" stm:"cmd-delete"` + CmdPauseTube int64 `yaml:"cmd-pause-tube" stm:"cmd-pause-tube"` + PauseTimeLeft float64 `yaml:"pause-time-left" stm:"pause-time-left"` +} + +func newBeanstalkConn(conf Config, log *logger.Logger) beanstalkConn { + return &beanstalkClient{ + Logger: log, + client: socket.New(socket.Config{ + Address: conf.Address, + ConnectTimeout: conf.Timeout.Duration(), + ReadTimeout: conf.Timeout.Duration(), + WriteTimeout: conf.Timeout.Duration(), + TLSConf: nil, + }), + } +} + +const ( + cmdQuit = "quit" + cmdStats = "stats" + cmdListTubes = "list-tubes" + cmdStatsTube = "stats-tube" +) + +type beanstalkClient struct { + *logger.Logger + + client socket.Client +} + +func (c *beanstalkClient) connect() error { + return c.client.Connect() +} + +func (c *beanstalkClient) disconnect() error { + _, _, _ = c.query(cmdQuit) + return c.client.Disconnect() +} + +func (c *beanstalkClient) queryStats() (*beanstalkdStats, error) { + cmd := cmdStats + + resp, data, err := c.query(cmd) + if err != nil { + return nil, err + } + if resp != "OK" { + return nil, fmt.Errorf("command '%s' bad response: %s", cmd, resp) + } + + var stats beanstalkdStats + + if err := yaml.Unmarshal(data, &stats); err != nil { + return nil, err + } + + return &stats, nil +} + +func (c *beanstalkClient) queryListTubes() ([]string, error) { + cmd := cmdListTubes + + resp, data, err := c.query(cmd) + if err != nil { + return nil, err + } + if resp != "OK" { + return nil, fmt.Errorf("command '%s' bad response: %s", cmd, resp) + } + + var tubes []string + + if err := yaml.Unmarshal(data, &tubes); err != nil { + return nil, err + } + + return tubes, nil +} + +func (c *beanstalkClient) queryStatsTube(tubeName string) (*tubeStats, error) { + cmd := fmt.Sprintf("%s %s", cmdStatsTube, tubeName) + + resp, data, err := c.query(cmd) + if err != nil { + return nil, err + } + if resp == "NOT_FOUND" { + return nil, nil + } + if resp != "OK" { + return nil, fmt.Errorf("command '%s' bad response: %s", cmd, resp) + } + + var stats tubeStats + if err := yaml.Unmarshal(data, &stats); err != nil { + return nil, err + } + + return &stats, nil +} + +func (c *beanstalkClient) query(command string) (string, []byte, error) { + var resp string + var length int + var body []byte + var err error + + c.Debugf("executing command: %s", command) + + const limitReadLines = 1000 + var num int + + clientErr := c.client.Command(command+"\r\n", func(line []byte) bool { + if resp == "" { + s := string(line) + c.Debugf("command '%s' response: '%s'", command, s) + + resp, length, err = parseResponseLine(s) + if err != nil { + err = fmt.Errorf("command '%s' line '%s': %v", command, s, err) + } + return err == nil && resp == "OK" + } + + if num++; num >= limitReadLines { + err = fmt.Errorf("command '%s': read line limit exceeded (%d)", command, limitReadLines) + return false + } + + body = append(body, line...) + body = append(body, '\n') + + return len(body) < length + }) + if clientErr != nil { + return "", nil, fmt.Errorf("command '%s' client error: %v", command, clientErr) + } + if err != nil { + return "", nil, err + } + + return resp, body, nil +} + +func parseResponseLine(line string) (string, int, error) { + parts := strings.Fields(line) + if len(parts) == 0 { + return "", 0, errors.New("empty response") + } + + resp := parts[0] + + if resp != "OK" { + return resp, 0, nil + } + + if len(parts) < 2 { + return "", 0, errors.New("missing bytes count") + } + + length, err := strconv.Atoi(parts[1]) + if err != nil { + return "", 0, errors.New("invalid bytes count") + } + + return resp, length, nil +} diff --git a/src/go/plugin/go.d/modules/beanstalk/collect.go b/src/go/plugin/go.d/modules/beanstalk/collect.go new file mode 100644 index 000000000..f85b24028 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/collect.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package beanstalk + +import ( + "fmt" + "slices" + "time" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/stm" +) + +func (b *Beanstalk) collect() (map[string]int64, error) { + if b.conn == nil { + conn, err := b.establishConn() + if err != nil { + return nil, err + } + b.conn = conn + } + + mx := make(map[string]int64) + + if err := b.collectStats(mx); err != nil { + b.Cleanup() + return nil, err + } + if err := b.collectTubesStats(mx); err != nil { + return mx, err + } + + return mx, nil +} + +func (b *Beanstalk) collectStats(mx map[string]int64) error { + stats, err := b.conn.queryStats() + if err != nil { + return err + } + for k, v := range stm.ToMap(stats) { + mx[k] = v + } + return nil +} + +func (b *Beanstalk) collectTubesStats(mx map[string]int64) error { + now := time.Now() + + if now.Sub(b.lastDiscoverTubesTime) > b.discoverTubesEvery { + tubes, err := b.conn.queryListTubes() + if err != nil { + return err + } + + b.Debugf("discovered tubes (%d): %v", len(tubes), tubes) + v := slices.DeleteFunc(tubes, func(s string) bool { return !b.tubeSr.MatchString(s) }) + if len(tubes) != len(v) { + b.Debugf("discovered tubes after filtering (%d): %v", len(v), v) + } + + b.discoveredTubes = v + b.lastDiscoverTubesTime = now + } + + seen := make(map[string]bool) + + for i, tube := range b.discoveredTubes { + if tube == "" { + continue + } + + stats, err := b.conn.queryStatsTube(tube) + if err != nil { + return err + } + + if stats == nil { + b.Infof("tube '%s' stats object not found (tube does not exist)", tube) + b.discoveredTubes[i] = "" + continue + } + if stats.Name == "" { + b.Debugf("tube '%s' stats object has an empty name, ignoring it", tube) + b.discoveredTubes[i] = "" + continue + } + + seen[stats.Name] = true + if !b.seenTubes[stats.Name] { + b.seenTubes[stats.Name] = true + b.addTubeCharts(stats.Name) + } + + px := fmt.Sprintf("tube_%s_", stats.Name) + for k, v := range stm.ToMap(stats) { + mx[px+k] = v + } + } + + for tube := range b.seenTubes { + if !seen[tube] { + delete(b.seenTubes, tube) + b.removeTubeCharts(tube) + } + } + + return nil +} + +func (b *Beanstalk) establishConn() (beanstalkConn, error) { + conn := b.newConn(b.Config, b.Logger) + + if err := conn.connect(); err != nil { + return nil, err + } + + return conn, nil +} diff --git a/src/go/plugin/go.d/modules/beanstalk/config_schema.json b/src/go/plugin/go.d/modules/beanstalk/config_schema.json new file mode 100644 index 000000000..aa600ac03 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/config_schema.json @@ -0,0 +1,54 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Beanstalk collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "address": { + "title": "Address", + "description": "The IP address and port where the Beanstalk service listens for connections.", + "type": "string", + "default": "127.0.0.1:11300" + }, + "timeout": { + "title": "Timeout", + "description": "Timeout for establishing a connection and communication (reading and writing) in seconds.", + "type": "number", + "minimum": 0.5, + "default": 1 + }, + "tube_selector": { + "title": "Tube selector", + "description": "Specifies a [pattern](https://github.com/netdata/netdata/tree/master/src/libnetdata/simple_pattern#readme) for which Beanstalk tubes Netdata will collect statistics. Only tubes whose names match the provided pattern will be included.", + "type": "string", + "minimum": 1, + "default": "*" + } + }, + "required": [ + "address" + ], + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "uiOptions": { + "fullPage": true + }, + "timeout": { + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)." + }, + "tube_selector": { + "ui:help": "Leave blank or use `*` to collect data for all tubes." + } + } +} diff --git a/src/go/plugin/go.d/modules/beanstalk/init.go b/src/go/plugin/go.d/modules/beanstalk/init.go new file mode 100644 index 000000000..50916b3a7 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/init.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package beanstalk + +import ( + "errors" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/matcher" +) + +func (b *Beanstalk) validateConfig() error { + if b.Address == "" { + return errors.New("beanstalk address is required") + } + return nil +} + +func (b *Beanstalk) initTubeSelector() (matcher.Matcher, error) { + if b.TubeSelector == "" { + return matcher.TRUE(), nil + } + + m, err := matcher.NewSimplePatternsMatcher(b.TubeSelector) + if err != nil { + return nil, err + } + + return m, nil +} diff --git a/src/go/plugin/go.d/modules/beanstalk/integrations/beanstalk.md b/src/go/plugin/go.d/modules/beanstalk/integrations/beanstalk.md new file mode 100644 index 000000000..c8efd988a --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/integrations/beanstalk.md @@ -0,0 +1,253 @@ +<!--startmeta +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/modules/beanstalk/README.md" +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/plugin/go.d/modules/beanstalk/metadata.yaml" +sidebar_label: "Beanstalk" +learn_status: "Published" +learn_rel_path: "Collecting Metrics/Message Brokers" +most_popular: False +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" +endmeta--> + +# Beanstalk + + +<img src="https://netdata.cloud/img/beanstalk.svg" width="150"/> + + +Plugin: go.d.plugin +Module: beanstalk + +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" /> + +## Overview + +This collector monitors Beanstalk server performance and provides detailed statistics for each tube. + + +Using the [beanstalkd protocol](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt), it communicates with the Beanstalk daemon to gather essential metrics that help understand the server's performance and activity. +Executed commands: + +- [stats](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L553). +- [list-tubes](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L688). +- [stats-tube](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L497). + + +This collector is supported on all platforms. + +This collector supports collecting metrics from multiple instances of this integration, including remote instances. + + +### Default Behavior + +#### Auto-Detection + +By default, it detects Beanstalk instances running on localhost that are listening on port 11300. + + +#### Limits + +The default configuration for this integration does not impose any limits on data collection. + +#### Performance Impact + +The default configuration for this integration is not expected to impose a significant performance impact on the system. + + +## Metrics + +Metrics grouped by *scope*. + +The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels. + + + +### Per Beanstalk instance + +These metrics refer to the entire monitored application. + +This scope has no labels. + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| beanstalk.current_jobs | ready, buried, urgent, delayed, reserved | jobs | +| beanstalk.jobs_rate | created | jobs/s | +| beanstalk.jobs_timeouts | timeouts | jobs/s | +| beanstalk.current_tubes | tubes | tubes | +| beanstalk.commands_rate | put, peek, peek-ready, peek-delayed, peek-buried, reserve, reserve-with-timeout, touch, use, watch, ignore, delete, bury, kick, stats, stats-job, stats-tube, list-tubes, list-tube-used, list-tubes-watched, pause-tube | commands/s | +| beanstalk.current_connections | open, producers, workers, waiting | connections | +| beanstalk.connections_rate | created | connections/s | +| beanstalk.binlog_records | written, migrated | records/s | +| beanstalk.cpu_usage | user, system | percent | +| beanstalk.uptime | uptime | seconds | + +### Per tube + +Metrics related to Beanstalk tubes. This set of metrics is provided for each tube. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| tube_name | Tube name. | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| beanstalk.tube_current_jobs | ready, buried, urgent, delayed, reserved | jobs | +| beanstalk.tube_jobs_rate | created | jobs/s | +| beanstalk.tube_commands_rate | delete, pause-tube | commands/s | +| beanstalk.tube_current_connections | using, waiting, watching | connections | +| beanstalk.tube_pause_time | since, left | seconds | + + + +## Alerts + + +The following alerts are available: + +| Alert name | On metric | Description | +|:------------|:----------|:------------| +| [ beanstalk_server_buried_jobs ](https://github.com/netdata/netdata/blob/master/src/health/health.d/beanstalkd.conf) | beanstalk.current_jobs | number of buried jobs across all tubes. You need to manually kick them so they can be processed. Presence of buried jobs in a tube does not affect new jobs. | + + +## Setup + +### Prerequisites + +No action required. + +### Configuration + +#### File + +The configuration file name for this integration is `go.d/beanstalk.conf`. + + +You can edit the configuration file using the `edit-config` script from the +Netdata [config directory](/docs/netdata-agent/configuration/README.md#the-netdata-config-directory). + +```bash +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata +sudo ./edit-config go.d/beanstalk.conf +``` +#### Options + +The following options can be defined globally: update_every, autodetection_retry. + + +<details open><summary>Config options</summary> + +| Name | Description | Default | Required | +|:----|:-----------|:-------|:--------:| +| update_every | Data collection frequency. | 1 | no | +| autodetection_retry | Recheck interval in seconds. Zero means no recheck will be scheduled. | 0 | no | +| address | The IP address and port where the Beanstalk service listens for connections. | 127.0.0.1:11300 | yes | +| timeout | Connection, read, and write timeout duration in seconds. The timeout includes name resolution. | 1 | no | +| tube_selector | Specifies a [pattern](https://github.com/netdata/netdata/tree/master/src/libnetdata/simple_pattern#readme) for which Beanstalk tubes Netdata will collect statistics. | * | no | + +</details> + +#### Examples + +##### Basic + +A basic example configuration. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + address: 127.0.0.1:11300 + +``` +</details> + +##### Multi-instance + +> **Note**: When you define multiple jobs, their names must be unique. + +Collecting metrics from local and remote instances. + + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + address: 127.0.0.1:11300 + + - name: remote + address: 203.0.113.0:11300 + +``` +</details> + + + +## Troubleshooting + +### Debug Mode + +**Important**: Debug mode is not supported for data collection jobs created via the UI using the Dyncfg feature. + +To troubleshoot issues with the `beanstalk` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m beanstalk + ``` + +### Getting Logs + +If you're encountering problems with the `beanstalk` collector, follow these steps to retrieve logs and identify potential issues: + +- **Run the command** specific to your system (systemd, non-systemd, or Docker container). +- **Examine the output** for any warnings or error messages that might indicate issues. These messages should provide clues about the root cause of the problem. + +#### System with systemd + +Use the following command to view logs generated since the last Netdata service restart: + +```bash +journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep beanstalk +``` + +#### System without systemd + +Locate the collector log file, typically at `/var/log/netdata/collector.log`, and use `grep` to filter for collector's name: + +```bash +grep beanstalk /var/log/netdata/collector.log +``` + +**Note**: This method shows logs from all restarts. Focus on the **latest entries** for troubleshooting current issues. + +#### Docker Container + +If your Netdata runs in a Docker container named "netdata" (replace if different), use this command: + +```bash +docker logs netdata 2>&1 | grep beanstalk +``` + + diff --git a/src/go/plugin/go.d/modules/beanstalk/metadata.yaml b/src/go/plugin/go.d/modules/beanstalk/metadata.yaml new file mode 100644 index 000000000..60aaf77e5 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/metadata.yaml @@ -0,0 +1,255 @@ +plugin_name: go.d.plugin +modules: + - meta: + id: collector-go.d.plugin-beanstalk + plugin_name: go.d.plugin + module_name: beanstalk + monitored_instance: + name: Beanstalk + link: https://beanstalkd.github.io/ + categories: + - data-collection.message-brokers + icon_filename: "beanstalk.svg" + related_resources: + integrations: + list: [] + info_provided_to_referring_integrations: + description: "" + keywords: + - beanstalk + - beanstalkd + - message + most_popular: false + overview: + data_collection: + metrics_description: | + This collector monitors Beanstalk server performance and provides detailed statistics for each tube. + method_description: | + Using the [beanstalkd protocol](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt), it communicates with the Beanstalk daemon to gather essential metrics that help understand the server's performance and activity. + Executed commands: + + - [stats](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L553). + - [list-tubes](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L688). + - [stats-tube](https://github.com/beanstalkd/beanstalkd/blob/91c54fc05dc759ef27459ce4383934e1a4f2fb4b/doc/protocol.txt#L497). + supported_platforms: + include: [] + exclude: [] + multi_instance: true + additional_permissions: + description: "" + default_behavior: + auto_detection: + description: | + By default, it detects Beanstalk instances running on localhost that are listening on port 11300. + limits: + description: "" + performance_impact: + description: "" + setup: + prerequisites: + list: [] + configuration: + file: + name: go.d/beanstalk.conf + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 1 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: address + description: The IP address and port where the Beanstalk service listens for connections. + default_value: 127.0.0.1:11300 + required: true + - name: timeout + description: Connection, read, and write timeout duration in seconds. The timeout includes name resolution. + default_value: 1 + required: false + - name: tube_selector + description: "Specifies a [pattern](https://github.com/netdata/netdata/tree/master/src/libnetdata/simple_pattern#readme) for which Beanstalk tubes Netdata will collect statistics." + default_value: "*" + required: false + examples: + folding: + enabled: true + title: Config + list: + - name: Basic + description: A basic example configuration. + config: | + jobs: + - name: local + address: 127.0.0.1:11300 + - name: Multi-instance + description: | + > **Note**: When you define multiple jobs, their names must be unique. + + Collecting metrics from local and remote instances. + config: | + jobs: + - name: local + address: 127.0.0.1:11300 + + - name: remote + address: 203.0.113.0:11300 + troubleshooting: + problems: + list: [] + alerts: + - name: beanstalk_server_buried_jobs + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/beanstalkd.conf + metric: beanstalk.current_jobs + info: number of buried jobs across all tubes. You need to manually kick them so they can be processed. Presence of buried jobs in a tube does not affect new jobs. + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: global + description: "These metrics refer to the entire monitored application." + labels: [] + metrics: + - name: beanstalk.current_jobs + description: Current Jobs + unit: "jobs" + chart_type: stacked + dimensions: + - name: ready + - name: buried + - name: urgent + - name: delayed + - name: reserved + - name: beanstalk.jobs_rate + description: Jobs Rate + unit: "jobs/s" + chart_type: line + dimensions: + - name: created + - name: beanstalk.jobs_timeouts + description: Timed Out Jobs + unit: "jobs/s" + chart_type: line + dimensions: + - name: timeouts + - name: beanstalk.current_tubes + description: Current Tubes + unit: "tubes" + chart_type: line + dimensions: + - name: tubes + - name: beanstalk.commands_rate + description: Commands Rate + unit: "commands/s" + chart_type: stacked + dimensions: + - name: put + - name: peek + - name: peek-ready + - name: peek-delayed + - name: peek-buried + - name: reserve + - name: reserve-with-timeout + - name: touch + - name: use + - name: watch + - name: ignore + - name: delete + - name: bury + - name: kick + - name: stats + - name: stats-job + - name: stats-tube + - name: list-tubes + - name: list-tube-used + - name: list-tubes-watched + - name: pause-tube + - name: beanstalk.current_connections + description: Current Connections + unit: "connections" + chart_type: line + dimensions: + - name: open + - name: producers + - name: workers + - name: waiting + - name: beanstalk.connections_rate + description: Connections Rate + unit: "connections/s" + chart_type: area + dimensions: + - name: created + - name: beanstalk.binlog_records + description: Binlog Records + unit: "records/s" + chart_type: line + dimensions: + - name: written + - name: migrated + - name: beanstalk.cpu_usage + description: Cpu Usage + unit: "percent" + chart_type: stacked + dimensions: + - name: user + - name: system + - name: beanstalk.uptime + description: seconds + unit: "seconds" + chart_type: line + dimensions: + - name: uptime + - name: tube + description: "Metrics related to Beanstalk tubes. This set of metrics is provided for each tube." + labels: + - name: tube_name + description: Tube name. + metrics: + - name: beanstalk.tube_current_jobs + description: Tube Current Jobs + unit: "jobs" + chart_type: stacked + dimensions: + - name: ready + - name: buried + - name: urgent + - name: delayed + - name: reserved + - name: beanstalk.tube_jobs_rate + description: Tube Jobs Rate + unit: "jobs/s" + chart_type: line + dimensions: + - name: created + - name: beanstalk.tube_commands_rate + description: Tube Commands + unit: "commands/s" + chart_type: stacked + dimensions: + - name: delete + - name: pause-tube + - name: beanstalk.tube_current_connections + description: Tube Current Connections + unit: "connections" + chart_type: stacked + dimensions: + - name: using + - name: waiting + - name: watching + - name: beanstalk.tube_pause_time + description: Tube Pause Time + unit: "seconds" + chart_type: line + dimensions: + - name: since + - name: left diff --git a/src/go/plugin/go.d/modules/beanstalk/testdata/config.json b/src/go/plugin/go.d/modules/beanstalk/testdata/config.json new file mode 100644 index 000000000..c8da279a8 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/testdata/config.json @@ -0,0 +1,6 @@ +{ + "update_every": 123, + "address": "ok", + "timeout": 123.123, + "tube_selector": "ok" +} diff --git a/src/go/plugin/go.d/modules/beanstalk/testdata/config.yaml b/src/go/plugin/go.d/modules/beanstalk/testdata/config.yaml new file mode 100644 index 000000000..7fe212a96 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/testdata/config.yaml @@ -0,0 +1,4 @@ +update_every: 123 +address: "ok" +timeout: 123.123 +tube_selector: "ok" diff --git a/src/go/plugin/go.d/modules/beanstalk/testdata/list-tubes.txt b/src/go/plugin/go.d/modules/beanstalk/testdata/list-tubes.txt new file mode 100644 index 000000000..4fec61ef1 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/testdata/list-tubes.txt @@ -0,0 +1,3 @@ +OK 14 +--- +- default diff --git a/src/go/plugin/go.d/modules/beanstalk/testdata/stats-tube-default.txt b/src/go/plugin/go.d/modules/beanstalk/testdata/stats-tube-default.txt new file mode 100644 index 000000000..888ff3da4 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/testdata/stats-tube-default.txt @@ -0,0 +1,16 @@ +OK 265 +--- +name: default +current-jobs-urgent: 0 +current-jobs-ready: 0 +current-jobs-reserved: 0 +current-jobs-delayed: 0 +current-jobs-buried: 0 +total-jobs: 0 +current-using: 2 +current-watching: 2 +current-waiting: 0 +cmd-delete: 0 +cmd-pause-tube: 0 +pause: 0 +pause-time-left: 0 diff --git a/src/go/plugin/go.d/modules/beanstalk/testdata/stats.txt b/src/go/plugin/go.d/modules/beanstalk/testdata/stats.txt new file mode 100644 index 000000000..69b06e4c5 --- /dev/null +++ b/src/go/plugin/go.d/modules/beanstalk/testdata/stats.txt @@ -0,0 +1,50 @@ +OK 913 +--- +current-jobs-urgent: 0 +current-jobs-ready: 0 +current-jobs-reserved: 0 +current-jobs-delayed: 0 +current-jobs-buried: 0 +cmd-put: 0 +cmd-peek: 0 +cmd-peek-ready: 0 +cmd-peek-delayed: 0 +cmd-peek-buried: 0 +cmd-reserve: 0 +cmd-reserve-with-timeout: 0 +cmd-delete: 0 +cmd-release: 0 +cmd-use: 0 +cmd-watch: 0 +cmd-ignore: 0 +cmd-bury: 0 +cmd-kick: 0 +cmd-touch: 0 +cmd-stats: 23619 +cmd-stats-job: 0 +cmd-stats-tube: 18964 +cmd-list-tubes: 317 +cmd-list-tube-used: 0 +cmd-list-tubes-watched: 0 +cmd-pause-tube: 0 +job-timeouts: 0 +total-jobs: 0 +max-job-size: 65535 +current-tubes: 1 +current-connections: 2 +current-producers: 0 +current-workers: 0 +current-waiting: 0 +total-connections: 72 +pid: 1 +version: 1.10 +rusage-utime: 1.602079 +rusage-stime: 3.922748 +uptime: 105881 +binlog-oldest-index: 0 +binlog-current-index: 0 +binlog-records-migrated: 0 +binlog-records-written: 0 +binlog-max-size: 10485760 +id: 5a0667a881cd05e0 +hostname: c6796814b94b |