diff options
Diffstat (limited to 'src/go/collectors/go.d.plugin/modules/filecheck')
18 files changed, 1644 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/README.md b/src/go/collectors/go.d.plugin/modules/filecheck/README.md new file mode 120000 index 000000000..24dc78d8d --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/README.md @@ -0,0 +1 @@ +integrations/files_and_directories.md
\ No newline at end of file diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/charts.go b/src/go/collectors/go.d.plugin/modules/filecheck/charts.go new file mode 100644 index 000000000..7c7ada7a5 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/charts.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import "github.com/netdata/netdata/go/go.d.plugin/agent/module" + +var ( + fileCharts = module.Charts{ + fileExistenceChart.Copy(), + fileModTimeAgoChart.Copy(), + fileSizeChart.Copy(), + } + + fileExistenceChart = module.Chart{ + ID: "file_existence", + Title: "File Existence (0: not exists, 1: exists)", + Units: "boolean", + Fam: "files", + Ctx: "filecheck.file_existence", + Vars: module.Vars{ + {ID: "num_of_files"}, + }, + } + fileModTimeAgoChart = module.Chart{ + ID: "file_mtime_ago", + Title: "File Time Since the Last Modification", + Units: "seconds", + Fam: "files", + Ctx: "filecheck.file_mtime_ago", + } + fileSizeChart = module.Chart{ + ID: "file_size", + Title: "File Size", + Units: "bytes", + Fam: "files", + Ctx: "filecheck.file_size", + } +) + +var ( + dirCharts = module.Charts{ + dirExistenceChart.Copy(), + dirModTimeChart.Copy(), + dirNumOfFilesChart.Copy(), + dirSizeChart.Copy(), + } + + dirExistenceChart = module.Chart{ + ID: "dir_existence", + Title: "Dir Existence (0: not exists, 1: exists)", + Units: "boolean", + Fam: "dirs", + Ctx: "filecheck.dir_existence", + Vars: module.Vars{ + {ID: "num_of_dirs"}, + }, + } + dirModTimeChart = module.Chart{ + ID: "dir_mtime_ago", + Title: "Dir Time Since the Last Modification", + Units: "seconds", + Fam: "dirs", + Ctx: "filecheck.dir_mtime_ago", + } + dirNumOfFilesChart = module.Chart{ + ID: "dir_num_of_files", + Title: "Dir Number of Files", + Units: "files", + Fam: "dirs", + Ctx: "filecheck.dir_num_of_files", + } + dirSizeChart = module.Chart{ + ID: "dir_size", + Title: "Dir Size", + Units: "bytes", + Fam: "dirs", + Ctx: "filecheck.dir_size", + } +) diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/collect.go b/src/go/collectors/go.d.plugin/modules/filecheck/collect.go new file mode 100644 index 000000000..921846a75 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/collect.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "regexp" + "runtime" + "strings" +) + +func (fc *Filecheck) collect() (map[string]int64, error) { + ms := make(map[string]int64) + + fc.collectFiles(ms) + fc.collectDirs(ms) + + return ms, nil +} + +func hasMeta(path string) bool { + magicChars := `*?[` + if runtime.GOOS != "windows" { + magicChars = `*?[\` + } + return strings.ContainsAny(path, magicChars) +} + +func removeDuplicates(s []string) []string { + set := make(map[string]bool, len(s)) + uniq := s[:0] + for _, v := range s { + if !set[v] { + set[v] = true + uniq = append(uniq, v) + } + } + return uniq +} + +var reSpace = regexp.MustCompile(`\s`) diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/collect_dirs.go b/src/go/collectors/go.d.plugin/modules/filecheck/collect_dirs.go new file mode 100644 index 000000000..69a2e2f5c --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/collect_dirs.go @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +func (fc *Filecheck) collectDirs(ms map[string]int64) { + curTime := time.Now() + if time.Since(fc.lastDiscoveryDirs) >= fc.DiscoveryEvery.Duration() { + fc.lastDiscoveryDirs = curTime + fc.curDirs = fc.discoveryDirs() + fc.updateDirsCharts(fc.curDirs) + } + + for _, path := range fc.curDirs { + fc.collectDir(ms, path, curTime) + } + ms["num_of_dirs"] = int64(len(fc.curDirs)) +} + +func (fc *Filecheck) collectDir(ms map[string]int64, path string, curTime time.Time) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + ms[dirDimID(path, "exists")] = 0 + } else { + ms[dirDimID(path, "exists")] = 1 + } + fc.Debug(err) + return + } + + if !info.IsDir() { + return + } + + ms[dirDimID(path, "exists")] = 1 + ms[dirDimID(path, "mtime_ago")] = int64(curTime.Sub(info.ModTime()).Seconds()) + if num, err := calcDirNumOfFiles(path); err == nil { + ms[dirDimID(path, "num_of_files")] = int64(num) + } + if fc.Dirs.CollectDirSize { + if size, err := calcDirSize(path); err == nil { + ms[dirDimID(path, "size_bytes")] = size + } + } +} + +func (fc *Filecheck) discoveryDirs() (dirs []string) { + for _, path := range fc.Dirs.Include { + if hasMeta(path) { + continue + } + dirs = append(dirs, path) + } + + for _, path := range fc.Dirs.Include { + if !hasMeta(path) { + continue + } + matches, _ := filepath.Glob(path) + for _, v := range matches { + fi, err := os.Lstat(v) + if err == nil && fi.IsDir() { + dirs = append(dirs, v) + } + } + } + return removeDuplicates(dirs) +} + +func (fc *Filecheck) updateDirsCharts(dirs []string) { + set := make(map[string]bool, len(dirs)) + for _, path := range dirs { + set[path] = true + if !fc.collectedDirs[path] { + fc.collectedDirs[path] = true + fc.addDirToCharts(path) + } + } + for path := range fc.collectedDirs { + if !set[path] { + delete(fc.collectedDirs, path) + fc.removeDirFromCharts(path) + } + } +} + +func (fc *Filecheck) addDirToCharts(path string) { + for _, chart := range *fc.Charts() { + if !strings.HasPrefix(chart.ID, "dir_") { + continue + } + + var id string + switch chart.ID { + case dirExistenceChart.ID: + id = dirDimID(path, "exists") + case dirModTimeChart.ID: + id = dirDimID(path, "mtime_ago") + case dirNumOfFilesChart.ID: + id = dirDimID(path, "num_of_files") + case dirSizeChart.ID: + id = dirDimID(path, "size_bytes") + default: + fc.Warningf("add dimension: couldn't dim id for '%s' chart (dir '%s')", chart.ID, path) + continue + } + + dim := &module.Dim{ID: id, Name: reSpace.ReplaceAllString(path, "_")} + + if err := chart.AddDim(dim); err != nil { + fc.Warning(err) + continue + } + chart.MarkNotCreated() + } +} + +func (fc *Filecheck) removeDirFromCharts(path string) { + for _, chart := range *fc.Charts() { + if !strings.HasPrefix(chart.ID, "dir_") { + continue + } + + var id string + switch chart.ID { + case dirExistenceChart.ID: + id = dirDimID(path, "exists") + case dirModTimeChart.ID: + id = dirDimID(path, "mtime_ago") + case dirNumOfFilesChart.ID: + id = dirDimID(path, "num_of_files") + case dirSizeChart.ID: + id = dirDimID(path, "size_bytes") + default: + fc.Warningf("remove dimension: couldn't dim id for '%s' chart (dir '%s')", chart.ID, path) + continue + } + + if err := chart.MarkDimRemove(id, true); err != nil { + fc.Warning(err) + continue + } + chart.MarkNotCreated() + } +} + +func dirDimID(path, metric string) string { + return fmt.Sprintf("dir_%s_%s", reSpace.ReplaceAllString(path, "_"), metric) +} + +func calcDirNumOfFiles(dirpath string) (int, error) { + f, err := os.Open(dirpath) + if err != nil { + return 0, err + } + defer func() { _ = f.Close() }() + // TODO: include dirs? + names, err := f.Readdirnames(-1) + return len(names), err +} + +func calcDirSize(dirpath string) (int64, error) { + var size int64 + err := filepath.Walk(dirpath, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/collect_files.go b/src/go/collectors/go.d.plugin/modules/filecheck/collect_files.go new file mode 100644 index 000000000..afbc4052a --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/collect_files.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +func (fc *Filecheck) collectFiles(ms map[string]int64) { + curTime := time.Now() + if time.Since(fc.lastDiscoveryFiles) >= fc.DiscoveryEvery.Duration() { + fc.lastDiscoveryFiles = curTime + fc.curFiles = fc.discoveryFiles() + fc.updateFilesCharts(fc.curFiles) + } + + for _, path := range fc.curFiles { + fc.collectFile(ms, path, curTime) + } + ms["num_of_files"] = int64(len(fc.curFiles)) +} + +func (fc *Filecheck) collectFile(ms map[string]int64, path string, curTime time.Time) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + ms[fileDimID(path, "exists")] = 0 + } else { + ms[fileDimID(path, "exists")] = 1 + } + fc.Debug(err) + return + } + + if info.IsDir() { + return + } + + ms[fileDimID(path, "exists")] = 1 + ms[fileDimID(path, "size_bytes")] = info.Size() + ms[fileDimID(path, "mtime_ago")] = int64(curTime.Sub(info.ModTime()).Seconds()) +} + +func (fc *Filecheck) discoveryFiles() (files []string) { + for _, path := range fc.Files.Include { + if hasMeta(path) { + continue + } + files = append(files, path) + } + + for _, path := range fc.Files.Include { + if !hasMeta(path) { + continue + } + matches, _ := filepath.Glob(path) + for _, v := range matches { + fi, err := os.Lstat(v) + if err == nil && fi.Mode().IsRegular() { + files = append(files, v) + } + } + } + return removeDuplicates(files) +} + +func (fc *Filecheck) updateFilesCharts(files []string) { + set := make(map[string]bool, len(files)) + for _, path := range files { + set[path] = true + if !fc.collectedFiles[path] { + fc.collectedFiles[path] = true + fc.addFileToCharts(path) + } + } + for path := range fc.collectedFiles { + if !set[path] { + delete(fc.collectedFiles, path) + fc.removeFileFromCharts(path) + } + } +} + +func (fc *Filecheck) addFileToCharts(path string) { + for _, chart := range *fc.Charts() { + if !strings.HasPrefix(chart.ID, "file_") { + continue + } + + var id string + switch chart.ID { + case fileExistenceChart.ID: + id = fileDimID(path, "exists") + case fileModTimeAgoChart.ID: + id = fileDimID(path, "mtime_ago") + case fileSizeChart.ID: + id = fileDimID(path, "size_bytes") + default: + fc.Warningf("add dimension: couldn't dim id for '%s' chart (file '%s')", chart.ID, path) + continue + } + + dim := &module.Dim{ID: id, Name: reSpace.ReplaceAllString(path, "_")} + + if err := chart.AddDim(dim); err != nil { + fc.Warning(err) + continue + } + chart.MarkNotCreated() + } +} + +func (fc *Filecheck) removeFileFromCharts(path string) { + for _, chart := range *fc.Charts() { + if !strings.HasPrefix(chart.ID, "file_") { + continue + } + + var id string + switch chart.ID { + case fileExistenceChart.ID: + id = fileDimID(path, "exists") + case fileModTimeAgoChart.ID: + id = fileDimID(path, "mtime_ago") + case fileSizeChart.ID: + id = fileDimID(path, "size_bytes") + default: + fc.Warningf("remove dimension: couldn't dim id for '%s' chart (file '%s')", chart.ID, path) + continue + } + + if err := chart.MarkDimRemove(id, true); err != nil { + fc.Warning(err) + continue + } + chart.MarkNotCreated() + } +} + +func fileDimID(path, metric string) string { + return fmt.Sprintf("file_%s_%s", reSpace.ReplaceAllString(path, "_"), metric) +} diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/config_schema.json b/src/go/collectors/go.d.plugin/modules/filecheck/config_schema.json new file mode 100644 index 000000000..3e8d75d6c --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/config_schema.json @@ -0,0 +1,121 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Filecheck collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "files": { + "title": "File selector", + "description": "Configuration for monitoring specific files. If left empy, no files will be monitored.", + "type": [ + "object", + "null" + ], + "properties": { + "include": { + "title": "Include", + "description": "Include files that match any of the specified include [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Filepath", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + }, + "exclude": { + "title": "Exclude", + "description": "Exclude files that match any of the specified exclude [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Filepath", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + } + }, + "required": [ + "include" + ] + }, + "collect_dir_size": { + "title": "Collect directory size", + "description": "Enable the collection of directory sizes for each monitored directory. Enabling this option may introduce additional overhead on both Netdata and the host system, particularly if directories contain a large number of subdirectories and files.", + "type": "boolean", + "default": false + }, + "dirs": { + "title": "Directory selector", + "description": "Configuration for monitoring specific directories. If left empy, no directories will be monitored.", + "type": [ + "object", + "null" + ], + "properties": { + "include": { + "title": "Include", + "description": "Include directories that match any of the specified include [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Directory", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + }, + "exclude": { + "title": "Exclude", + "description": "Exclude directories that match any of the specified exclude [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Directory", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + } + }, + "required": [ + "include" + ] + } + }, + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "uiOptions": { + "fullPage": true + }, + "files": { + "ui:help": "The logic for inclusion and exclusion is as follows: `(include1 OR include2) AND !(exclude1 OR exclude2)`.", + "ui:collapsible": true + }, + "dirs": { + "ui:help": "The logic for inclusion and exclusion is as follows: `(include1 OR include2) AND !(exclude1 OR exclude2)`.", + "ui:collapsible": true + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/filecheck.go b/src/go/collectors/go.d.plugin/modules/filecheck/filecheck.go new file mode 100644 index 000000000..7d496fb0b --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/filecheck.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + _ "embed" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("filecheck", module.Creator{ + JobConfigSchema: configSchema, + Defaults: module.Defaults{ + UpdateEvery: 10, + }, + Create: func() module.Module { return New() }, + }) +} + +func New() *Filecheck { + return &Filecheck{ + Config: Config{ + DiscoveryEvery: web.Duration(time.Second * 30), + Files: filesConfig{}, + Dirs: dirsConfig{ + CollectDirSize: true, + }, + }, + collectedFiles: make(map[string]bool), + collectedDirs: make(map[string]bool), + } +} + +type ( + Config struct { + UpdateEvery int `yaml:"update_every" json:"update_every"` + DiscoveryEvery web.Duration `yaml:"discovery_every" json:"discovery_every"` + Files filesConfig `yaml:"files" json:"files"` + Dirs dirsConfig `yaml:"dirs" json:"dirs"` + } + filesConfig struct { + Include []string `yaml:"include" json:"include"` + Exclude []string `yaml:"exclude" json:"exclude"` + } + dirsConfig struct { + Include []string `yaml:"include" json:"include"` + Exclude []string `yaml:"exclude" json:"exclude"` + CollectDirSize bool `yaml:"collect_dir_size" json:"collect_dir_size"` + } +) + +type Filecheck struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + + lastDiscoveryFiles time.Time + curFiles []string + collectedFiles map[string]bool + + lastDiscoveryDirs time.Time + curDirs []string + collectedDirs map[string]bool +} + +func (fc *Filecheck) Configuration() any { + return fc.Config +} + +func (fc *Filecheck) Init() error { + err := fc.validateConfig() + if err != nil { + fc.Errorf("error on validating config: %v", err) + return err + } + + charts, err := fc.initCharts() + if err != nil { + fc.Errorf("error on charts initialization: %v", err) + return err + } + fc.charts = charts + + fc.Debugf("monitored files: %v", fc.Files.Include) + fc.Debugf("monitored dirs: %v", fc.Dirs.Include) + + return nil +} + +func (fc *Filecheck) Check() error { + return nil +} + +func (fc *Filecheck) Charts() *module.Charts { + return fc.charts +} + +func (fc *Filecheck) Collect() map[string]int64 { + ms, err := fc.collect() + if err != nil { + fc.Error(err) + } + + if len(ms) == 0 { + return nil + } + return ms +} + +func (fc *Filecheck) Cleanup() { +} diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/filecheck_test.go b/src/go/collectors/go.d.plugin/modules/filecheck/filecheck_test.go new file mode 100644 index 000000000..3a9121fc8 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/filecheck_test.go @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "os" + "strings" + "testing" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + } { + require.NotNil(t, data, name) + } +} + +func TestFilecheck_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Filecheck{}, dataConfigJSON, dataConfigYAML) +} + +func TestFilecheck_Cleanup(t *testing.T) { + assert.NotPanics(t, New().Cleanup) +} + +func TestFilecheck_Init(t *testing.T) { + tests := map[string]struct { + config Config + wantNumOfCharts int + wantFail bool + }{ + "default": { + config: New().Config, + wantFail: true, + }, + "empty files->include and dirs->include": { + config: Config{ + Files: filesConfig{}, + Dirs: dirsConfig{}, + }, + wantFail: true, + }, + "files->include and dirs->include": { + config: Config{ + Files: filesConfig{ + Include: []string{ + "/path/to/file1", + "/path/to/file2", + }, + }, + Dirs: dirsConfig{ + Include: []string{ + "/path/to/dir1", + "/path/to/dir2", + }, + CollectDirSize: true, + }, + }, + wantNumOfCharts: len(fileCharts) + len(dirCharts), + }, + "only files->include": { + config: Config{ + Files: filesConfig{ + Include: []string{ + "/path/to/file1", + "/path/to/file2", + }, + }, + }, + wantNumOfCharts: len(fileCharts), + }, + "only dirs->include": { + config: Config{ + Dirs: dirsConfig{ + Include: []string{ + "/path/to/dir1", + "/path/to/dir2", + }, + CollectDirSize: true, + }, + }, + wantNumOfCharts: len(dirCharts), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + fc := New() + fc.Config = test.config + + if test.wantFail { + assert.Error(t, fc.Init()) + } else { + require.NoError(t, fc.Init()) + assert.Equal(t, test.wantNumOfCharts, len(*fc.Charts())) + } + }) + } +} + +func TestFilecheck_Check(t *testing.T) { + tests := map[string]struct { + prepare func() *Filecheck + }{ + "collect files": {prepare: prepareFilecheckFiles}, + "collect files filepath pattern": {prepare: prepareFilecheckGlobFiles}, + "collect only non existent files": {prepare: prepareFilecheckNonExistentFiles}, + "collect dirs": {prepare: prepareFilecheckDirs}, + "collect dirs filepath pattern": {prepare: prepareFilecheckGlobDirs}, + "collect only non existent dirs": {prepare: prepareFilecheckNonExistentDirs}, + "collect files and dirs": {prepare: prepareFilecheckFilesDirs}, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + fc := test.prepare() + require.NoError(t, fc.Init()) + + assert.NoError(t, fc.Check()) + }) + } +} + +func TestFilecheck_Collect(t *testing.T) { + // TODO: should use TEMP dir and create files/dirs dynamically during a test case + tests := map[string]struct { + prepare func() *Filecheck + wantCollected map[string]int64 + }{ + "collect files": { + prepare: prepareFilecheckFiles, + wantCollected: map[string]int64{ + "file_testdata/empty_file.log_exists": 1, + "file_testdata/empty_file.log_mtime_ago": 5081, + "file_testdata/empty_file.log_size_bytes": 0, + "file_testdata/file.log_exists": 1, + "file_testdata/file.log_mtime_ago": 4161, + "file_testdata/file.log_size_bytes": 5707, + "file_testdata/non_existent_file.log_exists": 0, + "num_of_files": 3, + "num_of_dirs": 0, + }, + }, + "collect files filepath pattern": { + prepare: prepareFilecheckGlobFiles, + wantCollected: map[string]int64{ + "file_testdata/empty_file.log_exists": 1, + "file_testdata/empty_file.log_mtime_ago": 5081, + "file_testdata/empty_file.log_size_bytes": 0, + "file_testdata/file.log_exists": 1, + "file_testdata/file.log_mtime_ago": 4161, + "file_testdata/file.log_size_bytes": 5707, + "num_of_files": 2, + "num_of_dirs": 0, + }, + }, + "collect only non existent files": { + prepare: prepareFilecheckNonExistentFiles, + wantCollected: map[string]int64{ + "file_testdata/non_existent_file.log_exists": 0, + "num_of_files": 1, + "num_of_dirs": 0, + }, + }, + "collect dirs": { + prepare: prepareFilecheckDirs, + wantCollected: map[string]int64{ + "dir_testdata/dir_exists": 1, + "dir_testdata/dir_mtime_ago": 4087, + "dir_testdata/dir_num_of_files": 3, + "dir_testdata/dir_size_bytes": 8160, + "dir_testdata/non_existent_dir_exists": 0, + "num_of_files": 0, + "num_of_dirs": 2, + }, + }, + "collect dirs filepath pattern": { + prepare: prepareFilecheckGlobDirs, + wantCollected: map[string]int64{ + "dir_testdata/dir_exists": 1, + "dir_testdata/dir_mtime_ago": 4087, + "dir_testdata/dir_num_of_files": 3, + "dir_testdata/dir_size_bytes": 8160, + "dir_testdata/non_existent_dir_exists": 0, + "num_of_files": 0, + "num_of_dirs": 2, + }, + }, + "collect dirs w/o size": { + prepare: prepareFilecheckDirsWithoutSize, + wantCollected: map[string]int64{ + "dir_testdata/dir_exists": 1, + "dir_testdata/dir_mtime_ago": 4087, + "dir_testdata/dir_num_of_files": 3, + "dir_testdata/non_existent_dir_exists": 0, + "num_of_files": 0, + "num_of_dirs": 2, + }, + }, + "collect only non existent dirs": { + prepare: prepareFilecheckNonExistentDirs, + wantCollected: map[string]int64{ + "dir_testdata/non_existent_dir_exists": 0, + "num_of_files": 0, + "num_of_dirs": 1, + }, + }, + "collect files and dirs": { + prepare: prepareFilecheckFilesDirs, + wantCollected: map[string]int64{ + "dir_testdata/dir_exists": 1, + "dir_testdata/dir_mtime_ago": 4120, + "dir_testdata/dir_num_of_files": 3, + "dir_testdata/dir_size_bytes": 8160, + "dir_testdata/non_existent_dir_exists": 0, + "file_testdata/empty_file.log_exists": 1, + "file_testdata/empty_file.log_mtime_ago": 5176, + "file_testdata/empty_file.log_size_bytes": 0, + "file_testdata/file.log_exists": 1, + "file_testdata/file.log_mtime_ago": 4256, + "file_testdata/file.log_size_bytes": 5707, + "file_testdata/non_existent_file.log_exists": 0, + "num_of_files": 3, + "num_of_dirs": 2, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + fc := test.prepare() + require.NoError(t, fc.Init()) + + collected := fc.Collect() + + copyModTime(test.wantCollected, collected) + assert.Equal(t, test.wantCollected, collected) + ensureCollectedHasAllChartsDimsVarsIDs(t, fc, collected) + }) + } +} + +func ensureCollectedHasAllChartsDimsVarsIDs(t *testing.T, fc *Filecheck, collected map[string]int64) { + // TODO: check other charts + for _, chart := range *fc.Charts() { + if chart.Obsolete { + continue + } + switch chart.ID { + case fileExistenceChart.ID, dirExistenceChart.ID: + for _, dim := range chart.Dims { + _, ok := collected[dim.ID] + assert.Truef(t, ok, "collected metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID) + } + for _, v := range chart.Vars { + _, ok := collected[v.ID] + assert.Truef(t, ok, "collected metrics has no data for var '%s' chart '%s'", v.ID, chart.ID) + } + } + } +} + +func prepareFilecheckFiles() *Filecheck { + fc := New() + fc.Config.Files.Include = []string{ + "testdata/empty_file.log", + "testdata/file.log", + "testdata/non_existent_file.log", + } + return fc +} + +func prepareFilecheckGlobFiles() *Filecheck { + fc := New() + fc.Config.Files.Include = []string{ + "testdata/*.log", + } + return fc +} + +func prepareFilecheckNonExistentFiles() *Filecheck { + fc := New() + fc.Config.Files.Include = []string{ + "testdata/non_existent_file.log", + } + return fc +} + +func prepareFilecheckDirs() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/dir", + "testdata/non_existent_dir", + } + return fc +} + +func prepareFilecheckGlobDirs() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/*ir", + "testdata/non_existent_dir", + } + return fc +} + +func prepareFilecheckDirsWithoutSize() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/dir", + "testdata/non_existent_dir", + } + fc.Config.Dirs.CollectDirSize = false + return fc +} + +func prepareFilecheckNonExistentDirs() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/non_existent_dir", + } + return fc +} + +func prepareFilecheckFilesDirs() *Filecheck { + fc := New() + fc.Config.Files.Include = []string{ + "testdata/empty_file.log", + "testdata/file.log", + "testdata/non_existent_file.log", + } + fc.Config.Dirs.Include = []string{ + "testdata/dir", + "testdata/non_existent_dir", + } + return fc +} + +func copyModTime(dst, src map[string]int64) { + if src == nil || dst == nil { + return + } + for key := range src { + if strings.Contains(key, "mtime") { + dst[key] = src[key] + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/init.go b/src/go/collectors/go.d.plugin/modules/filecheck/init.go new file mode 100644 index 000000000..af64620e8 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/init.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "errors" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +func (fc *Filecheck) validateConfig() error { + if len(fc.Files.Include) == 0 && len(fc.Dirs.Include) == 0 { + return errors.New("both 'files->include' and 'dirs->include' are empty") + } + return nil +} + +func (fc *Filecheck) initCharts() (*module.Charts, error) { + charts := &module.Charts{} + + if len(fc.Files.Include) > 0 { + if err := charts.Add(*fileCharts.Copy()...); err != nil { + return nil, err + } + } + + if len(fc.Dirs.Include) > 0 { + if err := charts.Add(*dirCharts.Copy()...); err != nil { + return nil, err + } + if !fc.Dirs.CollectDirSize { + if err := charts.Remove(dirSizeChart.ID); err != nil { + return nil, err + } + } + } + + if len(*charts) == 0 { + return nil, errors.New("empty charts") + } + return charts, nil +} diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/integrations/files_and_directories.md b/src/go/collectors/go.d.plugin/modules/filecheck/integrations/files_and_directories.md new file mode 100644 index 000000000..fe203271b --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/integrations/files_and_directories.md @@ -0,0 +1,226 @@ +<!--startmeta +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/filecheck/README.md" +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/filecheck/metadata.yaml" +sidebar_label: "Files and directories" +learn_status: "Published" +learn_rel_path: "Collecting Metrics/Linux Systems" +most_popular: False +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" +endmeta--> + +# Files and directories + + +<img src="https://netdata.cloud/img/filesystem.svg" width="150"/> + + +Plugin: go.d.plugin +Module: filecheck + +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" /> + +## Overview + +This collector monitors files and directories. + + + + +This collector is supported on all platforms. + +This collector supports collecting metrics from multiple instances of this integration, including remote instances. + +This collector requires the DAC_READ_SEARCH capability, but it is set automatically during installation, so no manual configuration is needed. + + +### Default Behavior + +#### Auto-Detection + +This integration doesn't support auto-detection. + +#### Limits + +The default configuration for this integration does not impose any limits on data collection. + +#### Performance Impact + +The default configuration for this integration is not expected to impose a significant performance impact on the system. + + +## Metrics + +Metrics grouped by *scope*. + +The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels. + + + +### Per Files and directories instance + +TBD + +This scope has no labels. + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| filecheck.file_existence | a dimension per file | boolean | +| filecheck.file_mtime_ago | a dimension per file | seconds | +| filecheck.file_size | a dimension per file | bytes | +| filecheck.dir_existence | a dimension per directory | boolean | +| filecheck.dir_mtime_ago | a dimension per directory | seconds | +| filecheck.dir_num_of_files | a dimension per directory | files | +| filecheck.dir_size | a dimension per directory | bytes | + + + +## Alerts + +There are no alerts configured by default for this integration. + + +## Setup + +### Prerequisites + +No action required. + +### Configuration + +#### File + +The configuration file name for this integration is `go.d/filecheck.conf`. + + +You can edit the configuration file using the `edit-config` script from the +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration.md#the-netdata-config-directory). + +```bash +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata +sudo ./edit-config go.d/filecheck.conf +``` +#### Options + +The following options can be defined globally: update_every, autodetection_retry. + + +<details><summary>Config options</summary> + +| Name | Description | Default | Required | +|:----|:-----------|:-------|:--------:| +| update_every | Data collection frequency. | 10 | no | +| autodetection_retry | Recheck interval in seconds. Zero means no recheck will be scheduled. | 0 | no | +| files | Files matching the selector will be monitored. | | yes | +| dirs | List of directories to monitor. | | yes | +| discovery_every | Files and directories discovery interval. | 60 | no | + +##### files + +Files matching the selector will be monitored. + +- Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) +- Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) +- Syntax: + +```yaml +files: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 +``` + + +##### dirs + +Directories matching the selector will be monitored. + +- Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) +- Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) +- Syntax: + +```yaml +dirs: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 +``` + + +</details> + +#### Examples + +##### Files + +Files monitoring example configuration. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: files_example + files: + include: + - '/path/to/file1' + - '/path/to/file2' + - '/path/to/*.log' + +``` +</details> + +##### Directories + +Directories monitoring example configuration. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: files_example + dirs: + collect_dir_size: no + include: + - '/path/to/dir1' + - '/path/to/dir2' + - '/path/to/dir3*' + +``` +</details> + + + +## Troubleshooting + +### Debug Mode + +To troubleshoot issues with the `filecheck` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m filecheck + ``` + + diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/metadata.yaml b/src/go/collectors/go.d.plugin/modules/filecheck/metadata.yaml new file mode 100644 index 000000000..57a121ec1 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/metadata.yaml @@ -0,0 +1,188 @@ +plugin_name: go.d.plugin +modules: + - meta: + id: collector-go.d.plugin-filecheck + plugin_name: go.d.plugin + module_name: filecheck + monitored_instance: + name: Files and directories + link: "" + icon_filename: filesystem.svg + categories: + - data-collection.linux-systems + keywords: + - files + - directories + related_resources: + integrations: + list: [] + info_provided_to_referring_integrations: + description: "" + most_popular: false + overview: + data_collection: + metrics_description: | + This collector monitors files and directories. + method_description: "" + supported_platforms: + include: [] + exclude: [] + multi_instance: true + additional_permissions: + description: | + This collector requires the DAC_READ_SEARCH capability, but it is set automatically during installation, so no manual configuration is needed. + default_behavior: + auto_detection: + description: "" + limits: + description: "" + performance_impact: + description: "" + setup: + prerequisites: + list: [] + configuration: + file: + name: go.d/filecheck.conf + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 10 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: files + description: Files matching the selector will be monitored. + default_value: "" + required: true + detailed_description: | + Files matching the selector will be monitored. + + - Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) + - Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) + - Syntax: + + ```yaml + files: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 + ``` + - name: dirs + description: List of directories to monitor. + default_value: "" + required: true + detailed_description: | + Directories matching the selector will be monitored. + + - Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) + - Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) + - Syntax: + + ```yaml + dirs: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 + ``` + - name: discovery_every + description: Files and directories discovery interval. + default_value: 60 + required: false + examples: + folding: + title: Config + enabled: true + list: + - name: Files + description: Files monitoring example configuration. + config: | + jobs: + - name: files_example + files: + include: + - '/path/to/file1' + - '/path/to/file2' + - '/path/to/*.log' + - name: Directories + description: Directories monitoring example configuration. + config: | + jobs: + - name: files_example + dirs: + collect_dir_size: no + include: + - '/path/to/dir1' + - '/path/to/dir2' + - '/path/to/dir3*' + troubleshooting: + problems: + list: [] + alerts: [] + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: global + description: TBD + labels: [] + metrics: + - name: filecheck.file_existence + description: 'File Existence (0: not exists, 1: exists)' + unit: boolean + chart_type: line + dimensions: + - name: a dimension per file + - name: filecheck.file_mtime_ago + description: File Time Since the Last Modification + unit: seconds + chart_type: line + dimensions: + - name: a dimension per file + - name: filecheck.file_size + description: File Size + unit: bytes + chart_type: line + dimensions: + - name: a dimension per file + - name: filecheck.dir_existence + description: 'Dir Existence (0: not exists, 1: exists)' + unit: boolean + chart_type: line + dimensions: + - name: a dimension per directory + - name: filecheck.dir_mtime_ago + description: Dir Time Since the Last Modification + unit: seconds + chart_type: line + dimensions: + - name: a dimension per directory + - name: filecheck.dir_num_of_files + description: Dir Number of Files + unit: files + chart_type: line + dimensions: + - name: a dimension per directory + - name: filecheck.dir_size + description: Dir Size + unit: bytes + chart_type: line + dimensions: + - name: a dimension per directory diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/testdata/config.json b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/config.json new file mode 100644 index 000000000..93d286f84 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/config.json @@ -0,0 +1,21 @@ +{ + "update_every": 123, + "discovery_every": 123.123, + "files": { + "include": [ + "ok" + ], + "exclude": [ + "ok" + ] + }, + "dirs": { + "include": [ + "ok" + ], + "exclude": [ + "ok" + ], + "collect_dir_size": true + } +} diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/testdata/config.yaml b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/config.yaml new file mode 100644 index 000000000..494a21855 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/config.yaml @@ -0,0 +1,13 @@ +update_every: 123 +discovery_every: 123.123 +files: + include: + - "ok" + exclude: + - "ok" +dirs: + include: + - "ok" + exclude: + - "ok" + collect_dir_size: yes diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/empty_file.log b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/empty_file.log new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/empty_file.log diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/file.log b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/file.log new file mode 100644 index 000000000..c1c152a81 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/file.log @@ -0,0 +1,61 @@ +198.51.100.1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 301 4715 4113 174 465 https TLSv1.2 ECDHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 1130 1202 409 450 https TLSv1 DHE-RSA-AES256-SHA light beer +198.51.100.1:83 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 201 4020 1217 492 135 https TLSv1.2 PSK-RC4-SHA light wine +test.example.org:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 401 3784 2349 266 63 http TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:83 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 201 2149 3834 178 197 https TLSv1.1 AES256-SHA dark wine +198.51.100.1:80 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 200 1442 4125 23 197 https TLSv1.3 DHE-RSA-AES256-SHA light wine +test.example.com:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 300 4134 3965 259 296 https TLSv1.3 PSK-RC4-SHA dark wine +test.example.com:84 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/1.1" 401 1224 3352 135 468 http SSLv2 PSK-RC4-SHA light wine +localhost:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/2.0" 200 2504 4754 58 371 http TLSv1.1 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/1.1" 200 4898 2787 398 476 http SSLv2 DHE-RSA-AES256-SHA dark beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 100 4957 1848 324 158 https TLSv1.2 AES256-SHA dark wine +test.example.org:80 localhost - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/2" 301 1752 1717 75 317 https SSLv3 PSK-RC4-SHA dark wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 301 3799 4120 71 17 http TLSv1.3 ECDHE-RSA-AES256-SHA dark beer +198.51.100.1:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 1870 3945 392 323 http TLSv1.1 PSK-RC4-SHA light beer +test.example.com:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2.0" 200 1261 3535 52 271 https TLSv1.1 DHE-RSA-AES256-SHA dark wine +test.example.com:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 101 3228 3545 476 168 http TLSv1.1 AES256-SHA light beer +test.example.com:80 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 300 4731 1574 362 184 https SSLv2 ECDHE-RSA-AES256-SHA light wine +198.51.100.1:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/1.1" 300 4868 1803 23 388 https TLSv1.3 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/1.1" 100 3744 3546 296 437 http SSLv2 DHE-RSA-AES256-SHA light beer +test.example.org:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 401 4858 1493 151 240 http SSLv2 AES256-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2.0" 300 1367 4284 45 443 https TLSv1.1 AES256-SHA light beer +localhost:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 100 4392 4982 143 110 http SSLv3 AES256-SHA light beer +2001:db8:1ce::1:84 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/1.1" 101 4606 3311 410 273 https TLSv1 PSK-RC4-SHA dark beer +198.51.100.1:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.net HTTP/2.0" 100 1163 1526 10 186 https SSLv2 AES256-SHA light beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 301 3262 3789 144 124 https TLSv1.3 DHE-RSA-AES256-SHA light wine +198.51.100.1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 400 1365 1447 325 186 http TLSv1.2 PSK-RC4-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 4546 4409 295 153 http SSLv3 ECDHE-RSA-AES256-SHA light beer +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 300 2297 3318 139 227 https TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 100 4671 4285 371 7 https SSLv3 ECDHE-RSA-AES256-SHA dark beer +test.example.org:83 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 400 3651 1135 172 159 https TLSv1.1 DHE-RSA-AES256-SHA light beer +localhost:82 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 3958 3959 350 121 https SSLv2 DHE-RSA-AES256-SHA dark beer +localhost:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 200 1652 3813 190 11 https SSLv3 AES256-SHA dark wine +test.example.org:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 101 1228 2344 251 366 https TLSv1 ECDHE-RSA-AES256-SHA light beer +test.example.org:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 200 1860 3118 187 419 https TLSv1 PSK-RC4-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +localhost:82 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/1.1" 401 4518 3837 18 219 http TLSv1.3 DHE-RSA-AES256-SHA dark beer +localhost:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.net HTTP/2" 201 2108 2472 257 470 http TLSv1.1 PSK-RC4-SHA dark beer +2001:db8:1ce::1:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2" 101 2020 1076 262 106 https TLSv1.3 PSK-RC4-SHA light wine +localhost:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/1.1" 100 4815 3052 49 322 https TLSv1.3 DHE-RSA-AES256-SHA light beer +2001:db8:1ce::1:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 300 1642 4001 421 194 https TLSv1 PSK-RC4-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2" 201 3805 2597 25 187 http TLSv1.1 AES256-SHA dark wine +2001:db8:1ce::1:84 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/2.0" 301 3435 1760 474 318 https TLSv1.2 ECDHE-RSA-AES256-SHA light wine +localhost:84 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2.0" 101 1911 4082 356 301 https TLSv1 DHE-RSA-AES256-SHA light beer +2001:db8:1ce::1:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2" 100 2536 1664 115 474 http SSLv3 PSK-RC4-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/1.1" 401 3757 3987 441 469 http SSLv2 ECDHE-RSA-AES256-SHA dark wine +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:83 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 400 1221 4244 232 421 https TLSv1.1 ECDHE-RSA-AES256-SHA dark wine +localhost:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 101 2001 2405 6 140 http TLSv1 DHE-RSA-AES256-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +198.51.100.1:81 localhost - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/2.0" 400 4442 4396 64 49 https TLSv1.1 AES256-SHA light beer +2001:db8:1ce::1:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 401 1461 4623 46 47 https TLSv1.3 ECDHE-RSA-AES256-SHA light beer +Unmatched! The rat the cat the dog chased killed ate the malt!
\ No newline at end of file diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/subdir/empty_file.log b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/subdir/empty_file.log new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/dir/subdir/empty_file.log diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/testdata/empty_file.log b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/empty_file.log new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/empty_file.log diff --git a/src/go/collectors/go.d.plugin/modules/filecheck/testdata/file.log b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/file.log new file mode 100644 index 000000000..e0db68517 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/filecheck/testdata/file.log @@ -0,0 +1,42 @@ +198.51.100.1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 301 4715 4113 174 465 https TLSv1.2 ECDHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 1130 1202 409 450 https TLSv1 DHE-RSA-AES256-SHA light beer +198.51.100.1:83 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 201 4020 1217 492 135 https TLSv1.2 PSK-RC4-SHA light wine +test.example.org:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 401 3784 2349 266 63 http TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:83 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 201 2149 3834 178 197 https TLSv1.1 AES256-SHA dark wine +198.51.100.1:80 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 200 1442 4125 23 197 https TLSv1.3 DHE-RSA-AES256-SHA light wine +test.example.com:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 300 4134 3965 259 296 https TLSv1.3 PSK-RC4-SHA dark wine +test.example.com:84 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/1.1" 401 1224 3352 135 468 http SSLv2 PSK-RC4-SHA light wine +localhost:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/2.0" 200 2504 4754 58 371 http TLSv1.1 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/1.1" 200 4898 2787 398 476 http SSLv2 DHE-RSA-AES256-SHA dark beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 100 4957 1848 324 158 https TLSv1.2 AES256-SHA dark wine +test.example.org:80 localhost - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/2" 301 1752 1717 75 317 https SSLv3 PSK-RC4-SHA dark wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 301 3799 4120 71 17 http TLSv1.3 ECDHE-RSA-AES256-SHA dark beer +198.51.100.1:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 1870 3945 392 323 http TLSv1.1 PSK-RC4-SHA light beer +test.example.com:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2.0" 200 1261 3535 52 271 https TLSv1.1 DHE-RSA-AES256-SHA dark wine +test.example.com:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 101 3228 3545 476 168 http TLSv1.1 AES256-SHA light beer +test.example.com:80 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 300 4731 1574 362 184 https SSLv2 ECDHE-RSA-AES256-SHA light wine +198.51.100.1:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/1.1" 300 4868 1803 23 388 https TLSv1.3 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/1.1" 100 3744 3546 296 437 http SSLv2 DHE-RSA-AES256-SHA light beer +test.example.org:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 401 4858 1493 151 240 http SSLv2 AES256-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2.0" 300 1367 4284 45 443 https TLSv1.1 AES256-SHA light beer +localhost:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 100 4392 4982 143 110 http SSLv3 AES256-SHA light beer +2001:db8:1ce::1:84 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/1.1" 101 4606 3311 410 273 https TLSv1 PSK-RC4-SHA dark beer +198.51.100.1:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.net HTTP/2.0" 100 1163 1526 10 186 https SSLv2 AES256-SHA light beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 301 3262 3789 144 124 https TLSv1.3 DHE-RSA-AES256-SHA light wine +198.51.100.1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 400 1365 1447 325 186 http TLSv1.2 PSK-RC4-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 4546 4409 295 153 http SSLv3 ECDHE-RSA-AES256-SHA light beer +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 300 2297 3318 139 227 https TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 100 4671 4285 371 7 https SSLv3 ECDHE-RSA-AES256-SHA dark beer +test.example.org:83 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 400 3651 1135 172 159 https TLSv1.1 DHE-RSA-AES256-SHA light beer +localhost:82 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 3958 3959 350 121 https SSLv2 DHE-RSA-AES256-SHA dark beer +localhost:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 200 1652 3813 190 11 https SSLv3 AES256-SHA dark wine +test.example.org:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 101 1228 2344 251 366 https TLSv1 ECDHE-RSA-AES256-SHA light beer +test.example.org:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 200 1860 3118 187 419 https TLSv1 PSK-RC4-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt!
\ No newline at end of file |