From f09848204fa5283d21ea43e262ee41aa578e1808 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 26 Aug 2024 10:15:24 +0200 Subject: Merging upstream version 1.47.0. Signed-off-by: Daniel Baumann --- src/go/plugin/go.d/modules/filecheck/README.md | 1 + src/go/plugin/go.d/modules/filecheck/cache.go | 28 ++ src/go/plugin/go.d/modules/filecheck/charts.go | 266 ++++++++++++++++ src/go/plugin/go.d/modules/filecheck/collect.go | 40 +++ .../plugin/go.d/modules/filecheck/collect_dirs.go | 91 ++++++ .../plugin/go.d/modules/filecheck/collect_files.go | 59 ++++ .../go.d/modules/filecheck/config_schema.json | 164 ++++++++++ src/go/plugin/go.d/modules/filecheck/discover.go | 43 +++ src/go/plugin/go.d/modules/filecheck/filecheck.go | 128 ++++++++ .../go.d/modules/filecheck/filecheck_test.go | 350 +++++++++++++++++++++ src/go/plugin/go.d/modules/filecheck/init.go | 38 +++ .../integrations/files_and_directories.md | 280 +++++++++++++++++ src/go/plugin/go.d/modules/filecheck/metadata.yaml | 198 ++++++++++++ .../go.d/modules/filecheck/testdata/config.json | 21 ++ .../go.d/modules/filecheck/testdata/config.yaml | 13 + .../modules/filecheck/testdata/dir/empty_file.log | 0 .../go.d/modules/filecheck/testdata/dir/file.log | 61 ++++ .../filecheck/testdata/dir/subdir/empty_file.log | 0 .../go.d/modules/filecheck/testdata/empty_file.log | 0 .../go.d/modules/filecheck/testdata/file.log | 42 +++ 20 files changed, 1823 insertions(+) create mode 120000 src/go/plugin/go.d/modules/filecheck/README.md create mode 100644 src/go/plugin/go.d/modules/filecheck/cache.go create mode 100644 src/go/plugin/go.d/modules/filecheck/charts.go create mode 100644 src/go/plugin/go.d/modules/filecheck/collect.go create mode 100644 src/go/plugin/go.d/modules/filecheck/collect_dirs.go create mode 100644 src/go/plugin/go.d/modules/filecheck/collect_files.go create mode 100644 src/go/plugin/go.d/modules/filecheck/config_schema.json create mode 100644 src/go/plugin/go.d/modules/filecheck/discover.go create mode 100644 src/go/plugin/go.d/modules/filecheck/filecheck.go create mode 100644 src/go/plugin/go.d/modules/filecheck/filecheck_test.go create mode 100644 src/go/plugin/go.d/modules/filecheck/init.go create mode 100644 src/go/plugin/go.d/modules/filecheck/integrations/files_and_directories.md create mode 100644 src/go/plugin/go.d/modules/filecheck/metadata.yaml create mode 100644 src/go/plugin/go.d/modules/filecheck/testdata/config.json create mode 100644 src/go/plugin/go.d/modules/filecheck/testdata/config.yaml create mode 100644 src/go/plugin/go.d/modules/filecheck/testdata/dir/empty_file.log create mode 100644 src/go/plugin/go.d/modules/filecheck/testdata/dir/file.log create mode 100644 src/go/plugin/go.d/modules/filecheck/testdata/dir/subdir/empty_file.log create mode 100644 src/go/plugin/go.d/modules/filecheck/testdata/empty_file.log create mode 100644 src/go/plugin/go.d/modules/filecheck/testdata/file.log (limited to 'src/go/plugin/go.d/modules/filecheck') diff --git a/src/go/plugin/go.d/modules/filecheck/README.md b/src/go/plugin/go.d/modules/filecheck/README.md new file mode 120000 index 00000000..24dc78d8 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/README.md @@ -0,0 +1 @@ +integrations/files_and_directories.md \ No newline at end of file diff --git a/src/go/plugin/go.d/modules/filecheck/cache.go b/src/go/plugin/go.d/modules/filecheck/cache.go new file mode 100644 index 00000000..1acd6f82 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/cache.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +func newSeenItems() *seenItems { + return &seenItems{ + items: make(map[string]*seenItem), + } +} + +type ( + seenItems struct { + items map[string]*seenItem + } + seenItem struct { + hasExistenceCharts bool + hasOtherCharts bool + } +) + +func (c *seenItems) getp(path string) *seenItem { + item, ok := c.items[path] + if !ok { + item = &seenItem{} + c.items[path] = item + } + return item +} diff --git a/src/go/plugin/go.d/modules/filecheck/charts.go b/src/go/plugin/go.d/modules/filecheck/charts.go new file mode 100644 index 00000000..6d00463a --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/charts.go @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "fmt" + "strings" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" +) + +const ( + prioFileExistenceStatus = module.Priority + iota + prioFileModificationTimeAgo + prioFileSize + + prioDirExistenceStatus + prioDirModificationTimeAgo + prioDirSize + prioDirFilesCount +) + +var ( + fileExistenceStatusChartTmpl = module.Chart{ + ID: "file_%s_existence_status", + Title: "File existence", + Units: "status", + Fam: "file existence", + Ctx: "filecheck.file_existence_status", + Priority: prioFileExistenceStatus, + Dims: module.Dims{ + {ID: "file_%s_existence_status_exist", Name: "exist"}, + {ID: "file_%s_existence_status_not_exist", Name: "not_exist"}, + }, + } + + fileModificationTimeAgoChartTmpl = module.Chart{ + ID: "file_%s_modification_time_ago", + Title: "File time since the last modification", + Units: "seconds", + Fam: "file mtime", + Ctx: "filecheck.file_modification_time_ago", + Priority: prioFileModificationTimeAgo, + Dims: module.Dims{ + {ID: "file_%s_mtime_ago", Name: "mtime_ago"}, + }, + } + fileSizeChartTmpl = module.Chart{ + ID: "file_%s_size", + Title: "File size", + Units: "bytes", + Fam: "file size", + Ctx: "filecheck.file_size_bytes", + Priority: prioFileSize, + Dims: module.Dims{ + {ID: "file_%s_size_bytes", Name: "size"}, + }, + } +) + +var ( + dirExistenceStatusChartTmpl = module.Chart{ + ID: "dir_%s_existence_status", + Title: "Directory existence", + Units: "status", + Fam: "dir existence", + Ctx: "filecheck.dir_existence_status", + Priority: prioDirExistenceStatus, + Dims: module.Dims{ + {ID: "dir_%s_existence_status_exist", Name: "exist"}, + {ID: "dir_%s_existence_status_not_exist", Name: "not_exist"}, + }, + } + + dirModificationTimeAgoChartTmpl = module.Chart{ + ID: "dir_%s_modification_time_ago", + Title: "Directory time since the last modification", + Units: "seconds", + Fam: "dir mtime", + Ctx: "filecheck.dir_modification_time_ago", + Priority: prioDirModificationTimeAgo, + Dims: module.Dims{ + {ID: "dir_%s_mtime_ago", Name: "mtime_ago"}, + }, + } + dirSizeChartTmpl = module.Chart{ + ID: "dir_%s_size", + Title: "Directory size", + Units: "bytes", + Fam: "dir size", + Ctx: "filecheck.dir_size_bytes", + Priority: prioDirSize, + Dims: module.Dims{ + {ID: "dir_%s_size_bytes", Name: "size"}, + }, + } + dirFilesCountChartTmpl = module.Chart{ + ID: "dir_%s_files_count", + Title: "Directory files count", + Units: "files", + Fam: "dir files", + Ctx: "filecheck.dir_files_count", + Priority: prioDirFilesCount, + Dims: module.Dims{ + {ID: "dir_%s_files_count", Name: "files"}, + }, + } +) + +func (f *Filecheck) updateFileCharts(infos []*statInfo) { + seen := make(map[string]bool) + + for _, info := range infos { + seen[info.path] = true + + sf := f.seenFiles.getp(info.path) + + if !sf.hasExistenceCharts { + sf.hasExistenceCharts = true + f.addFileCharts(info.path, + fileExistenceStatusChartTmpl.Copy(), + ) + } + + if !sf.hasOtherCharts && info.fi != nil { + sf.hasOtherCharts = true + f.addFileCharts(info.path, + fileModificationTimeAgoChartTmpl.Copy(), + fileSizeChartTmpl.Copy(), + ) + + } else if sf.hasOtherCharts && info.fi == nil { + sf.hasOtherCharts = false + f.removeFileOtherCharts(info.path) + } + } + + for path := range f.seenFiles.items { + if !seen[path] { + delete(f.seenFiles.items, path) + f.removeFileAllCharts(path) + } + } +} + +func (f *Filecheck) updateDirCharts(infos []*statInfo) { + seen := make(map[string]bool) + + for _, info := range infos { + seen[info.path] = true + + sd := f.seenDirs.getp(info.path) + + if !sd.hasExistenceCharts { + sd.hasExistenceCharts = true + f.addDirCharts(info.path, + dirExistenceStatusChartTmpl.Copy(), + ) + } + + if !sd.hasOtherCharts && info.fi != nil { + sd.hasOtherCharts = true + f.addDirCharts(info.path, + dirModificationTimeAgoChartTmpl.Copy(), + dirFilesCountChartTmpl.Copy(), + ) + if f.Dirs.CollectDirSize { + f.addDirCharts(info.path, + dirSizeChartTmpl.Copy(), + ) + } + + } else if sd.hasOtherCharts && info.fi == nil { + sd.hasOtherCharts = false + f.removeDirOtherCharts(info.path) + } + } + + for path := range f.seenDirs.items { + if !seen[path] { + delete(f.seenDirs.items, path) + f.removeDirAllCharts(path) + } + } +} + +func (f *Filecheck) addFileCharts(filePath string, chartsTmpl ...*module.Chart) { + cs := append(module.Charts{}, chartsTmpl...) + charts := cs.Copy() + + for _, chart := range *charts { + chart.ID = fmt.Sprintf(chart.ID, cleanPath(filePath)) + chart.Labels = []module.Label{ + {Key: "file_path", Value: filePath}, + } + for _, dim := range chart.Dims { + dim.ID = fmt.Sprintf(dim.ID, filePath) + } + } + + if err := f.Charts().Add(*charts...); err != nil { + f.Warning(err) + } +} + +func (f *Filecheck) addDirCharts(dirPath string, chartsTmpl ...*module.Chart) { + cs := append(module.Charts{}, chartsTmpl...) + charts := cs.Copy() + + for _, chart := range *charts { + chart.ID = fmt.Sprintf(chart.ID, cleanPath(dirPath)) + chart.Labels = []module.Label{ + {Key: "dir_path", Value: dirPath}, + } + for _, dim := range chart.Dims { + dim.ID = fmt.Sprintf(dim.ID, dirPath) + } + } + + if err := f.Charts().Add(*charts...); err != nil { + f.Warning(err) + } +} + +func (f *Filecheck) removeFileAllCharts(filePath string) { + px := fmt.Sprintf("file_%s_", cleanPath(filePath)) + f.removeCharts(func(id string) bool { + return strings.HasPrefix(id, px) + }) +} + +func (f *Filecheck) removeFileOtherCharts(filePath string) { + px := fmt.Sprintf("file_%s_", cleanPath(filePath)) + f.removeCharts(func(id string) bool { + return strings.HasPrefix(id, px) && !strings.HasSuffix(id, "existence_status") + }) +} + +func (f *Filecheck) removeDirAllCharts(dirPath string) { + px := fmt.Sprintf("dir_%s_", cleanPath(dirPath)) + f.removeCharts(func(id string) bool { + return strings.HasPrefix(id, px) + }) +} + +func (f *Filecheck) removeDirOtherCharts(dirPath string) { + px := fmt.Sprintf("dir_%s_", cleanPath(dirPath)) + f.removeCharts(func(id string) bool { + return strings.HasPrefix(id, px) && !strings.HasSuffix(id, "existence_status") + }) +} + +func (f *Filecheck) removeCharts(match func(id string) bool) { + for _, chart := range *f.Charts() { + if match(chart.ID) { + chart.MarkRemove() + chart.MarkNotCreated() + } + } +} + +func cleanPath(path string) string { + path = strings.ReplaceAll(path, " ", "_") + path = strings.ReplaceAll(path, ".", "_") + return path +} diff --git a/src/go/plugin/go.d/modules/filecheck/collect.go b/src/go/plugin/go.d/modules/filecheck/collect.go new file mode 100644 index 00000000..077ad86c --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/collect.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "errors" + "io/fs" + "os" +) + +func (f *Filecheck) collect() (map[string]int64, error) { + mx := make(map[string]int64) + + f.collectFiles(mx) + f.collectDirs(mx) + + return mx, nil +} + +type statInfo struct { + path string + exists bool + fi fs.FileInfo +} + +func getStatInfo(path string) *statInfo { + fi, err := os.Stat(path) + if err != nil { + return &statInfo{ + path: path, + exists: !errors.Is(err, fs.ErrNotExist), + } + } + + return &statInfo{ + path: path, + exists: true, + fi: fi, + } +} diff --git a/src/go/plugin/go.d/modules/filecheck/collect_dirs.go b/src/go/plugin/go.d/modules/filecheck/collect_dirs.go new file mode 100644 index 00000000..143915d4 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/collect_dirs.go @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "fmt" + "os" + "path/filepath" + "time" +) + +func (f *Filecheck) collectDirs(mx map[string]int64) { + now := time.Now() + + if f.isTimeToDiscoverDirs(now) { + f.lastDiscDirsTime = now + f.curDirs = f.discoveryDirs() + } + + var infos []*statInfo + + for _, dir := range f.curDirs { + si := getStatInfo(dir) + infos = append(infos, si) + + f.collectDir(mx, si, now) + } + + f.updateDirCharts(infos) +} + +func (f *Filecheck) collectDir(mx map[string]int64, si *statInfo, now time.Time) { + px := fmt.Sprintf("dir_%s_", si.path) + + mx[px+"existence_status_exist"] = 0 + mx[px+"existence_status_not_exist"] = 0 + if !si.exists { + mx[px+"existence_status_not_exist"] = 1 + } else { + mx[px+"existence_status_exist"] = 1 + } + + if si.fi == nil || !si.fi.IsDir() { + return + } + + mx[px+"mtime_ago"] = int64(now.Sub(si.fi.ModTime()).Seconds()) + + if v, err := calcFilesInDir(si.path); err == nil { + mx[px+"files_count"] = v + } + if f.Dirs.CollectDirSize { + if v, err := calcDirSize(si.path); err == nil { + mx[px+"size_bytes"] = v + } + } +} + +func (f *Filecheck) discoveryDirs() (dirs []string) { + return discoverFilesOrDirs(f.Dirs.Include, func(v string, fi os.FileInfo) bool { + return fi.IsDir() && !f.dirsFilter.MatchString(v) + }) +} + +func (f *Filecheck) isTimeToDiscoverDirs(now time.Time) bool { + return now.After(f.lastDiscDirsTime.Add(f.DiscoveryEvery.Duration())) +} + +func calcFilesInDir(dirPath string) (int64, error) { + f, err := os.Open(dirPath) + if err != nil { + return 0, err + } + defer func() { _ = f.Close() }() + names, err := f.Readdirnames(-1) + return int64(len(names)), err +} + +func calcDirSize(dirPath string) (int64, error) { + var size int64 + err := filepath.Walk(dirPath, func(_ string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if !fi.IsDir() { + size += fi.Size() + } + return nil + }) + return size, err +} diff --git a/src/go/plugin/go.d/modules/filecheck/collect_files.go b/src/go/plugin/go.d/modules/filecheck/collect_files.go new file mode 100644 index 00000000..4c465c11 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/collect_files.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "fmt" + "os" + "time" +) + +func (f *Filecheck) collectFiles(mx map[string]int64) { + now := time.Now() + + if f.isTimeToDiscoverFiles(now) { + f.lastDiscFilesTime = now + f.curFiles = f.discoverFiles() + } + + var infos []*statInfo + + for _, file := range f.curFiles { + si := getStatInfo(file) + + infos = append(infos, si) + + f.collectFile(mx, si, now) + } + + f.updateFileCharts(infos) +} + +func (f *Filecheck) collectFile(mx map[string]int64, si *statInfo, now time.Time) { + px := fmt.Sprintf("file_%s_", si.path) + + mx[px+"existence_status_exist"] = 0 + mx[px+"existence_status_not_exist"] = 0 + if !si.exists { + mx[px+"existence_status_not_exist"] = 1 + } else { + mx[px+"existence_status_exist"] = 1 + } + + if si.fi == nil || !si.fi.Mode().IsRegular() { + return + } + + mx[px+"mtime_ago"] = int64(now.Sub(si.fi.ModTime()).Seconds()) + mx[px+"size_bytes"] = si.fi.Size() +} + +func (f *Filecheck) discoverFiles() (files []string) { + return discoverFilesOrDirs(f.Files.Include, func(absPath string, fi os.FileInfo) bool { + return fi.Mode().IsRegular() && !f.filesFilter.MatchString(absPath) + }) +} + +func (f *Filecheck) isTimeToDiscoverFiles(now time.Time) bool { + return now.After(f.lastDiscFilesTime.Add(f.DiscoveryEvery.Duration())) +} diff --git a/src/go/plugin/go.d/modules/filecheck/config_schema.json b/src/go/plugin/go.d/modules/filecheck/config_schema.json new file mode 100644 index 00000000..c64bb941 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/config_schema.json @@ -0,0 +1,164 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Filecheck collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "discovery_every": { + "title": "Scan interval", + "description": "Scan frequency interval (seconds) for files and directories with patterns (globs) in their paths.", + "type": "integer", + "minimum": 1, + "default": 60 + }, + "files": { + "title": "File selector", + "description": "Configuration for monitoring specific files. If left empy, no files will be monitored.", + "type": [ + "object", + "null" + ], + "properties": { + "include": { + "title": "Include", + "description": "Include files that match any of the specified include [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Filepath", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + }, + "exclude": { + "title": "Exclude", + "description": "Exclude files that match any of the specified exclude [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Filepath", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + } + }, + "required": [ + "include" + ] + }, + "dirs": { + "title": "Directory selector", + "description": "Configuration for monitoring specific directories. If left empy, no directories will be monitored.", + "type": [ + "object", + "null" + ], + "properties": { + "collect_dir_size": { + "title": "Collect directory size", + "description": "Enable the collection of directory sizes for each monitored directory. **Enabling this option may introduce additional overhead** on both Netdata and the host system, particularly if directories contain a large number of subdirectories and files.", + "type": "boolean", + "default": false + }, + "include": { + "title": "Include", + "description": "Include directories that match any of the specified include [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Directory", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + }, + "exclude": { + "title": "Exclude", + "description": "Exclude directories that match any of the specified exclude [patterns](https://golang.org/pkg/path/filepath/#Match).", + "type": [ + "array", + "null" + ], + "items": { + "title": "Directory", + "type": "string", + "pattern": "^$|^/" + }, + "uniqueItems": true + } + }, + "required": [ + "include" + ] + } + }, + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "uiOptions": { + "fullPage": true + }, + "ui:flavour": "tabs", + "ui:options": { + "tabs": [ + { + "title": "Base", + "fields": [ + "update_every", + "discovery_every" + ] + }, + { + "title": "Files", + "fields": [ + "files" + ] + }, + { + "title": "Directories", + "fields": [ + "dirs" + ] + } + ] + }, + "files": { + "ui:help": "The logic for inclusion and exclusion is as follows: `(include1 OR include2) AND !(exclude1 OR exclude2)`.", + "ui:collapsible": true, + "include": { + "ui:listFlavour": "list" + }, + "exclude": { + "ui:listFlavour": "list" + } + }, + "dirs": { + "ui:help": "The logic for inclusion and exclusion is as follows: `(include1 OR include2) AND !(exclude1 OR exclude2)`.", + "ui:collapsible": true, + "include": { + "ui:listFlavour": "list" + }, + "exclude": { + "ui:listFlavour": "list" + } + } + } +} diff --git a/src/go/plugin/go.d/modules/filecheck/discover.go b/src/go/plugin/go.d/modules/filecheck/discover.go new file mode 100644 index 00000000..29ae552c --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/discover.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "os" + "path/filepath" + "runtime" + "slices" + "strings" +) + +func discoverFilesOrDirs(includePaths []string, fn func(absPath string, fi os.FileInfo) bool) []string { + var paths []string + + for _, path := range includePaths { + if !hasMeta(path) { + paths = append(paths, path) + continue + } + + ps, _ := filepath.Glob(path) + for _, path := range ps { + if fi, err := os.Lstat(path); err == nil && fn(path, fi) { + paths = append(paths, path) + } + } + + } + + slices.Sort(paths) + paths = slices.Compact(paths) + + return paths +} + +func hasMeta(path string) bool { + magicChars := `*?[` + if runtime.GOOS != "windows" { + magicChars = `*?[\` + } + return strings.ContainsAny(path, magicChars) +} diff --git a/src/go/plugin/go.d/modules/filecheck/filecheck.go b/src/go/plugin/go.d/modules/filecheck/filecheck.go new file mode 100644 index 00000000..8d19c7c6 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/filecheck.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + _ "embed" + "time" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/matcher" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("filecheck", module.Creator{ + JobConfigSchema: configSchema, + Defaults: module.Defaults{ + UpdateEvery: 10, + }, + Create: func() module.Module { return New() }, + Config: func() any { return &Config{} }, + }) +} + +func New() *Filecheck { + return &Filecheck{ + Config: Config{ + DiscoveryEvery: web.Duration(time.Minute * 1), + Files: filesConfig{}, + Dirs: dirsConfig{CollectDirSize: false}, + }, + charts: &module.Charts{}, + seenFiles: newSeenItems(), + seenDirs: newSeenItems(), + } +} + +type ( + Config struct { + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"` + DiscoveryEvery web.Duration `yaml:"discovery_every,omitempty" json:"discovery_every"` + Files filesConfig `yaml:"files" json:"files"` + Dirs dirsConfig `yaml:"dirs" json:"dirs"` + } + filesConfig struct { + Include []string `yaml:"include" json:"include"` + Exclude []string `yaml:"exclude,omitempty" json:"exclude"` + } + dirsConfig struct { + Include []string `yaml:"include" json:"include"` + Exclude []string `yaml:"exclude,omitempty" json:"exclude"` + CollectDirSize bool `yaml:"collect_dir_size" json:"collect_dir_size"` + } +) + +type Filecheck struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + + filesFilter matcher.Matcher + lastDiscFilesTime time.Time + curFiles []string + seenFiles *seenItems + + dirsFilter matcher.Matcher + lastDiscDirsTime time.Time + curDirs []string + seenDirs *seenItems +} + +func (f *Filecheck) Configuration() any { + return f.Config +} + +func (f *Filecheck) Init() error { + err := f.validateConfig() + if err != nil { + f.Errorf("config validation: %v", err) + return err + } + + ff, err := f.initFilesFilter() + if err != nil { + f.Errorf("files filter initialization: %v", err) + return err + } + f.filesFilter = ff + + df, err := f.initDirsFilter() + if err != nil { + f.Errorf("dirs filter initialization: %v", err) + return err + } + f.dirsFilter = df + + f.Debugf("monitored files: %v", f.Files.Include) + f.Debugf("monitored dirs: %v", f.Dirs.Include) + + return nil +} + +func (f *Filecheck) Check() error { + return nil +} + +func (f *Filecheck) Charts() *module.Charts { + return f.charts +} + +func (f *Filecheck) Collect() map[string]int64 { + mx, err := f.collect() + if err != nil { + f.Error(err) + } + + if len(mx) == 0 { + return nil + } + + return mx +} + +func (f *Filecheck) Cleanup() {} diff --git a/src/go/plugin/go.d/modules/filecheck/filecheck_test.go b/src/go/plugin/go.d/modules/filecheck/filecheck_test.go new file mode 100644 index 00000000..43024b0b --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/filecheck_test.go @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "os" + "strings" + "testing" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + } { + require.NotNil(t, data, name) + } +} + +func TestFilecheck_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Filecheck{}, dataConfigJSON, dataConfigYAML) +} + +func TestFilecheck_Cleanup(t *testing.T) { + assert.NotPanics(t, New().Cleanup) +} + +func TestFilecheck_Init(t *testing.T) { + tests := map[string]struct { + config Config + wantFail bool + }{ + "default": { + wantFail: true, + config: New().Config, + }, + "empty files->include and dirs->include": { + wantFail: true, + config: Config{ + Files: filesConfig{}, + Dirs: dirsConfig{}, + }, + }, + "files->include and dirs->include": { + wantFail: false, + config: Config{ + Files: filesConfig{ + Include: []string{ + "/path/to/file1", + "/path/to/file2", + }, + }, + Dirs: dirsConfig{ + Include: []string{ + "/path/to/dir1", + "/path/to/dir2", + }, + CollectDirSize: true, + }, + }, + }, + "only files->include": { + wantFail: false, + config: Config{ + Files: filesConfig{ + Include: []string{ + "/path/to/file1", + "/path/to/file2", + }, + }, + }, + }, + "only dirs->include": { + wantFail: false, + config: Config{ + Dirs: dirsConfig{ + Include: []string{ + "/path/to/dir1", + "/path/to/dir2", + }, + CollectDirSize: true, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + fc := New() + fc.Config = test.config + + if test.wantFail { + assert.Error(t, fc.Init()) + } else { + require.NoError(t, fc.Init()) + } + }) + } +} + +func TestFilecheck_Check(t *testing.T) { + tests := map[string]struct { + prepare func() *Filecheck + }{ + "collect files": {prepare: prepareFilecheckFiles}, + "collect files filepath pattern": {prepare: prepareFilecheckGlobFiles}, + "collect only non existent files": {prepare: prepareFilecheckNonExistentFiles}, + "collect dirs": {prepare: prepareFilecheckDirs}, + "collect dirs filepath pattern": {prepare: prepareFilecheckGlobDirs}, + "collect only non existent dirs": {prepare: prepareFilecheckNonExistentDirs}, + "collect files and dirs": {prepare: prepareFilecheckFilesDirs}, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + fc := test.prepare() + require.NoError(t, fc.Init()) + + assert.NoError(t, fc.Check()) + }) + } +} + +func TestFilecheck_Collect(t *testing.T) { + // TODO: should use TEMP dir and create files/dirs dynamically during a test case + tests := map[string]struct { + prepare func() *Filecheck + wantCollected map[string]int64 + }{ + "collect files": { + prepare: prepareFilecheckFiles, + wantCollected: map[string]int64{ + "file_testdata/empty_file.log_existence_status_exist": 1, + "file_testdata/empty_file.log_existence_status_not_exist": 0, + "file_testdata/empty_file.log_mtime_ago": 517996, + "file_testdata/empty_file.log_size_bytes": 0, + "file_testdata/file.log_existence_status_exist": 1, + "file_testdata/file.log_existence_status_not_exist": 0, + "file_testdata/file.log_mtime_ago": 517996, + "file_testdata/file.log_size_bytes": 5707, + "file_testdata/non_existent_file.log_existence_status_exist": 0, + "file_testdata/non_existent_file.log_existence_status_not_exist": 1, + }, + }, + "collect files filepath pattern": { + prepare: prepareFilecheckGlobFiles, + wantCollected: map[string]int64{ + "file_testdata/empty_file.log_existence_status_exist": 1, + "file_testdata/empty_file.log_existence_status_not_exist": 0, + "file_testdata/empty_file.log_mtime_ago": 517985, + "file_testdata/empty_file.log_size_bytes": 0, + "file_testdata/file.log_existence_status_exist": 1, + "file_testdata/file.log_existence_status_not_exist": 0, + "file_testdata/file.log_mtime_ago": 517985, + "file_testdata/file.log_size_bytes": 5707, + }, + }, + "collect only non existent files": { + prepare: prepareFilecheckNonExistentFiles, + wantCollected: map[string]int64{ + "file_testdata/non_existent_file.log_existence_status_exist": 0, + "file_testdata/non_existent_file.log_existence_status_not_exist": 1, + }, + }, + "collect dirs": { + prepare: prepareFilecheckDirs, + wantCollected: map[string]int64{ + "dir_testdata/dir_existence_status_exist": 1, + "dir_testdata/dir_existence_status_not_exist": 0, + "dir_testdata/dir_files_count": 3, + "dir_testdata/dir_mtime_ago": 517914, + "dir_testdata/non_existent_dir_existence_status_exist": 0, + "dir_testdata/non_existent_dir_existence_status_not_exist": 1, + }, + }, + "collect dirs filepath pattern": { + prepare: prepareFilecheckGlobDirs, + wantCollected: map[string]int64{ + "dir_testdata/dir_existence_status_exist": 1, + "dir_testdata/dir_existence_status_not_exist": 0, + "dir_testdata/dir_files_count": 3, + "dir_testdata/dir_mtime_ago": 517902, + "dir_testdata/non_existent_dir_existence_status_exist": 0, + "dir_testdata/non_existent_dir_existence_status_not_exist": 1, + }, + }, + "collect dirs w/o size": { + prepare: prepareFilecheckDirsWithoutSize, + wantCollected: map[string]int64{ + "dir_testdata/dir_existence_status_exist": 1, + "dir_testdata/dir_existence_status_not_exist": 0, + "dir_testdata/dir_files_count": 3, + "dir_testdata/dir_mtime_ago": 517892, + "dir_testdata/non_existent_dir_existence_status_exist": 0, + "dir_testdata/non_existent_dir_existence_status_not_exist": 1, + }, + }, + "collect only non existent dirs": { + prepare: prepareFilecheckNonExistentDirs, + wantCollected: map[string]int64{ + "dir_testdata/non_existent_dir_existence_status_exist": 0, + "dir_testdata/non_existent_dir_existence_status_not_exist": 1, + }, + }, + "collect files and dirs": { + prepare: prepareFilecheckFilesDirs, + wantCollected: map[string]int64{ + "dir_testdata/dir_existence_status_exist": 1, + "dir_testdata/dir_existence_status_not_exist": 0, + "dir_testdata/dir_files_count": 3, + "dir_testdata/dir_mtime_ago": 517858, + "dir_testdata/dir_size_bytes": 8160, + "dir_testdata/non_existent_dir_existence_status_exist": 0, + "dir_testdata/non_existent_dir_existence_status_not_exist": 1, + "file_testdata/empty_file.log_existence_status_exist": 1, + "file_testdata/empty_file.log_existence_status_not_exist": 0, + "file_testdata/empty_file.log_mtime_ago": 517858, + "file_testdata/empty_file.log_size_bytes": 0, + "file_testdata/file.log_existence_status_exist": 1, + "file_testdata/file.log_existence_status_not_exist": 0, + "file_testdata/file.log_mtime_ago": 517858, + "file_testdata/file.log_size_bytes": 5707, + "file_testdata/non_existent_file.log_existence_status_exist": 0, + "file_testdata/non_existent_file.log_existence_status_not_exist": 1, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + fc := test.prepare() + require.NoError(t, fc.Init()) + + mx := fc.Collect() + + copyModTime(test.wantCollected, mx) + assert.Equal(t, test.wantCollected, mx) + testMetricsHasAllChartsDims(t, fc, mx) + }) + } +} + +func testMetricsHasAllChartsDims(t *testing.T, fc *Filecheck, mx map[string]int64) { + for _, chart := range *fc.Charts() { + if chart.Obsolete { + continue + } + for _, dim := range chart.Dims { + _, ok := mx[dim.ID] + assert.Truef(t, ok, "mx metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID) + } + } +} + +func prepareFilecheckFiles() *Filecheck { + fc := New() + fc.Config.Files.Include = []string{ + "testdata/empty_file.log", + "testdata/file.log", + "testdata/non_existent_file.log", + } + return fc +} + +func prepareFilecheckGlobFiles() *Filecheck { + fc := New() + fc.Config.Files.Include = []string{ + "testdata/*.log", + } + return fc +} + +func prepareFilecheckNonExistentFiles() *Filecheck { + fc := New() + fc.Config.Files.Include = []string{ + "testdata/non_existent_file.log", + } + return fc +} + +func prepareFilecheckDirs() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/dir", + "testdata/non_existent_dir", + } + return fc +} + +func prepareFilecheckGlobDirs() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/*ir", + "testdata/non_existent_dir", + } + return fc +} + +func prepareFilecheckDirsWithoutSize() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/dir", + "testdata/non_existent_dir", + } + return fc +} + +func prepareFilecheckNonExistentDirs() *Filecheck { + fc := New() + fc.Config.Dirs.Include = []string{ + "testdata/non_existent_dir", + } + return fc +} + +func prepareFilecheckFilesDirs() *Filecheck { + fc := New() + fc.Config.Dirs.CollectDirSize = true + fc.Config.Files.Include = []string{ + "testdata/empty_file.log", + "testdata/file.log", + "testdata/non_existent_file.log", + } + fc.Config.Dirs.Include = []string{ + "testdata/dir", + "testdata/non_existent_dir", + } + return fc +} + +func copyModTime(dst, src map[string]int64) { + if src == nil || dst == nil { + return + } + for key := range src { + if strings.Contains(key, "mtime") { + dst[key] = src[key] + } + } +} diff --git a/src/go/plugin/go.d/modules/filecheck/init.go b/src/go/plugin/go.d/modules/filecheck/init.go new file mode 100644 index 00000000..20b30964 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/init.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package filecheck + +import ( + "errors" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/matcher" +) + +func (f *Filecheck) validateConfig() error { + if len(f.Files.Include) == 0 && len(f.Dirs.Include) == 0 { + return errors.New("both 'files->include' and 'dirs->include' are empty") + } + return nil +} + +func (f *Filecheck) initFilesFilter() (matcher.Matcher, error) { + return newFilter(f.Files.Exclude) +} + +func (f *Filecheck) initDirsFilter() (matcher.Matcher, error) { + return newFilter(f.Dirs.Exclude) +} + +func newFilter(patterns []string) (matcher.Matcher, error) { + filter := matcher.FALSE() + + for _, s := range patterns { + m, err := matcher.NewGlobMatcher(s) + if err != nil { + return nil, err + } + filter = matcher.Or(filter, m) + } + + return filter, nil +} diff --git a/src/go/plugin/go.d/modules/filecheck/integrations/files_and_directories.md b/src/go/plugin/go.d/modules/filecheck/integrations/files_and_directories.md new file mode 100644 index 00000000..ed131a12 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/integrations/files_and_directories.md @@ -0,0 +1,280 @@ + + +# Files and directories + + + + + +Plugin: go.d.plugin +Module: filecheck + + + +## Overview + +This collector monitors the existence, last modification time, and size of arbitrary files and directories on the system. + + + + +This collector is supported on all platforms. + +This collector supports collecting metrics from multiple instances of this integration, including remote instances. + +This collector requires the DAC_READ_SEARCH capability when monitoring files not normally accessible to the Netdata user, but it is set automatically during installation, so no manual configuration is needed. + + +### Default Behavior + +#### Auto-Detection + +This integration doesn't support auto-detection. + +#### Limits + +The default configuration for this integration does not impose any limits on data collection. + +#### Performance Impact + +The default configuration for this integration is not expected to impose a significant performance impact on the system. + + +## Metrics + +Metrics grouped by *scope*. + +The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels. + + + +### Per file + +These metrics refer to the File. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| file_path | File absolute path | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| filecheck.file_existence_status | exist, not_exist | status | +| filecheck.file_modification_time_ago | mtime_ago | seconds | +| filecheck.file_size_bytes | size | bytes | + +### Per directory + +These metrics refer to the Directory. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| dir_path | Directory absolute path | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| filecheck.dir_existence_status | exist, not_exist | status | +| filecheck.dir_modification_time_ago | mtime_ago | seconds | +| filecheck.dir_size_bytes | size | bytes | +| filecheck.dir_files count | files | files | + + + +## Alerts + +There are no alerts configured by default for this integration. + + +## Setup + +### Prerequisites + +No action required. + +### Configuration + +#### File + +The configuration file name for this integration is `go.d/filecheck.conf`. + + +You can edit the configuration file using the `edit-config` script from the +Netdata [config directory](/docs/netdata-agent/configuration/README.md#the-netdata-config-directory). + +```bash +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata +sudo ./edit-config go.d/filecheck.conf +``` +#### Options + +The following options can be defined globally: update_every, autodetection_retry. + + +
Config options + +| Name | Description | Default | Required | +|:----|:-----------|:-------|:--------:| +| update_every | Data collection frequency. | 10 | no | +| autodetection_retry | Recheck interval in seconds. Zero means no recheck will be scheduled. | 0 | no | +| files | List of files to monitor. | | yes | +| dirs | List of directories to monitor. | | yes | +| discovery_every | Files and directories discovery interval. | 60 | no | + +##### files + +Files matching the selector will be monitored. + +- Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) +- Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) +- Syntax: + +```yaml +files: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 +``` + + +##### dirs + +Directories matching the selector will be monitored. + +- Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) +- Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) +- Syntax: + +```yaml +dirs: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 +``` + + +
+ +#### Examples + +##### Files + +Files monitoring example configuration. + +
Config + +```yaml +jobs: + - name: files_example + files: + include: + - '/path/to/file1' + - '/path/to/file2' + - '/path/to/*.log' + +``` +
+ +##### Directories + +Directories monitoring example configuration. + +
Config + +```yaml +jobs: + - name: files_example + dirs: + collect_dir_size: no + include: + - '/path/to/dir1' + - '/path/to/dir2' + - '/path/to/dir3*' + +``` +
+ + + +## Troubleshooting + +### Debug Mode + +**Important**: Debug mode is not supported for data collection jobs created via the UI using the Dyncfg feature. + +To troubleshoot issues with the `filecheck` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m filecheck + ``` + +### Getting Logs + +If you're encountering problems with the `filecheck` collector, follow these steps to retrieve logs and identify potential issues: + +- **Run the command** specific to your system (systemd, non-systemd, or Docker container). +- **Examine the output** for any warnings or error messages that might indicate issues. These messages should provide clues about the root cause of the problem. + +#### System with systemd + +Use the following command to view logs generated since the last Netdata service restart: + +```bash +journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep filecheck +``` + +#### System without systemd + +Locate the collector log file, typically at `/var/log/netdata/collector.log`, and use `grep` to filter for collector's name: + +```bash +grep filecheck /var/log/netdata/collector.log +``` + +**Note**: This method shows logs from all restarts. Focus on the **latest entries** for troubleshooting current issues. + +#### Docker Container + +If your Netdata runs in a Docker container named "netdata" (replace if different), use this command: + +```bash +docker logs netdata 2>&1 | grep filecheck +``` + + diff --git a/src/go/plugin/go.d/modules/filecheck/metadata.yaml b/src/go/plugin/go.d/modules/filecheck/metadata.yaml new file mode 100644 index 00000000..446226f2 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/metadata.yaml @@ -0,0 +1,198 @@ +plugin_name: go.d.plugin +modules: + - meta: + id: collector-go.d.plugin-filecheck + plugin_name: go.d.plugin + module_name: filecheck + monitored_instance: + name: Files and directories + link: "" + icon_filename: filesystem.svg + categories: + - data-collection.other + keywords: + - files + - directories + related_resources: + integrations: + list: [] + info_provided_to_referring_integrations: + description: "" + most_popular: false + overview: + data_collection: + metrics_description: | + This collector monitors the existence, last modification time, and size of arbitrary files and directories on the system. + method_description: "" + supported_platforms: + include: [] + exclude: [] + multi_instance: true + additional_permissions: + description: | + This collector requires the DAC_READ_SEARCH capability when monitoring files not normally accessible to the Netdata user, but it is set automatically during installation, so no manual configuration is needed. + default_behavior: + auto_detection: + description: "" + limits: + description: "" + performance_impact: + description: "" + setup: + prerequisites: + list: [] + configuration: + file: + name: go.d/filecheck.conf + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 10 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: files + description: List of files to monitor. + default_value: "" + required: true + detailed_description: | + Files matching the selector will be monitored. + + - Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) + - Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) + - Syntax: + + ```yaml + files: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 + ``` + - name: dirs + description: List of directories to monitor. + default_value: "" + required: true + detailed_description: | + Directories matching the selector will be monitored. + + - Logic: (pattern1 OR pattern2) AND !(pattern3 or pattern4) + - Pattern syntax: [shell file name pattern](https://golang.org/pkg/path/filepath/#Match) + - Syntax: + + ```yaml + dirs: + includes: + - pattern1 + - pattern2 + excludes: + - pattern3 + - pattern4 + ``` + - name: discovery_every + description: Files and directories discovery interval. + default_value: 60 + required: false + examples: + folding: + title: Config + enabled: true + list: + - name: Files + description: Files monitoring example configuration. + config: | + jobs: + - name: files_example + files: + include: + - '/path/to/file1' + - '/path/to/file2' + - '/path/to/*.log' + - name: Directories + description: Directories monitoring example configuration. + config: | + jobs: + - name: files_example + dirs: + collect_dir_size: no + include: + - '/path/to/dir1' + - '/path/to/dir2' + - '/path/to/dir3*' + troubleshooting: + problems: + list: [] + alerts: [] + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: file + description: These metrics refer to the File. + labels: + - name: file_path + description: File absolute path + metrics: + - name: filecheck.file_existence_status + description: File existence + unit: status + chart_type: line + dimensions: + - name: exist + - name: not_exist + - name: filecheck.file_modification_time_ago + description: File time since the last modification + unit: seconds + chart_type: line + dimensions: + - name: mtime_ago + - name: filecheck.file_size_bytes + description: File size + unit: bytes + chart_type: line + dimensions: + - name: size + - name: directory + description: These metrics refer to the Directory. + labels: + - name: dir_path + description: Directory absolute path + metrics: + - name: filecheck.dir_existence_status + description: Directory existence + unit: status + chart_type: line + dimensions: + - name: exist + - name: not_exist + - name: filecheck.dir_modification_time_ago + description: Directory time since the last modification + unit: seconds + chart_type: line + dimensions: + - name: mtime_ago + - name: filecheck.dir_size_bytes + description: Directory size + unit: bytes + chart_type: line + dimensions: + - name: size + - name: filecheck.dir_files count + description: Directory files count + unit: files + chart_type: line + dimensions: + - name: files diff --git a/src/go/plugin/go.d/modules/filecheck/testdata/config.json b/src/go/plugin/go.d/modules/filecheck/testdata/config.json new file mode 100644 index 00000000..93d286f8 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/testdata/config.json @@ -0,0 +1,21 @@ +{ + "update_every": 123, + "discovery_every": 123.123, + "files": { + "include": [ + "ok" + ], + "exclude": [ + "ok" + ] + }, + "dirs": { + "include": [ + "ok" + ], + "exclude": [ + "ok" + ], + "collect_dir_size": true + } +} diff --git a/src/go/plugin/go.d/modules/filecheck/testdata/config.yaml b/src/go/plugin/go.d/modules/filecheck/testdata/config.yaml new file mode 100644 index 00000000..494a2185 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/testdata/config.yaml @@ -0,0 +1,13 @@ +update_every: 123 +discovery_every: 123.123 +files: + include: + - "ok" + exclude: + - "ok" +dirs: + include: + - "ok" + exclude: + - "ok" + collect_dir_size: yes diff --git a/src/go/plugin/go.d/modules/filecheck/testdata/dir/empty_file.log b/src/go/plugin/go.d/modules/filecheck/testdata/dir/empty_file.log new file mode 100644 index 00000000..e69de29b diff --git a/src/go/plugin/go.d/modules/filecheck/testdata/dir/file.log b/src/go/plugin/go.d/modules/filecheck/testdata/dir/file.log new file mode 100644 index 00000000..c1c152a8 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/testdata/dir/file.log @@ -0,0 +1,61 @@ +198.51.100.1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 301 4715 4113 174 465 https TLSv1.2 ECDHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 1130 1202 409 450 https TLSv1 DHE-RSA-AES256-SHA light beer +198.51.100.1:83 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 201 4020 1217 492 135 https TLSv1.2 PSK-RC4-SHA light wine +test.example.org:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 401 3784 2349 266 63 http TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:83 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 201 2149 3834 178 197 https TLSv1.1 AES256-SHA dark wine +198.51.100.1:80 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 200 1442 4125 23 197 https TLSv1.3 DHE-RSA-AES256-SHA light wine +test.example.com:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 300 4134 3965 259 296 https TLSv1.3 PSK-RC4-SHA dark wine +test.example.com:84 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/1.1" 401 1224 3352 135 468 http SSLv2 PSK-RC4-SHA light wine +localhost:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/2.0" 200 2504 4754 58 371 http TLSv1.1 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/1.1" 200 4898 2787 398 476 http SSLv2 DHE-RSA-AES256-SHA dark beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 100 4957 1848 324 158 https TLSv1.2 AES256-SHA dark wine +test.example.org:80 localhost - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/2" 301 1752 1717 75 317 https SSLv3 PSK-RC4-SHA dark wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 301 3799 4120 71 17 http TLSv1.3 ECDHE-RSA-AES256-SHA dark beer +198.51.100.1:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 1870 3945 392 323 http TLSv1.1 PSK-RC4-SHA light beer +test.example.com:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2.0" 200 1261 3535 52 271 https TLSv1.1 DHE-RSA-AES256-SHA dark wine +test.example.com:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 101 3228 3545 476 168 http TLSv1.1 AES256-SHA light beer +test.example.com:80 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 300 4731 1574 362 184 https SSLv2 ECDHE-RSA-AES256-SHA light wine +198.51.100.1:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/1.1" 300 4868 1803 23 388 https TLSv1.3 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/1.1" 100 3744 3546 296 437 http SSLv2 DHE-RSA-AES256-SHA light beer +test.example.org:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 401 4858 1493 151 240 http SSLv2 AES256-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2.0" 300 1367 4284 45 443 https TLSv1.1 AES256-SHA light beer +localhost:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 100 4392 4982 143 110 http SSLv3 AES256-SHA light beer +2001:db8:1ce::1:84 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/1.1" 101 4606 3311 410 273 https TLSv1 PSK-RC4-SHA dark beer +198.51.100.1:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.net HTTP/2.0" 100 1163 1526 10 186 https SSLv2 AES256-SHA light beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 301 3262 3789 144 124 https TLSv1.3 DHE-RSA-AES256-SHA light wine +198.51.100.1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 400 1365 1447 325 186 http TLSv1.2 PSK-RC4-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 4546 4409 295 153 http SSLv3 ECDHE-RSA-AES256-SHA light beer +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 300 2297 3318 139 227 https TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 100 4671 4285 371 7 https SSLv3 ECDHE-RSA-AES256-SHA dark beer +test.example.org:83 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 400 3651 1135 172 159 https TLSv1.1 DHE-RSA-AES256-SHA light beer +localhost:82 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 3958 3959 350 121 https SSLv2 DHE-RSA-AES256-SHA dark beer +localhost:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 200 1652 3813 190 11 https SSLv3 AES256-SHA dark wine +test.example.org:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 101 1228 2344 251 366 https TLSv1 ECDHE-RSA-AES256-SHA light beer +test.example.org:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 200 1860 3118 187 419 https TLSv1 PSK-RC4-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +localhost:82 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/1.1" 401 4518 3837 18 219 http TLSv1.3 DHE-RSA-AES256-SHA dark beer +localhost:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.net HTTP/2" 201 2108 2472 257 470 http TLSv1.1 PSK-RC4-SHA dark beer +2001:db8:1ce::1:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2" 101 2020 1076 262 106 https TLSv1.3 PSK-RC4-SHA light wine +localhost:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/1.1" 100 4815 3052 49 322 https TLSv1.3 DHE-RSA-AES256-SHA light beer +2001:db8:1ce::1:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 300 1642 4001 421 194 https TLSv1 PSK-RC4-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2" 201 3805 2597 25 187 http TLSv1.1 AES256-SHA dark wine +2001:db8:1ce::1:84 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/2.0" 301 3435 1760 474 318 https TLSv1.2 ECDHE-RSA-AES256-SHA light wine +localhost:84 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2.0" 101 1911 4082 356 301 https TLSv1 DHE-RSA-AES256-SHA light beer +2001:db8:1ce::1:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2" 100 2536 1664 115 474 http SSLv3 PSK-RC4-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/1.1" 401 3757 3987 441 469 http SSLv2 ECDHE-RSA-AES256-SHA dark wine +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:83 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 400 1221 4244 232 421 https TLSv1.1 ECDHE-RSA-AES256-SHA dark wine +localhost:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 101 2001 2405 6 140 http TLSv1 DHE-RSA-AES256-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +198.51.100.1:81 localhost - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/2.0" 400 4442 4396 64 49 https TLSv1.1 AES256-SHA light beer +2001:db8:1ce::1:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 401 1461 4623 46 47 https TLSv1.3 ECDHE-RSA-AES256-SHA light beer +Unmatched! The rat the cat the dog chased killed ate the malt! \ No newline at end of file diff --git a/src/go/plugin/go.d/modules/filecheck/testdata/dir/subdir/empty_file.log b/src/go/plugin/go.d/modules/filecheck/testdata/dir/subdir/empty_file.log new file mode 100644 index 00000000..e69de29b diff --git a/src/go/plugin/go.d/modules/filecheck/testdata/empty_file.log b/src/go/plugin/go.d/modules/filecheck/testdata/empty_file.log new file mode 100644 index 00000000..e69de29b diff --git a/src/go/plugin/go.d/modules/filecheck/testdata/file.log b/src/go/plugin/go.d/modules/filecheck/testdata/file.log new file mode 100644 index 00000000..e0db6851 --- /dev/null +++ b/src/go/plugin/go.d/modules/filecheck/testdata/file.log @@ -0,0 +1,42 @@ +198.51.100.1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 301 4715 4113 174 465 https TLSv1.2 ECDHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 1130 1202 409 450 https TLSv1 DHE-RSA-AES256-SHA light beer +198.51.100.1:83 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 201 4020 1217 492 135 https TLSv1.2 PSK-RC4-SHA light wine +test.example.org:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 401 3784 2349 266 63 http TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:83 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 201 2149 3834 178 197 https TLSv1.1 AES256-SHA dark wine +198.51.100.1:80 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/1.1" 200 1442 4125 23 197 https TLSv1.3 DHE-RSA-AES256-SHA light wine +test.example.com:82 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 300 4134 3965 259 296 https TLSv1.3 PSK-RC4-SHA dark wine +test.example.com:84 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/1.1" 401 1224 3352 135 468 http SSLv2 PSK-RC4-SHA light wine +localhost:82 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/2.0" 200 2504 4754 58 371 http TLSv1.1 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/1.1" 200 4898 2787 398 476 http SSLv2 DHE-RSA-AES256-SHA dark beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 100 4957 1848 324 158 https TLSv1.2 AES256-SHA dark wine +test.example.org:80 localhost - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.com HTTP/2" 301 1752 1717 75 317 https SSLv3 PSK-RC4-SHA dark wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:82 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 301 3799 4120 71 17 http TLSv1.3 ECDHE-RSA-AES256-SHA dark beer +198.51.100.1:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 1870 3945 392 323 http TLSv1.1 PSK-RC4-SHA light beer +test.example.com:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2.0" 200 1261 3535 52 271 https TLSv1.1 DHE-RSA-AES256-SHA dark wine +test.example.com:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.org HTTP/1.1" 101 3228 3545 476 168 http TLSv1.1 AES256-SHA light beer +test.example.com:80 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 300 4731 1574 362 184 https SSLv2 ECDHE-RSA-AES256-SHA light wine +198.51.100.1:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.org HTTP/1.1" 300 4868 1803 23 388 https TLSv1.3 DHE-RSA-AES256-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/1.1" 100 3744 3546 296 437 http SSLv2 DHE-RSA-AES256-SHA light beer +test.example.org:80 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.com HTTP/2.0" 401 4858 1493 151 240 http SSLv2 AES256-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! +test.example.com:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2.0" 300 1367 4284 45 443 https TLSv1.1 AES256-SHA light beer +localhost:81 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/2" 100 4392 4982 143 110 http SSLv3 AES256-SHA light beer +2001:db8:1ce::1:84 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/1.1" 101 4606 3311 410 273 https TLSv1 PSK-RC4-SHA dark beer +198.51.100.1:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.net HTTP/2.0" 100 1163 1526 10 186 https SSLv2 AES256-SHA light beer +test.example.org:83 localhost - - [22/Mar/2009:09:30:31 +0100] "POST /example.other HTTP/2" 301 3262 3789 144 124 https TLSv1.3 DHE-RSA-AES256-SHA light wine +198.51.100.1:84 203.0.113.1 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 400 1365 1447 325 186 http TLSv1.2 PSK-RC4-SHA dark beer +Unmatched! The rat the cat the dog chased killed ate the malt! +2001:db8:1ce::1:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 301 4546 4409 295 153 http SSLv3 ECDHE-RSA-AES256-SHA light beer +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.other HTTP/2.0" 300 2297 3318 139 227 https TLSv1 ECDHE-RSA-AES256-SHA dark wine +localhost:81 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.net HTTP/1.1" 100 4671 4285 371 7 https SSLv3 ECDHE-RSA-AES256-SHA dark beer +test.example.org:83 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 400 3651 1135 172 159 https TLSv1.1 DHE-RSA-AES256-SHA light beer +localhost:82 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "POST /example.com HTTP/1.1" 101 3958 3959 350 121 https SSLv2 DHE-RSA-AES256-SHA dark beer +localhost:84 2001:db8:2ce:2 - - [22/Mar/2009:09:30:31 +0100] "GET /example.org HTTP/2.0" 200 1652 3813 190 11 https SSLv3 AES256-SHA dark wine +test.example.org:83 2001:db8:2ce:1 - - [22/Mar/2009:09:30:31 +0100] "HEAD /example.other HTTP/2" 101 1228 2344 251 366 https TLSv1 ECDHE-RSA-AES256-SHA light beer +test.example.org:80 203.0.113.2 - - [22/Mar/2009:09:30:31 +0100] "POST /example.net HTTP/2.0" 200 1860 3118 187 419 https TLSv1 PSK-RC4-SHA light wine +Unmatched! The rat the cat the dog chased killed ate the malt! \ No newline at end of file -- cgit v1.2.3