diff options
Diffstat (limited to '')
14 files changed, 2147 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/README.md b/src/go/collectors/go.d.plugin/modules/httpcheck/README.md new file mode 120000 index 000000000..69f056137 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/README.md @@ -0,0 +1 @@ +integrations/http_endpoints.md
\ No newline at end of file diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/charts.go b/src/go/collectors/go.d.plugin/modules/httpcheck/charts.go new file mode 100644 index 000000000..efb0f874b --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/charts.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package httpcheck + +import ( + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +const ( + prioResponseTime = module.Priority + iota + prioResponseLength + prioResponseStatus + prioResponseInStatusDuration +) + +var httpCheckCharts = module.Charts{ + responseTimeChart.Copy(), + responseLengthChart.Copy(), + responseStatusChart.Copy(), + responseInStatusDurationChart.Copy(), +} + +var responseTimeChart = module.Chart{ + ID: "response_time", + Title: "HTTP Response Time", + Units: "ms", + Fam: "response", + Ctx: "httpcheck.response_time", + Priority: prioResponseTime, + Dims: module.Dims{ + {ID: "time"}, + }, +} + +var responseLengthChart = module.Chart{ + ID: "response_length", + Title: "HTTP Response Body Length", + Units: "characters", + Fam: "response", + Ctx: "httpcheck.response_length", + Priority: prioResponseLength, + Dims: module.Dims{ + {ID: "length"}, + }, +} + +var responseStatusChart = module.Chart{ + ID: "request_status", + Title: "HTTP Check Status", + Units: "boolean", + Fam: "status", + Ctx: "httpcheck.status", + Priority: prioResponseStatus, + Dims: module.Dims{ + {ID: "success"}, + {ID: "no_connection"}, + {ID: "timeout"}, + {ID: "redirect"}, + {ID: "bad_content"}, + {ID: "bad_status"}, + {ID: "bad_header"}, + }, +} + +var responseInStatusDurationChart = module.Chart{ + ID: "current_state_duration", + Title: "HTTP Current State Duration", + Units: "seconds", + Fam: "status", + Ctx: "httpcheck.in_state", + Priority: prioResponseInStatusDuration, + Dims: module.Dims{ + {ID: "in_state", Name: "time"}, + }, +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/collect.go b/src/go/collectors/go.d.plugin/modules/httpcheck/collect.go new file mode 100644 index 000000000..8d88dc02f --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/collect.go @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package httpcheck + +import ( + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +type reqErrCode int + +const ( + codeTimeout reqErrCode = iota + codeRedirect + codeNoConnection +) + +func (hc *HTTPCheck) collect() (map[string]int64, error) { + req, err := web.NewHTTPRequest(hc.Request) + if err != nil { + return nil, fmt.Errorf("error on creating HTTP requests to %s : %v", hc.Request.URL, err) + } + + if hc.CookieFile != "" { + if err := hc.readCookieFile(); err != nil { + return nil, fmt.Errorf("error on reading cookie file '%s': %v", hc.CookieFile, err) + } + } + + start := time.Now() + resp, err := hc.httpClient.Do(req) + dur := time.Since(start) + + defer closeBody(resp) + + var mx metrics + + if hc.isError(err, resp) { + hc.Debug(err) + hc.collectErrResponse(&mx, err) + } else { + mx.ResponseTime = durationToMs(dur) + hc.collectOKResponse(&mx, resp) + } + + if hc.metrics.Status != mx.Status { + mx.InState = hc.UpdateEvery + } else { + mx.InState = hc.metrics.InState + hc.UpdateEvery + } + hc.metrics = mx + + return stm.ToMap(mx), nil +} + +func (hc *HTTPCheck) isError(err error, resp *http.Response) bool { + return err != nil && !(errors.Is(err, web.ErrRedirectAttempted) && hc.acceptedStatuses[resp.StatusCode]) +} + +func (hc *HTTPCheck) collectErrResponse(mx *metrics, err error) { + switch code := decodeReqError(err); code { + case codeNoConnection: + mx.Status.NoConnection = true + case codeTimeout: + mx.Status.Timeout = true + case codeRedirect: + mx.Status.Redirect = true + default: + panic(fmt.Sprintf("unknown request error code : %d", code)) + } +} + +func (hc *HTTPCheck) collectOKResponse(mx *metrics, resp *http.Response) { + hc.Debugf("endpoint '%s' returned %d (%s) HTTP status code", hc.URL, resp.StatusCode, resp.Status) + + if !hc.acceptedStatuses[resp.StatusCode] { + mx.Status.BadStatusCode = true + return + } + + bs, err := io.ReadAll(resp.Body) + // golang net/http closes body on redirect + if err != nil && !errors.Is(err, io.EOF) && !strings.Contains(err.Error(), "read on closed response body") { + hc.Warningf("error on reading body : %v", err) + mx.Status.BadContent = true + return + } + + mx.ResponseLength = len(bs) + + if hc.reResponse != nil && !hc.reResponse.Match(bs) { + mx.Status.BadContent = true + return + } + + if ok := hc.checkHeader(resp); !ok { + mx.Status.BadHeader = true + return + } + + mx.Status.Success = true +} + +func (hc *HTTPCheck) checkHeader(resp *http.Response) bool { + for _, m := range hc.headerMatch { + value := resp.Header.Get(m.key) + + var ok bool + switch { + case value == "": + ok = m.exclude + case m.valMatcher == nil: + ok = !m.exclude + default: + ok = m.valMatcher.MatchString(value) + } + + if !ok { + hc.Debugf("header match: bad header: exlude '%v' key '%s' value '%s'", m.exclude, m.key, value) + return false + } + } + + return true +} + +func decodeReqError(err error) reqErrCode { + if err == nil { + panic("nil error") + } + + if errors.Is(err, web.ErrRedirectAttempted) { + return codeRedirect + } + var v net.Error + if errors.As(err, &v) && v.Timeout() { + return codeTimeout + } + return codeNoConnection +} + +func (hc *HTTPCheck) readCookieFile() error { + if hc.CookieFile == "" { + return nil + } + + fi, err := os.Stat(hc.CookieFile) + if err != nil { + return err + } + + if hc.cookieFileModTime.Equal(fi.ModTime()) { + hc.Debugf("cookie file '%s' modification time has not changed, using previously read data", hc.CookieFile) + return nil + } + + hc.Debugf("reading cookie file '%s'", hc.CookieFile) + + jar, err := loadCookieJar(hc.CookieFile) + if err != nil { + return err + } + + hc.httpClient.Jar = jar + hc.cookieFileModTime = fi.ModTime() + + return nil +} + +func closeBody(resp *http.Response) { + if resp == nil || resp.Body == nil { + return + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() +} + +func durationToMs(duration time.Duration) int { + return int(duration) / (int(time.Millisecond) / int(time.Nanosecond)) +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/config_schema.json b/src/go/collectors/go.d.plugin/modules/httpcheck/config_schema.json new file mode 100644 index 000000000..80db7b05c --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/config_schema.json @@ -0,0 +1,258 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "HTTPCheck collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 5 + }, + "url": { + "title": "URL", + "description": "The URL of the HTTP endpoint.", + "type": "string", + "format": "uri" + }, + "timeout": { + "title": "Timeout", + "description": "The timeout in seconds for the HTTP request.", + "type": "number", + "minimum": 0.5, + "default": 1 + }, + "not_follow_redirects": { + "title": "Not follow redirects", + "description": "If set, the client will not follow HTTP redirects automatically.", + "type": "boolean" + }, + "method": { + "title": "Method", + "description": "The [HTTP method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) to use for the request. An empty string means `GET`.", + "type": "string", + "default": "GET", + "examples": [ + "GET", + "POST", + "PUT", + "PATCH" + ] + }, + "body": { + "title": "Body", + "description": "The body content to send along with the HTTP request (if applicable).", + "type": "string" + }, + "status_accepted": { + "title": "Status code check", + "description": "Specifies the list of **HTTP response status codes** that are considered **acceptable**. Responses with status codes not included in this list will be categorized as 'bad status' in the status chart.", + "type": [ + "array", + "null" + ], + "items": { + "title": "Code", + "type": "integer", + "minimum": 100, + "default": 200 + }, + "minItems": 1, + "uniqueItems": true, + "default": [ + 200 + ] + }, + "response_match": { + "title": "Content check", + "description": "Specifies a [regular expression](https://regex101.com/) pattern to match against the content (body) of the HTTP response. This check is performed only if the response's status code is accepted.", + "type": "string" + }, + "header_match": { + "title": "Header check", + "description": "Specifies a set of rules to check for specific key-value pairs in the HTTP headers of the response.", + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "exclude": { + "title": "Exclude", + "description": "Determines whether the rule checks for the presence or absence of the specified key-value pair in the HTTP headers.", + "type": "boolean" + }, + "key": { + "title": "Header key", + "description": "Specifies the exact name of the HTTP header to check for.", + "type": "string" + }, + "value": { + "title": "Header value pattern", + "description": "Specifies the [matcher pattern](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/pkg/matcher#readme) to match against the value of the specified header.", + "type": "string" + } + }, + "required": [ + "key", + "value" + ] + } + }, + "username": { + "title": "Username", + "description": "The username for basic authentication.", + "type": "string", + "sensitive": true + }, + "password": { + "title": "Password", + "description": "The password for basic authentication.", + "type": "string", + "sensitive": true + }, + "cookie_file": { + "title": "Cookie file", + "description": "Specifies the path to the file containing cookies. For more information about the cookie file format, see [cookie file format](https://everything.curl.dev/http/cookies/fileformat).", + "type": "string" + }, + "proxy_url": { + "title": "Proxy URL", + "description": "The URL of the proxy server.", + "type": "string" + }, + "proxy_username": { + "title": "Proxy username", + "description": "The username for proxy authentication.", + "type": "string", + "sensitive": true + }, + "proxy_password": { + "title": "Proxy password", + "description": "The password for proxy authentication.", + "type": "string", + "sensitive": true + }, + "headers": { + "title": "Headers", + "description": "Additional HTTP headers to include in the request.", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "tls_skip_verify": { + "title": "Skip TLS verification", + "description": "If set, TLS certificate verification will be skipped.", + "type": "boolean" + }, + "tls_ca": { + "title": "TLS CA", + "description": "The path to the CA certificate file for TLS verification.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_cert": { + "title": "TLS certificate", + "description": "The path to the client certificate file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_key": { + "title": "TLS key", + "description": "The path to the client key file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + } + }, + "required": [ + "url", + "status_accepted" + ], + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "ui:flavour": "tabs", + "ui:options": { + "tabs": [ + { + "title": "Base", + "fields": [ + "update_every", + "url", + "timeout", + "not_follow_redirects", + "method", + "body" + ] + }, + { + "title": "Checks", + "fields": [ + "status_accepted", + "response_match", + "header_match" + ] + }, + { + "title": "Auth", + "fields": [ + "username", + "password", + "cookie_file" + ] + }, + { + "title": "TLS", + "fields": [ + "tls_skip_verify", + "tls_ca", + "tls_cert", + "tls_key" + ] + }, + { + "title": "Proxy", + "fields": [ + "proxy_url", + "proxy_username", + "proxy_password" + ] + }, + { + "title": "Headers", + "fields": [ + "headers" + ] + } + ] + }, + "uiOptions": { + "fullPage": true + }, + "url": { + "ui:placeholder": "http://127.0.0.1" + }, + "timeout": { + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)." + }, + "method": { + "ui:placeholder": "GET" + }, + "body": { + "ui:widget": "textarea" + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/cookiejar.go b/src/go/collectors/go.d.plugin/modules/httpcheck/cookiejar.go new file mode 100644 index 000000000..628867caa --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/cookiejar.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package httpcheck + +import ( + "bufio" + "fmt" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "strconv" + "strings" + "time" + + "golang.org/x/net/publicsuffix" +) + +// TODO: implement proper cookie auth support +// relevant forum topic: https://community.netdata.cloud/t/howto-http-endpoint-collector-with-cookie-and-user-pass/3981/5?u=ilyam8 + +// cookie file format: https://everything.curl.dev/http/cookies/fileformat +func loadCookieJar(path string) (http.CookieJar, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + + jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + if err != nil { + return nil, err + } + + sc := bufio.NewScanner(file) + + for sc.Scan() { + line, httpOnly := strings.CutPrefix(strings.TrimSpace(sc.Text()), "#HttpOnly_") + + if strings.HasPrefix(line, "#") || line == "" { + continue + } + + parts := strings.Fields(line) + if len(parts) != 6 && len(parts) != 7 { + return nil, fmt.Errorf("got %d fields in line '%s', want 6 or 7", len(parts), line) + } + + for i, v := range parts { + parts[i] = strings.TrimSpace(v) + } + + cookie := &http.Cookie{ + Domain: parts[0], + Path: parts[2], + Name: parts[5], + HttpOnly: httpOnly, + } + cookie.Secure, err = strconv.ParseBool(parts[3]) + if err != nil { + return nil, err + } + expires, err := strconv.ParseInt(parts[4], 10, 64) + if err != nil { + return nil, err + } + if expires > 0 { + cookie.Expires = time.Unix(expires, 0) + } + if len(parts) == 7 { + cookie.Value = parts[6] + } + + scheme := "http" + if cookie.Secure { + scheme = "https" + } + cookieURL := &url.URL{ + Scheme: scheme, + Host: cookie.Domain, + } + + cookies := jar.Cookies(cookieURL) + cookies = append(cookies, cookie) + jar.SetCookies(cookieURL, cookies) + } + + return jar, nil +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/httpcheck.go b/src/go/collectors/go.d.plugin/modules/httpcheck/httpcheck.go new file mode 100644 index 000000000..6d597d483 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/httpcheck.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package httpcheck + +import ( + _ "embed" + "errors" + "net/http" + "regexp" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("httpcheck", module.Creator{ + JobConfigSchema: configSchema, + Defaults: module.Defaults{ + UpdateEvery: 5, + }, + Create: func() module.Module { return New() }, + Config: func() any { return &Config{} }, + }) +} + +func New() *HTTPCheck { + return &HTTPCheck{ + Config: Config{ + HTTP: web.HTTP{ + Client: web.Client{ + Timeout: web.Duration(time.Second), + }, + }, + AcceptedStatuses: []int{200}, + }, + + acceptedStatuses: make(map[int]bool), + } +} + +type ( + Config struct { + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"` + web.HTTP `yaml:",inline" json:""` + AcceptedStatuses []int `yaml:"status_accepted" json:"status_accepted"` + ResponseMatch string `yaml:"response_match,omitempty" json:"response_match"` + CookieFile string `yaml:"cookie_file,omitempty" json:"cookie_file"` + HeaderMatch []headerMatchConfig `yaml:"header_match,omitempty" json:"header_match"` + } + headerMatchConfig struct { + Exclude bool `yaml:"exclude" json:"exclude"` + Key string `yaml:"key" json:"key"` + Value string `yaml:"value" json:"value"` + } +) + +type HTTPCheck struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + + httpClient *http.Client + + acceptedStatuses map[int]bool + reResponse *regexp.Regexp + headerMatch []headerMatch + cookieFileModTime time.Time + + metrics metrics +} + +func (hc *HTTPCheck) Configuration() any { + return hc.Config +} + +func (hc *HTTPCheck) Init() error { + if err := hc.validateConfig(); err != nil { + hc.Errorf("config validation: %v", err) + return err + } + + hc.charts = hc.initCharts() + + httpClient, err := hc.initHTTPClient() + if err != nil { + hc.Errorf("init HTTP client: %v", err) + return err + } + hc.httpClient = httpClient + + re, err := hc.initResponseMatchRegexp() + if err != nil { + hc.Errorf("init response match regexp: %v", err) + return err + } + hc.reResponse = re + + hm, err := hc.initHeaderMatch() + if err != nil { + hc.Errorf("init header match: %v", err) + return err + } + hc.headerMatch = hm + + for _, v := range hc.AcceptedStatuses { + hc.acceptedStatuses[v] = true + } + + hc.Debugf("using URL %s", hc.URL) + hc.Debugf("using HTTP timeout %s", hc.Timeout.Duration()) + hc.Debugf("using accepted HTTP statuses %v", hc.AcceptedStatuses) + if hc.reResponse != nil { + hc.Debugf("using response match regexp %s", hc.reResponse) + } + + return nil +} + +func (hc *HTTPCheck) Check() error { + mx, err := hc.collect() + if err != nil { + hc.Error(err) + return err + } + if len(mx) == 0 { + return errors.New("no metrics collected") + } + return nil +} + +func (hc *HTTPCheck) Charts() *module.Charts { + return hc.charts +} + +func (hc *HTTPCheck) Collect() map[string]int64 { + mx, err := hc.collect() + if err != nil { + hc.Error(err) + } + + if len(mx) == 0 { + return nil + } + return mx +} + +func (hc *HTTPCheck) Cleanup() { + if hc.httpClient != nil { + hc.httpClient.CloseIdleConnections() + } +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/httpcheck_test.go b/src/go/collectors/go.d.plugin/modules/httpcheck/httpcheck_test.go new file mode 100644 index 000000000..dde5761eb --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/httpcheck_test.go @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package httpcheck + +import ( + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + } { + require.NotNil(t, data, name) + } +} + +func TestHTTPCheck_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &HTTPCheck{}, dataConfigJSON, dataConfigYAML) +} + +func TestHTTPCheck_Init(t *testing.T) { + tests := map[string]struct { + wantFail bool + config Config + }{ + "success if url set": { + wantFail: false, + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: "http://127.0.0.1:38001"}, + }, + }, + }, + "fail with default": { + wantFail: true, + config: New().Config, + }, + "fail when URL not set": { + wantFail: true, + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: ""}, + }, + }, + }, + "fail if wrong response regex": { + wantFail: true, + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: "http://127.0.0.1:38001"}, + }, + ResponseMatch: "(?:qwe))", + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + httpCheck := New() + httpCheck.Config = test.config + + if test.wantFail { + assert.Error(t, httpCheck.Init()) + } else { + assert.NoError(t, httpCheck.Init()) + } + }) + } +} + +func TestHTTPCheck_Charts(t *testing.T) { + tests := map[string]struct { + prepare func(t *testing.T) *HTTPCheck + wantCharts bool + }{ + "no charts if not inited": { + wantCharts: false, + prepare: func(t *testing.T) *HTTPCheck { + return New() + }, + }, + "charts if inited": { + wantCharts: true, + prepare: func(t *testing.T) *HTTPCheck { + httpCheck := New() + httpCheck.URL = "http://127.0.0.1:38001" + require.NoError(t, httpCheck.Init()) + + return httpCheck + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + httpCheck := test.prepare(t) + + if test.wantCharts { + assert.NotNil(t, httpCheck.Charts()) + } else { + assert.Nil(t, httpCheck.Charts()) + } + }) + } +} + +func TestHTTPCheck_Cleanup(t *testing.T) { + httpCheck := New() + assert.NotPanics(t, httpCheck.Cleanup) + + httpCheck.URL = "http://127.0.0.1:38001" + require.NoError(t, httpCheck.Init()) + assert.NotPanics(t, httpCheck.Cleanup) +} + +func TestHTTPCheck_Check(t *testing.T) { + tests := map[string]struct { + prepare func() (httpCheck *HTTPCheck, cleanup func()) + wantFail bool + }{ + "success case": {wantFail: false, prepare: prepareSuccessCase}, + "timeout case": {wantFail: false, prepare: prepareTimeoutCase}, + "redirect success": {wantFail: false, prepare: prepareRedirectSuccessCase}, + "redirect fail": {wantFail: false, prepare: prepareRedirectFailCase}, + "bad status case": {wantFail: false, prepare: prepareBadStatusCase}, + "bad content case": {wantFail: false, prepare: prepareBadContentCase}, + "no connection case": {wantFail: false, prepare: prepareNoConnectionCase}, + "cookie auth case": {wantFail: false, prepare: prepareCookieAuthCase}, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + httpCheck, cleanup := test.prepare() + defer cleanup() + + require.NoError(t, httpCheck.Init()) + + if test.wantFail { + assert.Error(t, httpCheck.Check()) + } else { + assert.NoError(t, httpCheck.Check()) + } + }) + } + +} + +func TestHTTPCheck_Collect(t *testing.T) { + tests := map[string]struct { + prepare func() (httpCheck *HTTPCheck, cleanup func()) + update func(check *HTTPCheck) + wantMetrics map[string]int64 + }{ + "success case": { + prepare: prepareSuccessCase, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "timeout case": { + prepare: prepareTimeoutCase, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 0, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 1, + }, + }, + "redirect success case": { + prepare: prepareRedirectSuccessCase, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 0, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "redirect fail case": { + prepare: prepareRedirectFailCase, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 0, + "no_connection": 0, + "redirect": 1, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "bad status case": { + prepare: prepareBadStatusCase, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 1, + "in_state": 2, + "length": 0, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "bad content case": { + prepare: prepareBadContentCase, + wantMetrics: map[string]int64{ + "bad_content": 1, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 17, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "no connection case": { + prepare: prepareNoConnectionCase, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 0, + "no_connection": 1, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "header match include no value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Key: "header-key2"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "header match include with value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Key: "header-key2", Value: "= header-value"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "header match include no value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Key: "header-key99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "header match include with value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Key: "header-key2", Value: "= header-value99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "header match exclude no value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Exclude: true, Key: "header-key99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "header match exclude with value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Exclude: true, Key: "header-key2", Value: "= header-value99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "header match exclude no value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Exclude: true, Key: "header-key2"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "header match exclude with value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []headerMatchConfig{ + {Exclude: true, Key: "header-key2", Value: "= header-value"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "cookie auth case": { + prepare: prepareCookieAuthCase, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_header": 0, + "bad_status": 0, + "in_state": 2, + "length": 0, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + httpCheck, cleanup := test.prepare() + defer cleanup() + + if test.update != nil { + test.update(httpCheck) + } + + require.NoError(t, httpCheck.Init()) + + var mx map[string]int64 + + for i := 0; i < 2; i++ { + mx = httpCheck.Collect() + time.Sleep(time.Duration(httpCheck.UpdateEvery) * time.Second) + } + + copyResponseTime(test.wantMetrics, mx) + + require.Equal(t, test.wantMetrics, mx) + }) + } +} + +func prepareSuccessCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + httpCheck.ResponseMatch = "match" + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("header-key1", "header-value") + w.Header().Set("header-key2", "header-value") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("match")) + })) + + httpCheck.URL = srv.URL + + return httpCheck, srv.Close +} + +func prepareTimeoutCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + httpCheck.Timeout = web.Duration(time.Millisecond * 100) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + time.Sleep(httpCheck.Timeout.Duration() + time.Millisecond*100) + })) + + httpCheck.URL = srv.URL + + return httpCheck, srv.Close +} + +func prepareRedirectSuccessCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + httpCheck.NotFollowRedirect = true + httpCheck.AcceptedStatuses = []int{301} + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "https://example.com", http.StatusMovedPermanently) + })) + + httpCheck.URL = srv.URL + + return httpCheck, srv.Close +} + +func prepareRedirectFailCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + httpCheck.NotFollowRedirect = true + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "https://example.com", http.StatusMovedPermanently) + })) + + httpCheck.URL = srv.URL + + return httpCheck, srv.Close +} + +func prepareBadStatusCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + + httpCheck.URL = srv.URL + + return httpCheck, srv.Close +} + +func prepareBadContentCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + httpCheck.ResponseMatch = "no match" + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("hello and goodbye")) + })) + + httpCheck.URL = srv.URL + + return httpCheck, srv.Close +} + +func prepareNoConnectionCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + httpCheck.URL = "http://127.0.0.1:38001" + + return httpCheck, func() {} +} + +func prepareCookieAuthCase() (*HTTPCheck, func()) { + httpCheck := New() + httpCheck.UpdateEvery = 1 + httpCheck.CookieFile = "testdata/cookie.txt" + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if _, err := r.Cookie("JSESSIONID"); err != nil { + w.WriteHeader(http.StatusUnauthorized) + } else { + w.WriteHeader(http.StatusOK) + } + })) + + httpCheck.URL = srv.URL + + return httpCheck, srv.Close +} + +func copyResponseTime(dst, src map[string]int64) { + if v, ok := src["time"]; ok { + if _, ok := dst["time"]; ok { + dst["time"] = v + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/init.go b/src/go/collectors/go.d.plugin/modules/httpcheck/init.go new file mode 100644 index 000000000..a7f708191 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/init.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package httpcheck + +import ( + "errors" + "fmt" + "net/http" + "regexp" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "github.com/netdata/netdata/go/go.d.plugin/pkg/matcher" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +type headerMatch struct { + exclude bool + key string + valMatcher matcher.Matcher +} + +func (hc *HTTPCheck) validateConfig() error { + if hc.URL == "" { + return errors.New("'url' not set") + } + return nil +} + +func (hc *HTTPCheck) initHTTPClient() (*http.Client, error) { + return web.NewHTTPClient(hc.Client) +} + +func (hc *HTTPCheck) initResponseMatchRegexp() (*regexp.Regexp, error) { + if hc.ResponseMatch == "" { + return nil, nil + } + return regexp.Compile(hc.ResponseMatch) +} + +func (hc *HTTPCheck) initHeaderMatch() ([]headerMatch, error) { + if len(hc.HeaderMatch) == 0 { + return nil, nil + } + + var hms []headerMatch + + for _, v := range hc.HeaderMatch { + if v.Key == "" { + continue + } + + hm := headerMatch{ + exclude: v.Exclude, + key: v.Key, + valMatcher: nil, + } + + if v.Value != "" { + m, err := matcher.Parse(v.Value) + if err != nil { + return nil, fmt.Errorf("parse key '%s value '%s': %v", v.Key, v.Value, err) + } + if v.Exclude { + m = matcher.Not(m) + } + hm.valMatcher = m + } + + hms = append(hms, hm) + } + + return hms, nil +} + +func (hc *HTTPCheck) initCharts() *module.Charts { + charts := httpCheckCharts.Copy() + + for _, chart := range *charts { + chart.Labels = []module.Label{ + {Key: "url", Value: hc.URL}, + } + } + + return charts +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/integrations/http_endpoints.md b/src/go/collectors/go.d.plugin/modules/httpcheck/integrations/http_endpoints.md new file mode 100644 index 000000000..feb3133cd --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/integrations/http_endpoints.md @@ -0,0 +1,317 @@ +<!--startmeta +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/httpcheck/README.md" +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/httpcheck/metadata.yaml" +sidebar_label: "HTTP Endpoints" +learn_status: "Published" +learn_rel_path: "Collecting Metrics/Synthetic Checks" +most_popular: True +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" +endmeta--> + +# HTTP Endpoints + + +<img src="https://netdata.cloud/img/globe.svg" width="150"/> + + +Plugin: go.d.plugin +Module: httpcheck + +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" /> + +## Overview + +This collector monitors HTTP servers availability and response time. + + + + +This collector is supported on all platforms. + +This collector supports collecting metrics from multiple instances of this integration, including remote instances. + + +### Default Behavior + +#### Auto-Detection + +This integration doesn't support auto-detection. + +#### Limits + +The default configuration for this integration does not impose any limits on data collection. + +#### Performance Impact + +The default configuration for this integration is not expected to impose a significant performance impact on the system. + + +## Metrics + +Metrics grouped by *scope*. + +The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels. + + + +### Per target + +The metrics refer to the monitored target. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| url | url value that is set in the configuration file. | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| httpcheck.response_time | time | ms | +| httpcheck.response_length | length | characters | +| httpcheck.status | success, timeout, redirect, no_connection, bad_content, bad_header, bad_status | boolean | +| httpcheck.in_state | time | boolean | + + + +## Alerts + +There are no alerts configured by default for this integration. + + +## Setup + +### Prerequisites + +No action required. + +### Configuration + +#### File + +The configuration file name for this integration is `go.d/httpcheck.conf`. + + +You can edit the configuration file using the `edit-config` script from the +Netdata [config directory](/docs/netdata-agent/configuration/README.md#the-netdata-config-directory). + +```bash +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata +sudo ./edit-config go.d/httpcheck.conf +``` +#### Options + +The following options can be defined globally: update_every, autodetection_retry. + + +<details open><summary>Config options</summary> + +| Name | Description | Default | Required | +|:----|:-----------|:-------|:--------:| +| update_every | Data collection frequency. | 5 | no | +| autodetection_retry | Recheck interval in seconds. Zero means no recheck will be scheduled. | 0 | no | +| url | Server URL. | | yes | +| status_accepted | HTTP accepted response statuses. Anything else will result in 'bad status' in the status chart. | [200] | no | +| response_match | If the status code is accepted, the content of the response will be matched against this regular expression. | | no | +| headers_match | This option defines a set of rules that check for specific key-value pairs in the HTTP headers of the response. | [] | no | +| headers_match.exclude | This option determines whether the rule should check for the presence of the specified key-value pair or the absence of it. | no | no | +| headers_match.key | The exact name of the HTTP header to check for. | | yes | +| headers_match.value | The [pattern](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/pkg/matcher#supported-format) to match against the value of the specified header. | | no | +| cookie_file | Path to cookie file. See [cookie file format](https://everything.curl.dev/http/cookies/fileformat). | | no | +| timeout | HTTP request timeout. | 1 | no | +| username | Username for basic HTTP authentication. | | no | +| password | Password for basic HTTP authentication. | | no | +| proxy_url | Proxy URL. | | no | +| proxy_username | Username for proxy basic HTTP authentication. | | no | +| proxy_password | Password for proxy basic HTTP authentication. | | no | +| method | HTTP request method. | GET | no | +| body | HTTP request body. | | no | +| headers | HTTP request headers. | | no | +| not_follow_redirects | Redirect handling policy. Controls whether the client follows redirects. | no | no | +| tls_skip_verify | Server certificate chain and hostname validation policy. Controls whether the client performs this check. | no | no | +| tls_ca | Certification authority that the client uses when verifying the server's certificates. | | no | +| tls_cert | Client TLS certificate. | | no | +| tls_key | Client TLS key. | | no | + +</details> + +#### Examples + +##### Basic + +A basic example configuration. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:8080 + +``` +</details> + +##### With HTTP request headers + +Configuration with HTTP request headers that will be sent by the client. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:8080 + headers: + Host: localhost:8080 + User-Agent: netdata/go.d.plugin + Accept: */* + +``` +</details> + +##### With `status_accepted` + +A basic example configuration with non-default status_accepted. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:8080 + status_accepted: + - 200 + - 204 + +``` +</details> + +##### With `header_match` + +Example configurations with `header_match`. See the value [pattern](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/pkg/matcher#supported-format) syntax. + +<details open><summary>Config</summary> + +```yaml +jobs: + # The "X-Robots-Tag" header must be present in the HTTP response header, + # but the value of the header does not matter. + # This config checks for the presence of the header regardless of its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + + # The "X-Robots-Tag" header must be present in the HTTP response header + # only if its value is equal to "noindex, nofollow". + # This config checks both the presence of the header and its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + value: '= noindex,nofollow' + + # The "X-Robots-Tag" header must not be present in the HTTP response header + # but the value of the header does not matter. + # This config checks for the presence of the header regardless of its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + exclude: yes + + # The "X-Robots-Tag" header must not be present in the HTTP response header + # only if its value is equal to "noindex, nofollow". + # This config checks both the presence of the header and its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + exclude: yes + value: '= noindex,nofollow' + +``` +</details> + +##### HTTP authentication + +Basic HTTP authentication. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:8080 + username: username + password: password + +``` +</details> + +##### HTTPS with self-signed certificate + +Do not validate server certificate chain and hostname. + + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: https://127.0.0.1:8080 + tls_skip_verify: yes + +``` +</details> + +##### Multi-instance + +> **Note**: When you define multiple jobs, their names must be unique. + +Collecting metrics from local and remote instances. + + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:8080 + + - name: remote + url: http://192.0.2.1:8080 + +``` +</details> + + + +## Troubleshooting + +### Debug Mode + +To troubleshoot issues with the `httpcheck` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m httpcheck + ``` + + diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/metadata.yaml b/src/go/collectors/go.d.plugin/modules/httpcheck/metadata.yaml new file mode 100644 index 000000000..6b6b7d51c --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/metadata.yaml @@ -0,0 +1,291 @@ +plugin_name: go.d.plugin +modules: + - meta: + id: collector-go.d.plugin-httpcheck + plugin_name: go.d.plugin + module_name: httpcheck + monitored_instance: + name: HTTP Endpoints + link: "" + icon_filename: globe.svg + categories: + - data-collection.synthetic-checks + keywords: + - webserver + related_resources: + integrations: + list: [] + info_provided_to_referring_integrations: + description: "" + most_popular: true + overview: + data_collection: + metrics_description: | + This collector monitors HTTP servers availability and response time. + method_description: "" + supported_platforms: + include: [] + exclude: [] + multi_instance: true + additional_permissions: + description: "" + default_behavior: + auto_detection: + description: "" + limits: + description: "" + performance_impact: + description: "" + setup: + prerequisites: + list: [] + configuration: + file: + name: go.d/httpcheck.conf + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 5 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: url + description: Server URL. + default_value: "" + required: true + - name: status_accepted + description: "HTTP accepted response statuses. Anything else will result in 'bad status' in the status chart." + default_value: "[200]" + required: false + - name: response_match + description: If the status code is accepted, the content of the response will be matched against this regular expression. + default_value: "" + required: false + - name: headers_match + description: "This option defines a set of rules that check for specific key-value pairs in the HTTP headers of the response." + default_value: "[]" + required: false + - name: headers_match.exclude + description: "This option determines whether the rule should check for the presence of the specified key-value pair or the absence of it." + default_value: false + required: false + - name: headers_match.key + description: "The exact name of the HTTP header to check for." + default_value: "" + required: true + - name: headers_match.value + description: "The [pattern](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/pkg/matcher#supported-format) to match against the value of the specified header." + default_value: "" + required: false + - name: cookie_file + description: Path to cookie file. See [cookie file format](https://everything.curl.dev/http/cookies/fileformat). + default_value: "" + required: false + - name: timeout + description: HTTP request timeout. + default_value: 1 + required: false + - name: username + description: Username for basic HTTP authentication. + default_value: "" + required: false + - name: password + description: Password for basic HTTP authentication. + default_value: "" + required: false + - name: proxy_url + description: Proxy URL. + default_value: "" + required: false + - name: proxy_username + description: Username for proxy basic HTTP authentication. + default_value: "" + required: false + - name: proxy_password + description: Password for proxy basic HTTP authentication. + default_value: "" + required: false + - name: method + description: HTTP request method. + default_value: "GET" + required: false + - name: body + description: HTTP request body. + default_value: "" + required: false + - name: headers + description: HTTP request headers. + default_value: "" + required: false + - name: not_follow_redirects + description: Redirect handling policy. Controls whether the client follows redirects. + default_value: no + required: false + - name: tls_skip_verify + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check. + default_value: no + required: false + - name: tls_ca + description: Certification authority that the client uses when verifying the server's certificates. + default_value: "" + required: false + - name: tls_cert + description: Client TLS certificate. + default_value: "" + required: false + - name: tls_key + description: Client TLS key. + default_value: "" + required: false + examples: + folding: + title: Config + enabled: true + list: + - name: Basic + description: A basic example configuration. + config: | + jobs: + - name: local + url: http://127.0.0.1:8080 + - name: With HTTP request headers + description: Configuration with HTTP request headers that will be sent by the client. + config: | + jobs: + - name: local + url: http://127.0.0.1:8080 + headers: + Host: localhost:8080 + User-Agent: netdata/go.d.plugin + Accept: */* + - name: With `status_accepted` + description: A basic example configuration with non-default status_accepted. + config: | + jobs: + - name: local + url: http://127.0.0.1:8080 + status_accepted: + - 200 + - 204 + - name: With `header_match` + description: Example configurations with `header_match`. See the value [pattern](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/pkg/matcher#supported-format) syntax. + config: | + jobs: + # The "X-Robots-Tag" header must be present in the HTTP response header, + # but the value of the header does not matter. + # This config checks for the presence of the header regardless of its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + + # The "X-Robots-Tag" header must be present in the HTTP response header + # only if its value is equal to "noindex, nofollow". + # This config checks both the presence of the header and its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + value: '= noindex,nofollow' + + # The "X-Robots-Tag" header must not be present in the HTTP response header + # but the value of the header does not matter. + # This config checks for the presence of the header regardless of its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + exclude: yes + + # The "X-Robots-Tag" header must not be present in the HTTP response header + # only if its value is equal to "noindex, nofollow". + # This config checks both the presence of the header and its value. + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: X-Robots-Tag + exclude: yes + value: '= noindex,nofollow' + - name: HTTP authentication + description: Basic HTTP authentication. + config: | + jobs: + - name: local + url: http://127.0.0.1:8080 + username: username + password: password + - name: HTTPS with self-signed certificate + description: | + Do not validate server certificate chain and hostname. + config: | + jobs: + - name: local + url: https://127.0.0.1:8080 + tls_skip_verify: yes + - name: Multi-instance + description: | + > **Note**: When you define multiple jobs, their names must be unique. + + Collecting metrics from local and remote instances. + config: | + jobs: + - name: local + url: http://127.0.0.1:8080 + + - name: remote + url: http://192.0.2.1:8080 + troubleshooting: + problems: + list: [] + alerts: [] + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: target + description: The metrics refer to the monitored target. + labels: + - name: url + description: url value that is set in the configuration file. + metrics: + - name: httpcheck.response_time + description: HTTP Response Time + unit: ms + chart_type: line + dimensions: + - name: time + - name: httpcheck.response_length + description: HTTP Response Body Length + unit: characters + chart_type: line + dimensions: + - name: length + - name: httpcheck.status + description: HTTP Check Status + unit: boolean + chart_type: line + dimensions: + - name: success + - name: timeout + - name: redirect + - name: no_connection + - name: bad_content + - name: bad_header + - name: bad_status + - name: httpcheck.in_state + description: HTTP Current State Duration + unit: boolean + chart_type: line + dimensions: + - name: time diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/metrics.go b/src/go/collectors/go.d.plugin/modules/httpcheck/metrics.go new file mode 100644 index 000000000..676346fa0 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/metrics.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package httpcheck + +type metrics struct { + Status status `stm:""` + InState int `stm:"in_state"` + ResponseTime int `stm:"time"` + ResponseLength int `stm:"length"` +} + +type status struct { + Success bool `stm:"success"` // No error on request, body reading and checking its content + Timeout bool `stm:"timeout"` + Redirect bool `stm:"redirect"` + BadContent bool `stm:"bad_content"` + BadStatusCode bool `stm:"bad_status"` + BadHeader bool `stm:"bad_header"` + NoConnection bool `stm:"no_connection"` // All other errors basically +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/config.json b/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/config.json new file mode 100644 index 000000000..649393cdd --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/config.json @@ -0,0 +1,32 @@ +{ + "update_every": 123, + "url": "ok", + "body": "ok", + "method": "ok", + "headers": { + "ok": "ok" + }, + "username": "ok", + "password": "ok", + "proxy_url": "ok", + "proxy_username": "ok", + "proxy_password": "ok", + "timeout": 123.123, + "not_follow_redirects": true, + "tls_ca": "ok", + "tls_cert": "ok", + "tls_key": "ok", + "tls_skip_verify": true, + "status_accepted": [ + 123 + ], + "response_match": "ok", + "cookie_file": "ok", + "header_match": [ + { + "exclude": true, + "key": "ok", + "value": "ok" + } + ] +} diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/config.yaml b/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/config.yaml new file mode 100644 index 000000000..1a66590e6 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/config.yaml @@ -0,0 +1,25 @@ +update_every: 123 +url: "ok" +body: "ok" +method: "ok" +headers: + ok: "ok" +username: "ok" +password: "ok" +proxy_url: "ok" +proxy_username: "ok" +proxy_password: "ok" +timeout: 123.123 +not_follow_redirects: yes +tls_ca: "ok" +tls_cert: "ok" +tls_key: "ok" +tls_skip_verify: yes +status_accepted: + - 123 +response_match: "ok" +cookie_file: "ok" +header_match: + - exclude: yes + key: "ok" + value: "ok" diff --git a/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/cookie.txt b/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/cookie.txt new file mode 100644 index 000000000..2504c6ffa --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/httpcheck/testdata/cookie.txt @@ -0,0 +1,5 @@ +# HTTP Cookie File +# Generated by Wget on 2023-03-20 21:38:07. +# Edit at your own risk. + +127.0.0.1 FALSE / FALSE 0 JSESSIONID 23B508B767344EA167A4EB9B4DA4E59F
\ No newline at end of file |