diff options
Diffstat (limited to '')
13 files changed, 1414 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/modules/tengine/README.md b/src/go/collectors/go.d.plugin/modules/tengine/README.md new file mode 120000 index 000000000..e016ea0c7 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/README.md @@ -0,0 +1 @@ +integrations/tengine.md
\ No newline at end of file diff --git a/src/go/collectors/go.d.plugin/modules/tengine/apiclient.go b/src/go/collectors/go.d.plugin/modules/tengine/apiclient.go new file mode 100644 index 000000000..4f0251050 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/apiclient.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package tengine + +import ( + "bufio" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +const ( + bytesIn = "bytes_in" + bytesOut = "bytes_out" + connTotal = "conn_total" + reqTotal = "req_total" + http2xx = "http_2xx" + http3xx = "http_3xx" + http4xx = "http_4xx" + http5xx = "http_5xx" + httpOtherStatus = "http_other_status" + rt = "rt" + upsReq = "ups_req" + upsRT = "ups_rt" + upsTries = "ups_tries" + http200 = "http_200" + http206 = "http_206" + http302 = "http_302" + http304 = "http_304" + http403 = "http_403" + http404 = "http_404" + http416 = "http_416" + http499 = "http_499" + http500 = "http_500" + http502 = "http_502" + http503 = "http_503" + http504 = "http_504" + http508 = "http_508" + httpOtherDetailStatus = "http_other_detail_status" + httpUps4xx = "http_ups_4xx" + httpUps5xx = "http_ups_5xx" +) + +var defaultLineFormat = []string{ + bytesIn, + bytesOut, + connTotal, + reqTotal, + http2xx, + http3xx, + http4xx, + http5xx, + httpOtherStatus, + rt, + upsReq, + upsRT, + upsTries, + http200, + http206, + http302, + http304, + http403, + http404, + http416, + http499, + http500, + http502, + http503, + http504, + http508, + httpOtherDetailStatus, + httpUps4xx, + httpUps5xx, +} + +func newAPIClient(client *http.Client, request web.Request) *apiClient { + return &apiClient{httpClient: client, request: request} +} + +type apiClient struct { + httpClient *http.Client + request web.Request +} + +func (a apiClient) getStatus() (*tengineStatus, error) { + req, err := web.NewHTTPRequest(a.request) + if err != nil { + return nil, fmt.Errorf("error on creating request : %v", err) + } + + resp, err := a.doRequestOK(req) + defer closeBody(resp) + if err != nil { + return nil, err + } + + status, err := parseStatus(resp.Body) + if err != nil { + return nil, fmt.Errorf("error on parsing response : %v", err) + } + + return status, nil +} + +func (a apiClient) doRequestOK(req *http.Request) (*http.Response, error) { + resp, err := a.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error on request : %v", err) + } + if resp.StatusCode != http.StatusOK { + return resp, fmt.Errorf("%s returned HTTP code %d", req.URL, resp.StatusCode) + } + return resp, nil +} + +func closeBody(resp *http.Response) { + if resp != nil && resp.Body != nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } +} + +func parseStatus(r io.Reader) (*tengineStatus, error) { + var status tengineStatus + + s := bufio.NewScanner(r) + for s.Scan() { + m, err := parseStatusLine(s.Text(), defaultLineFormat) + if err != nil { + return nil, err + } + status = append(status, *m) + } + + return &status, nil +} + +func parseStatusLine(line string, lineFormat []string) (*metric, error) { + parts := strings.Split(line, ",") + + // NOTE: only default line format is supported + // TODO: custom line format? + // www.example.com,127.0.0.1:80,162,6242,1,1,1,0,0,0,0,10,1,10,1.... + i := findFirstInt(parts) + if i == -1 { + return nil, fmt.Errorf("invalid line : %s", line) + } + if len(parts[i:]) != len(lineFormat) { + return nil, fmt.Errorf("invalid line length, got %d, expected %d, line : %s", + len(parts[i:]), len(lineFormat), line) + } + + // skip "$host,$server_addr:$server_port" + parts = parts[i:] + + var m metric + for i, key := range lineFormat { + value := mustParseInt(parts[i]) + switch key { + default: + return nil, fmt.Errorf("unknown line format key: %s", key) + case bytesIn: + m.BytesIn = value + case bytesOut: + m.BytesOut = value + case connTotal: + m.ConnTotal = value + case reqTotal: + m.ReqTotal = value + case http2xx: + m.HTTP2xx = value + case http3xx: + m.HTTP3xx = value + case http4xx: + m.HTTP4xx = value + case http5xx: + m.HTTP5xx = value + case httpOtherStatus: + m.HTTPOtherStatus = value + case rt: + m.RT = value + case upsReq: + m.UpsReq = value + case upsRT: + m.UpsRT = value + case upsTries: + m.UpsTries = value + case http200: + m.HTTP200 = value + case http206: + m.HTTP206 = value + case http302: + m.HTTP302 = value + case http304: + m.HTTP304 = value + case http403: + m.HTTP403 = value + case http404: + m.HTTP404 = value + case http416: + m.HTTP416 = value + case http499: + m.HTTP499 = value + case http500: + m.HTTP500 = value + case http502: + m.HTTP502 = value + case http503: + m.HTTP503 = value + case http504: + m.HTTP504 = value + case http508: + m.HTTP508 = value + case httpOtherDetailStatus: + m.HTTPOtherDetailStatus = value + case httpUps4xx: + m.HTTPUps4xx = value + case httpUps5xx: + m.HTTPUps5xx = value + } + } + return &m, nil +} + +func findFirstInt(s []string) int { + for i, v := range s { + _, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + return i + } + return -1 +} + +func mustParseInt(value string) *int64 { + v, err := strconv.ParseInt(value, 10, 64) + if err != nil { + panic(err) + } + + return &v +} diff --git a/src/go/collectors/go.d.plugin/modules/tengine/charts.go b/src/go/collectors/go.d.plugin/modules/tengine/charts.go new file mode 100644 index 000000000..59b191dd5 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/charts.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package tengine + +import "github.com/netdata/netdata/go/go.d.plugin/agent/module" + +type ( + // Charts is an alias for module.Charts + Charts = module.Charts + // Dims is an alias for module.Dims + Dims = module.Dims +) + +var charts = Charts{ + { + ID: "bandwidth_total", + Title: "Bandwidth", + Units: "B/s", + Fam: "bandwidth", + Ctx: "tengine.bandwidth_total", + Type: module.Area, + Dims: Dims{ + {ID: "bytes_in", Name: "in", Algo: module.Incremental}, + {ID: "bytes_out", Name: "out", Algo: module.Incremental, Mul: -1}, + }, + }, + { + ID: "connections_total", + Title: "Connections", + Units: "connections/s", + Fam: "connections", + Ctx: "tengine.connections_total", + Dims: Dims{ + {ID: "conn_total", Name: "accepted", Algo: module.Incremental}, + }, + }, + { + ID: "requests_total", + Title: "Requests", + Units: "requests/s", + Fam: "requests", + Ctx: "tengine.requests_total", + Dims: Dims{ + {ID: "req_total", Name: "processed", Algo: module.Incremental}, + }, + }, + { + ID: "requests_per_response_code_family_total", + Title: "Requests Per Response Code Family", + Units: "requests/s", + Fam: "requests", + Ctx: "tengine.requests_per_response_code_family_total", + Type: module.Stacked, + Dims: Dims{ + {ID: "http_2xx", Name: "2xx", Algo: module.Incremental}, + {ID: "http_5xx", Name: "5xx", Algo: module.Incremental}, + {ID: "http_3xx", Name: "3xx", Algo: module.Incremental}, + {ID: "http_4xx", Name: "4xx", Algo: module.Incremental}, + {ID: "http_other_status", Name: "other", Algo: module.Incremental}, + }, + }, + { + ID: "requests_per_response_code_detailed_total", + Title: "Requests Per Response Code Detailed", + Units: "requests/s", + Ctx: "tengine.requests_per_response_code_detailed_total", + Fam: "requests", + Type: module.Stacked, + Dims: Dims{ + {ID: "http_200", Name: "200", Algo: module.Incremental}, + {ID: "http_206", Name: "206", Algo: module.Incremental}, + {ID: "http_302", Name: "302", Algo: module.Incremental}, + {ID: "http_304", Name: "304", Algo: module.Incremental}, + {ID: "http_403", Name: "403", Algo: module.Incremental}, + {ID: "http_404", Name: "404", Algo: module.Incremental}, + {ID: "http_416", Name: "419", Algo: module.Incremental}, + {ID: "http_499", Name: "499", Algo: module.Incremental}, + {ID: "http_500", Name: "500", Algo: module.Incremental}, + {ID: "http_502", Name: "502", Algo: module.Incremental}, + {ID: "http_503", Name: "503", Algo: module.Incremental}, + {ID: "http_504", Name: "504", Algo: module.Incremental}, + {ID: "http_508", Name: "508", Algo: module.Incremental}, + {ID: "http_other_detail_status", Name: "other", Algo: module.Incremental}, + }, + }, + { + ID: "requests_upstream_total", + Title: "Number Of Requests Calling For Upstream", + Units: "requests/s", + Fam: "upstream", + Ctx: "tengine.requests_upstream_total", + Dims: Dims{ + {ID: "ups_req", Name: "requests", Algo: module.Incremental}, + }, + }, + { + ID: "tries_upstream_total", + Title: "Number Of Times Calling For Upstream", + Units: "calls/s", + Fam: "upstream", + Ctx: "tengine.tries_upstream_total", + Dims: Dims{ + {ID: "ups_tries", Name: "calls", Algo: module.Incremental}, + }, + }, + { + ID: "requests_upstream_per_response_code_family_total", + Title: "Upstream Requests Per Response Code Family", + Units: "requests/s", + Fam: "upstream", + Type: module.Stacked, + Ctx: "tengine.requests_upstream_per_response_code_family_total", + Dims: Dims{ + {ID: "http_ups_4xx", Name: "4xx", Algo: module.Incremental}, + {ID: "http_ups_5xx", Name: "5xx", Algo: module.Incremental}, + }, + }, +} diff --git a/src/go/collectors/go.d.plugin/modules/tengine/collect.go b/src/go/collectors/go.d.plugin/modules/tengine/collect.go new file mode 100644 index 000000000..83dcba177 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/collect.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package tengine + +import ( + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" +) + +func (t *Tengine) collect() (map[string]int64, error) { + status, err := t.apiClient.getStatus() + if err != nil { + return nil, err + } + + mx := make(map[string]int64) + for _, m := range *status { + for k, v := range stm.ToMap(m) { + mx[k] += v + } + } + return mx, nil +} diff --git a/src/go/collectors/go.d.plugin/modules/tengine/config_schema.json b/src/go/collectors/go.d.plugin/modules/tengine/config_schema.json new file mode 100644 index 000000000..5493997a3 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/config_schema.json @@ -0,0 +1,177 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Tengine collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "url": { + "title": "URL", + "description": "The URL of the Tengine [status page](https://tengine.taobao.org/document/http_reqstat.html).", + "type": "string", + "default": "http://127.0.0.1/us", + "format": "uri" + }, + "timeout": { + "title": "Timeout", + "description": "The timeout in seconds for the HTTP request.", + "type": "number", + "minimum": 0.5, + "default": 1 + }, + "not_follow_redirects": { + "title": "Not follow redirects", + "description": "If set, the client will not follow HTTP redirects automatically.", + "type": "boolean" + }, + "username": { + "title": "Username", + "description": "The username for basic authentication.", + "type": "string", + "sensitive": true + }, + "password": { + "title": "Password", + "description": "The password for basic authentication.", + "type": "string", + "sensitive": true + }, + "proxy_url": { + "title": "Proxy URL", + "description": "The URL of the proxy server.", + "type": "string" + }, + "proxy_username": { + "title": "Proxy username", + "description": "The username for proxy authentication.", + "type": "string", + "sensitive": true + }, + "proxy_password": { + "title": "Proxy password", + "description": "The password for proxy authentication.", + "type": "string", + "sensitive": true + }, + "headers": { + "title": "Headers", + "description": "Additional HTTP headers to include in the request.", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "tls_skip_verify": { + "title": "Skip TLS verification", + "description": "If set, TLS certificate verification will be skipped.", + "type": "boolean" + }, + "tls_ca": { + "title": "TLS CA", + "description": "The path to the CA certificate file for TLS verification.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_cert": { + "title": "TLS certificate", + "description": "The path to the client certificate file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_key": { + "title": "TLS key", + "description": "The path to the client key file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "body": { + "title": "Body", + "type": "string" + }, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "ui:flavour": "tabs", + "ui:options": { + "tabs": [ + { + "title": "Base", + "fields": [ + "update_every", + "url", + "timeout", + "not_follow_redirects" + ] + }, + { + "title": "Auth", + "fields": [ + "username", + "password" + ] + }, + { + "title": "TLS", + "fields": [ + "tls_skip_verify", + "tls_ca", + "tls_cert", + "tls_key" + ] + }, + { + "title": "Proxy", + "fields": [ + "proxy_url", + "proxy_username", + "proxy_password" + ] + }, + { + "title": "Headers", + "fields": [ + "headers" + ] + } + ] + }, + "uiOptions": { + "fullPage": true + }, + "body": { + "ui:widget": "hidden" + }, + "method": { + "ui:widget": "hidden" + }, + "timeout": { + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)." + }, + "password": { + "ui:widget": "password" + }, + "proxy_password": { + "ui:widget": "password" + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/tengine/integrations/tengine.md b/src/go/collectors/go.d.plugin/modules/tengine/integrations/tengine.md new file mode 100644 index 000000000..a4e6c5f95 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/integrations/tengine.md @@ -0,0 +1,232 @@ +<!--startmeta +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/tengine/README.md" +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/tengine/metadata.yaml" +sidebar_label: "Tengine" +learn_status: "Published" +learn_rel_path: "Collecting Metrics/Web Servers and Web Proxies" +most_popular: False +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" +endmeta--> + +# Tengine + + +<img src="https://netdata.cloud/img/tengine.jpeg" width="150"/> + + +Plugin: go.d.plugin +Module: tengine + +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" /> + +## Overview + +This collector monitors Tengine servers. + + + + +This collector is supported on all platforms. + +This collector supports collecting metrics from multiple instances of this integration, including remote instances. + + +### Default Behavior + +#### Auto-Detection + +This integration doesn't support auto-detection. + +#### Limits + +The default configuration for this integration does not impose any limits on data collection. + +#### Performance Impact + +The default configuration for this integration is not expected to impose a significant performance impact on the system. + + +## Metrics + +Metrics grouped by *scope*. + +The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels. + + + +### Per Tengine instance + +These metrics refer to the entire monitored application. + +This scope has no labels. + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| tengine.bandwidth_total | in, out | B/s | +| tengine.connections_total | accepted | connections/s | +| tengine.requests_total | processed | requests/s | +| tengine.requests_per_response_code_family_total | 2xx, 3xx, 4xx, 5xx, other | requests/s | +| tengine.requests_per_response_code_detailed_total | 200, 206, 302, 304, 403, 404, 419, 499, 500, 502, 503, 504, 508, other | requests/s | +| tengine.requests_upstream_total | requests | requests/s | +| tengine.tries_upstream_total | calls | calls/s | +| tengine.requests_upstream_per_response_code_family_total | 4xx, 5xx | requests/s | + + + +## Alerts + +There are no alerts configured by default for this integration. + + +## Setup + +### Prerequisites + +#### Enable ngx_http_reqstat_module module. + +To enable the module, see the [official documentation](ngx_http_reqstat_module](https://tengine.taobao.org/document/http_reqstat.html). +The default line format is the only supported format. + + + +### Configuration + +#### File + +The configuration file name for this integration is `go.d/tengine.conf`. + + +You can edit the configuration file using the `edit-config` script from the +Netdata [config directory](/docs/netdata-agent/configuration/README.md#the-netdata-config-directory). + +```bash +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata +sudo ./edit-config go.d/tengine.conf +``` +#### Options + +The following options can be defined globally: update_every, autodetection_retry. + + +<details open><summary>Config options</summary> + +| Name | Description | Default | Required | +|:----|:-----------|:-------|:--------:| +| update_every | Data collection frequency. | 1 | no | +| autodetection_retry | Recheck interval in seconds. Zero means no recheck will be scheduled. | 0 | no | +| url | Server URL. | http://127.0.0.1/us | yes | +| timeout | HTTP request timeout. | 2 | no | +| username | Username for basic HTTP authentication. | | no | +| password | Password for basic HTTP authentication. | | no | +| proxy_url | Proxy URL. | | no | +| proxy_username | Username for proxy basic HTTP authentication. | | no | +| proxy_password | Password for proxy basic HTTP authentication. | | no | +| method | HTTP request method. | GET | no | +| body | HTTP request body. | | no | +| headers | HTTP request headers. | | no | +| not_follow_redirects | Redirect handling policy. Controls whether the client follows redirects. | no | no | +| tls_skip_verify | Server certificate chain and hostname validation policy. Controls whether the client performs this check. | no | no | +| tls_ca | Certification authority that the client uses when verifying the server's certificates. | | no | +| tls_cert | Client TLS certificate. | | no | +| tls_key | Client TLS key. | | no | + +</details> + +#### Examples + +##### Basic + +An example configuration. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1/us + +``` +</details> + +##### HTTP authentication + +Local server with basic HTTP authentication. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1/us + username: foo + password: bar + +``` +</details> + +##### HTTPS with self-signed certificate + +Tengine with enabled HTTPS and self-signed certificate. + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: https://127.0.0.1/us + tls_skip_verify: yes + +``` +</details> + +##### Multi-instance + +> **Note**: When you define multiple jobs, their names must be unique. + +Local and remote instances. + + +<details open><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1/us + + - name: remote + url: http://203.0.113.10/us + +``` +</details> + + + +## Troubleshooting + +### Debug Mode + +To troubleshoot issues with the `tengine` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m tengine + ``` + + diff --git a/src/go/collectors/go.d.plugin/modules/tengine/metadata.yaml b/src/go/collectors/go.d.plugin/modules/tengine/metadata.yaml new file mode 100644 index 000000000..b0778c9fc --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/metadata.yaml @@ -0,0 +1,245 @@ +plugin_name: go.d.plugin +modules: + - meta: + id: collector-go.d.plugin-tengine + plugin_name: go.d.plugin + module_name: tengine + monitored_instance: + name: Tengine + link: https://tengine.taobao.org/ + icon_filename: tengine.jpeg + categories: + - data-collection.web-servers-and-web-proxies + keywords: + - tengine + - web + - webserver + related_resources: + integrations: + list: [] + info_provided_to_referring_integrations: + description: "" + most_popular: false + overview: + data_collection: + metrics_description: | + This collector monitors Tengine servers. + method_description: "" + supported_platforms: + include: [] + exclude: [] + multi_instance: true + additional_permissions: + description: "" + default_behavior: + auto_detection: + description: "" + limits: + description: "" + performance_impact: + description: "" + setup: + prerequisites: + list: + - title: Enable ngx_http_reqstat_module module. + description: | + To enable the module, see the [official documentation](ngx_http_reqstat_module](https://tengine.taobao.org/document/http_reqstat.html). + The default line format is the only supported format. + configuration: + file: + name: go.d/tengine.conf + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 1 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: url + description: Server URL. + default_value: http://127.0.0.1/us + required: true + - name: timeout + description: HTTP request timeout. + default_value: 2 + required: false + - name: username + description: Username for basic HTTP authentication. + default_value: "" + required: false + - name: password + description: Password for basic HTTP authentication. + default_value: "" + required: false + - name: proxy_url + description: Proxy URL. + default_value: "" + required: false + - name: proxy_username + description: Username for proxy basic HTTP authentication. + default_value: "" + required: false + - name: proxy_password + description: Password for proxy basic HTTP authentication. + default_value: "" + required: false + - name: method + description: HTTP request method. + default_value: GET + required: false + - name: body + description: HTTP request body. + default_value: "" + required: false + - name: headers + description: HTTP request headers. + default_value: "" + required: false + - name: not_follow_redirects + description: Redirect handling policy. Controls whether the client follows redirects. + default_value: false + required: false + - name: tls_skip_verify + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check. + default_value: false + required: false + - name: tls_ca + description: Certification authority that the client uses when verifying the server's certificates. + default_value: "" + required: false + - name: tls_cert + description: Client TLS certificate. + default_value: "" + required: false + - name: tls_key + description: Client TLS key. + default_value: "" + required: false + examples: + folding: + title: Config + enabled: true + list: + - name: Basic + description: An example configuration. + config: | + jobs: + - name: local + url: http://127.0.0.1/us + - name: HTTP authentication + description: Local server with basic HTTP authentication. + config: | + jobs: + - name: local + url: http://127.0.0.1/us + username: foo + password: bar + - name: HTTPS with self-signed certificate + description: Tengine with enabled HTTPS and self-signed certificate. + config: | + jobs: + - name: local + url: https://127.0.0.1/us + tls_skip_verify: yes + - name: Multi-instance + description: | + > **Note**: When you define multiple jobs, their names must be unique. + + Local and remote instances. + config: | + jobs: + - name: local + url: http://127.0.0.1/us + + - name: remote + url: http://203.0.113.10/us + troubleshooting: + problems: + list: [] + alerts: [] + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: global + description: These metrics refer to the entire monitored application. + labels: [] + metrics: + - name: tengine.bandwidth_total + description: Bandwidth + unit: B/s + chart_type: area + dimensions: + - name: in + - name: out + - name: tengine.connections_total + description: Connections + unit: connections/s + chart_type: line + dimensions: + - name: accepted + - name: tengine.requests_total + description: Requests + unit: requests/s + chart_type: line + dimensions: + - name: processed + - name: tengine.requests_per_response_code_family_total + description: Requests Per Response Code Family + unit: requests/s + chart_type: stacked + dimensions: + - name: 2xx + - name: 3xx + - name: 4xx + - name: 5xx + - name: other + - name: tengine.requests_per_response_code_detailed_total + description: Requests Per Response Code Detailed + unit: requests/s + chart_type: stacked + dimensions: + - name: "200" + - name: "206" + - name: "302" + - name: "304" + - name: "403" + - name: "404" + - name: "419" + - name: "499" + - name: "500" + - name: "502" + - name: "503" + - name: "504" + - name: "508" + - name: other + - name: tengine.requests_upstream_total + description: Number Of Requests Calling For Upstream + unit: requests/s + chart_type: line + dimensions: + - name: requests + - name: tengine.tries_upstream_total + description: Number Of Times Calling For Upstream + unit: calls/s + chart_type: line + dimensions: + - name: calls + - name: tengine.requests_upstream_per_response_code_family_total + description: Upstream Requests Per Response Code Family + unit: requests/s + chart_type: stacked + dimensions: + - name: 4xx + - name: 5xx diff --git a/src/go/collectors/go.d.plugin/modules/tengine/metrics.go b/src/go/collectors/go.d.plugin/modules/tengine/metrics.go new file mode 100644 index 000000000..425559479 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/metrics.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package tengine + +/* +http://tengine.taobao.org/document/http_reqstat.html + +bytes_in total number of bytes received from client +bytes_out total number of bytes sent to client +conn_total total number of accepted connections +req_total total number of processed requests +http_2xx total number of 2xx requests +http_3xx total number of 3xx requests +http_4xx total number of 4xx requests +http_5xx total number of 5xx requests +http_other_status total number of other requests +rt accumulation or rt +ups_req total number of requests calling for upstream +ups_rt accumulation or upstream rt +ups_tries total number of times calling for upstream +http_200 total number of 200 requests +http_206 total number of 206 requests +http_302 total number of 302 requests +http_304 total number of 304 requests +http_403 total number of 403 requests +http_404 total number of 404 requests +http_416 total number of 416 requests +http_499 total number of 499 requests +http_500 total number of 500 requests +http_502 total number of 502 requests +http_503 total number of 503 requests +http_504 total number of 504 requests +http_508 total number of 508 requests +http_other_detail_status total number of requests of other status codes +http_ups_4xx total number of requests of upstream 4xx +http_ups_5xx total number of requests of upstream 5xx +*/ + +type ( + tengineStatus []metric + + metric struct { + Host string + ServerAddress string + BytesIn *int64 `stm:"bytes_in"` + BytesOut *int64 `stm:"bytes_out"` + ConnTotal *int64 `stm:"conn_total"` + ReqTotal *int64 `stm:"req_total"` + HTTP2xx *int64 `stm:"http_2xx"` + HTTP3xx *int64 `stm:"http_3xx"` + HTTP4xx *int64 `stm:"http_4xx"` + HTTP5xx *int64 `stm:"http_5xx"` + HTTPOtherStatus *int64 `stm:"http_other_status"` + RT *int64 `stm:"rt"` + UpsReq *int64 `stm:"ups_req"` + UpsRT *int64 `stm:"ups_rt"` + UpsTries *int64 `stm:"ups_tries"` + HTTP200 *int64 `stm:"http_200"` + HTTP206 *int64 `stm:"http_206"` + HTTP302 *int64 `stm:"http_302"` + HTTP304 *int64 `stm:"http_304"` + HTTP403 *int64 `stm:"http_403"` + HTTP404 *int64 `stm:"http_404"` + HTTP416 *int64 `stm:"http_416"` + HTTP499 *int64 `stm:"http_499"` + HTTP500 *int64 `stm:"http_500"` + HTTP502 *int64 `stm:"http_502"` + HTTP503 *int64 `stm:"http_503"` + HTTP504 *int64 `stm:"http_504"` + HTTP508 *int64 `stm:"http_508"` + HTTPOtherDetailStatus *int64 `stm:"http_other_detail_status"` + HTTPUps4xx *int64 `stm:"http_ups_4xx"` + HTTPUps5xx *int64 `stm:"http_ups_5xx"` + } +) diff --git a/src/go/collectors/go.d.plugin/modules/tengine/tengine.go b/src/go/collectors/go.d.plugin/modules/tengine/tengine.go new file mode 100644 index 000000000..f70e4eded --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/tengine.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package tengine + +import ( + _ "embed" + "errors" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("tengine", module.Creator{ + JobConfigSchema: configSchema, + Create: func() module.Module { return New() }, + Config: func() any { return &Config{} }, + }) +} + +func New() *Tengine { + return &Tengine{ + Config: Config{ + HTTP: web.HTTP{ + Request: web.Request{ + URL: "http://127.0.0.1/us", + }, + Client: web.Client{ + Timeout: web.Duration(time.Second * 2), + }, + }, + }, + charts: charts.Copy(), + } +} + +type Config struct { + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"` + web.HTTP `yaml:",inline" json:""` +} + +type Tengine struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + + apiClient *apiClient +} + +func (t *Tengine) Configuration() any { + return t.Config +} + +func (t *Tengine) Init() error { + if t.URL == "" { + t.Error("url not set") + return errors.New("url not set") + } + + client, err := web.NewHTTPClient(t.Client) + if err != nil { + t.Errorf("error on creating http client : %v", err) + return err + } + + t.apiClient = newAPIClient(client, t.Request) + + t.Debugf("using URL: %s", t.URL) + t.Debugf("using timeout: %s", t.Timeout) + + return nil +} + +func (t *Tengine) Check() error { + mx, err := t.collect() + if err != nil { + t.Error(err) + return err + } + if len(mx) == 0 { + return errors.New("no metrics collected") + } + return nil +} + +func (t *Tengine) Charts() *module.Charts { + return t.charts +} + +func (t *Tengine) Collect() map[string]int64 { + mx, err := t.collect() + + if err != nil { + t.Error(err) + return nil + } + + return mx +} + +func (t *Tengine) Cleanup() { + if t.apiClient != nil && t.apiClient.httpClient != nil { + t.apiClient.httpClient.CloseIdleConnections() + } +} diff --git a/src/go/collectors/go.d.plugin/modules/tengine/tengine_test.go b/src/go/collectors/go.d.plugin/modules/tengine/tengine_test.go new file mode 100644 index 000000000..d8b8ec997 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/tengine_test.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package tengine + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") + + dataStatusMetrics, _ = os.ReadFile("testdata/status.txt") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + "dataStatusMetrics": dataStatusMetrics, + } { + require.NotNil(t, data, name) + } +} + +func TestTengine_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Tengine{}, dataConfigJSON, dataConfigYAML) +} + +func TestTengine_Cleanup(t *testing.T) { + New().Cleanup() +} + +func TestTengine_Init(t *testing.T) { + job := New() + + require.NoError(t, job.Init()) + assert.NotNil(t, job.apiClient) +} + +func TestTengine_Check(t *testing.T) { + ts := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(dataStatusMetrics) + })) + defer ts.Close() + + job := New() + job.URL = ts.URL + require.NoError(t, job.Init()) + assert.NoError(t, job.Check()) +} + +func TestTengine_CheckNG(t *testing.T) { + job := New() + + job.URL = "http://127.0.0.1:38001/us" + require.NoError(t, job.Init()) + assert.Error(t, job.Check()) +} + +func TestTengine_Charts(t *testing.T) { assert.NotNil(t, New().Charts()) } + +func TestTengine_Collect(t *testing.T) { + ts := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(dataStatusMetrics) + })) + defer ts.Close() + + job := New() + job.URL = ts.URL + require.NoError(t, job.Init()) + require.NoError(t, job.Check()) + + expected := map[string]int64{ + "bytes_in": 5944, + "bytes_out": 20483, + "conn_total": 354, + "http_200": 1536, + "http_206": 0, + "http_2xx": 1536, + "http_302": 43, + "http_304": 0, + "http_3xx": 50, + "http_403": 1, + "http_404": 75, + "http_416": 0, + "http_499": 0, + "http_4xx": 80, + "http_500": 0, + "http_502": 1, + "http_503": 0, + "http_504": 0, + "http_508": 0, + "http_5xx": 1, + "http_other_detail_status": 11, + "http_other_status": 0, + "http_ups_4xx": 26, + "http_ups_5xx": 1, + "req_total": 1672, + "rt": 1339, + "ups_req": 268, + "ups_rt": 644, + "ups_tries": 268, + } + + assert.Equal(t, expected, job.Collect()) +} + +func TestTengine_InvalidData(t *testing.T) { + ts := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("hello and goodbye")) + })) + defer ts.Close() + + job := New() + job.URL = ts.URL + require.NoError(t, job.Init()) + assert.Error(t, job.Check()) +} + +func TestTengine_404(t *testing.T) { + ts := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + job := New() + job.URL = ts.URL + require.NoError(t, job.Init()) + assert.Error(t, job.Check()) +} diff --git a/src/go/collectors/go.d.plugin/modules/tengine/testdata/config.json b/src/go/collectors/go.d.plugin/modules/tengine/testdata/config.json new file mode 100644 index 000000000..984c3ed6e --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/testdata/config.json @@ -0,0 +1,20 @@ +{ + "update_every": 123, + "url": "ok", + "body": "ok", + "method": "ok", + "headers": { + "ok": "ok" + }, + "username": "ok", + "password": "ok", + "proxy_url": "ok", + "proxy_username": "ok", + "proxy_password": "ok", + "timeout": 123.123, + "not_follow_redirects": true, + "tls_ca": "ok", + "tls_cert": "ok", + "tls_key": "ok", + "tls_skip_verify": true +} diff --git a/src/go/collectors/go.d.plugin/modules/tengine/testdata/config.yaml b/src/go/collectors/go.d.plugin/modules/tengine/testdata/config.yaml new file mode 100644 index 000000000..8558b61cc --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/testdata/config.yaml @@ -0,0 +1,17 @@ +update_every: 123 +url: "ok" +body: "ok" +method: "ok" +headers: + ok: "ok" +username: "ok" +password: "ok" +proxy_url: "ok" +proxy_username: "ok" +proxy_password: "ok" +timeout: 123.123 +not_follow_redirects: yes +tls_ca: "ok" +tls_cert: "ok" +tls_key: "ok" +tls_skip_verify: yes diff --git a/src/go/collectors/go.d.plugin/modules/tengine/testdata/status.txt b/src/go/collectors/go.d.plugin/modules/tengine/testdata/status.txt new file mode 100644 index 000000000..dff2ec2d6 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/tengine/testdata/status.txt @@ -0,0 +1,3 @@ +100.127.0.91,100.127.0.91:80,1594,2181,6,7,7,0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +127.0.0.1,127.0.0.1:80,4350,18302,58,58,58,0,0,0,0,0,0,0,0,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +,0,0,290,1607,1471,50,80,1,0,1339,268,644,268,1471,0,43,0,1,75,0,0,0,1,0,0,0,11,26,1
\ No newline at end of file |