summaryrefslogtreecommitdiffstats
path: root/src/go/collectors/go.d.plugin/modules/weblog/collect.go
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/go/collectors/go.d.plugin/modules/weblog/collect.go560
1 files changed, 560 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/modules/weblog/collect.go b/src/go/collectors/go.d.plugin/modules/weblog/collect.go
new file mode 100644
index 000000000..d24620774
--- /dev/null
+++ b/src/go/collectors/go.d.plugin/modules/weblog/collect.go
@@ -0,0 +1,560 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package weblog
+
+import (
+ "fmt"
+ "io"
+ "runtime"
+ "strconv"
+ "strings"
+
+ "github.com/netdata/netdata/go/go.d.plugin/pkg/logs"
+ "github.com/netdata/netdata/go/go.d.plugin/pkg/stm"
+
+ "github.com/netdata/netdata/go/go.d.plugin/agent/module"
+)
+
+func (w *WebLog) logPanicStackIfAny() {
+ err := recover()
+ if err == nil {
+ return
+ }
+ w.Errorf("[ERROR] %s\n", err)
+ for depth := 0; ; depth++ {
+ _, file, line, ok := runtime.Caller(depth)
+ if !ok {
+ break
+ }
+ w.Errorf("======> %d: %v:%d", depth, file, line)
+ }
+ panic(err)
+}
+
+func (w *WebLog) collect() (map[string]int64, error) {
+ defer w.logPanicStackIfAny()
+ w.mx.reset()
+
+ var mx map[string]int64
+
+ n, err := w.collectLogLines()
+
+ if n > 0 || err == nil {
+ mx = stm.ToMap(w.mx)
+ }
+ return mx, err
+}
+
+func (w *WebLog) collectLogLines() (int, error) {
+ logOnce := true
+ var n int
+ for {
+ w.line.reset()
+ err := w.parser.ReadLine(w.line)
+ if err != nil {
+ if err == io.EOF {
+ return n, nil
+ }
+ if !logs.IsParseError(err) {
+ return n, err
+ }
+ n++
+ if logOnce {
+ w.Infof("unmatched line: %v (parser: %s)", err, w.parser.Info())
+ logOnce = false
+ }
+ w.collectUnmatched()
+ continue
+ }
+ n++
+ if w.line.empty() {
+ w.collectUnmatched()
+ } else {
+ w.collectLogLine()
+ }
+ }
+}
+
+func (w *WebLog) collectLogLine() {
+ w.mx.Requests.Inc()
+ w.collectVhost()
+ w.collectPort()
+ w.collectReqScheme()
+ w.collectReqClient()
+ w.collectReqMethod()
+ w.collectReqURL()
+ w.collectReqProto()
+ w.collectRespCode()
+ w.collectReqSize()
+ w.collectRespSize()
+ w.collectReqProcTime()
+ w.collectUpsRespTime()
+ w.collectSSLProto()
+ w.collectSSLCipherSuite()
+ w.collectCustomFields()
+}
+
+func (w *WebLog) collectUnmatched() {
+ w.mx.Requests.Inc()
+ w.mx.ReqUnmatched.Inc()
+}
+
+func (w *WebLog) collectVhost() {
+ if !w.line.hasVhost() {
+ return
+ }
+ c, ok := w.mx.ReqVhost.GetP(w.line.vhost)
+ if !ok {
+ w.addDimToVhostChart(w.line.vhost)
+ }
+ c.Inc()
+}
+
+func (w *WebLog) collectPort() {
+ if !w.line.hasPort() {
+ return
+ }
+ c, ok := w.mx.ReqPort.GetP(w.line.port)
+ if !ok {
+ w.addDimToPortChart(w.line.port)
+ }
+ c.Inc()
+}
+
+func (w *WebLog) collectReqClient() {
+ if !w.line.hasReqClient() {
+ return
+ }
+ if strings.ContainsRune(w.line.reqClient, ':') {
+ w.mx.ReqIPv6.Inc()
+ w.mx.UniqueIPv6.Insert(w.line.reqClient)
+ return
+ }
+ // NOTE: count hostname as IPv4 address
+ w.mx.ReqIPv4.Inc()
+ w.mx.UniqueIPv4.Insert(w.line.reqClient)
+}
+
+func (w *WebLog) collectReqScheme() {
+ if !w.line.hasReqScheme() {
+ return
+ }
+ if w.line.reqScheme == "https" {
+ w.mx.ReqHTTPSScheme.Inc()
+ } else {
+ w.mx.ReqHTTPScheme.Inc()
+ }
+}
+
+func (w *WebLog) collectReqMethod() {
+ if !w.line.hasReqMethod() {
+ return
+ }
+ c, ok := w.mx.ReqMethod.GetP(w.line.reqMethod)
+ if !ok {
+ w.addDimToReqMethodChart(w.line.reqMethod)
+ }
+ c.Inc()
+}
+
+func (w *WebLog) collectReqURL() {
+ if !w.line.hasReqURL() {
+ return
+ }
+ for _, p := range w.urlPatterns {
+ if !p.MatchString(w.line.reqURL) {
+ continue
+ }
+ c, _ := w.mx.ReqURLPattern.GetP(p.name)
+ c.Inc()
+
+ w.collectURLPatternStats(p.name)
+ return
+ }
+}
+
+func (w *WebLog) collectReqProto() {
+ if !w.line.hasReqProto() {
+ return
+ }
+ c, ok := w.mx.ReqVersion.GetP(w.line.reqProto)
+ if !ok {
+ w.addDimToReqVersionChart(w.line.reqProto)
+ }
+ c.Inc()
+}
+
+func (w *WebLog) collectRespCode() {
+ if !w.line.hasRespCode() {
+ return
+ }
+
+ code := w.line.respCode
+ switch {
+ case code >= 100 && code < 300, code == 304, code == 401:
+ w.mx.ReqSuccess.Inc()
+ case code >= 300 && code < 400:
+ w.mx.ReqRedirect.Inc()
+ case code >= 400 && code < 500:
+ w.mx.ReqBad.Inc()
+ case code >= 500 && code < 600:
+ w.mx.ReqError.Inc()
+ }
+
+ switch code / 100 {
+ case 1:
+ w.mx.Resp1xx.Inc()
+ case 2:
+ w.mx.Resp2xx.Inc()
+ case 3:
+ w.mx.Resp3xx.Inc()
+ case 4:
+ w.mx.Resp4xx.Inc()
+ case 5:
+ w.mx.Resp5xx.Inc()
+ }
+
+ codeStr := strconv.Itoa(code)
+ c, ok := w.mx.RespCode.GetP(codeStr)
+ if !ok {
+ w.addDimToRespCodesChart(codeStr)
+ }
+ c.Inc()
+}
+
+func (w *WebLog) collectReqSize() {
+ if !w.line.hasReqSize() {
+ return
+ }
+ w.mx.BytesReceived.Add(float64(w.line.reqSize))
+}
+
+func (w *WebLog) collectRespSize() {
+ if !w.line.hasRespSize() {
+ return
+ }
+ w.mx.BytesSent.Add(float64(w.line.respSize))
+}
+
+func (w *WebLog) collectReqProcTime() {
+ if !w.line.hasReqProcTime() {
+ return
+ }
+ w.mx.ReqProcTime.Observe(w.line.reqProcTime)
+ if w.mx.ReqProcTimeHist == nil {
+ return
+ }
+ w.mx.ReqProcTimeHist.Observe(w.line.reqProcTime)
+}
+
+func (w *WebLog) collectUpsRespTime() {
+ if !w.line.hasUpsRespTime() {
+ return
+ }
+ w.mx.UpsRespTime.Observe(w.line.upsRespTime)
+ if w.mx.UpsRespTimeHist == nil {
+ return
+ }
+ w.mx.UpsRespTimeHist.Observe(w.line.upsRespTime)
+}
+
+func (w *WebLog) collectSSLProto() {
+ if !w.line.hasSSLProto() {
+ return
+ }
+ c, ok := w.mx.ReqSSLProto.GetP(w.line.sslProto)
+ if !ok {
+ w.addDimToSSLProtoChart(w.line.sslProto)
+ }
+ c.Inc()
+}
+
+func (w *WebLog) collectSSLCipherSuite() {
+ if !w.line.hasSSLCipherSuite() {
+ return
+ }
+ c, ok := w.mx.ReqSSLCipherSuite.GetP(w.line.sslCipherSuite)
+ if !ok {
+ w.addDimToSSLCipherSuiteChart(w.line.sslCipherSuite)
+ }
+ c.Inc()
+}
+
+func (w *WebLog) collectURLPatternStats(name string) {
+ v, ok := w.mx.URLPatternStats[name]
+ if !ok {
+ return
+ }
+ if w.line.hasRespCode() {
+ status := strconv.Itoa(w.line.respCode)
+ c, ok := v.RespCode.GetP(status)
+ if !ok {
+ w.addDimToURLPatternRespCodesChart(name, status)
+ }
+ c.Inc()
+ }
+
+ if w.line.hasReqMethod() {
+ c, ok := v.ReqMethod.GetP(w.line.reqMethod)
+ if !ok {
+ w.addDimToURLPatternReqMethodsChart(name, w.line.reqMethod)
+ }
+ c.Inc()
+ }
+
+ if w.line.hasReqSize() {
+ v.BytesReceived.Add(float64(w.line.reqSize))
+ }
+
+ if w.line.hasRespSize() {
+ v.BytesSent.Add(float64(w.line.respSize))
+ }
+
+ if w.line.hasReqProcTime() {
+ v.ReqProcTime.Observe(w.line.reqProcTime)
+ }
+}
+
+func (w *WebLog) collectCustomFields() {
+ if !w.line.hasCustomFields() {
+ return
+ }
+
+ for _, cv := range w.line.custom.values {
+ _, _ = cv.name, cv.value
+
+ if patterns, ok := w.customFields[cv.name]; ok {
+ for _, pattern := range patterns {
+ if !pattern.MatchString(cv.value) {
+ continue
+ }
+ v, ok := w.mx.ReqCustomField[cv.name]
+ if !ok {
+ break
+ }
+ c, _ := v.GetP(pattern.name)
+ c.Inc()
+ break
+ }
+ } else if histogram, ok := w.customTimeFields[cv.name]; ok {
+ v, ok := w.mx.ReqCustomTimeField[cv.name]
+ if !ok {
+ continue
+ }
+ ctf, err := strconv.ParseFloat(cv.value, 64)
+ if err != nil || !isTimeValid(ctf) {
+ continue
+ }
+ v.Time.Observe(ctf)
+ if histogram != nil {
+ v.TimeHist.Observe(ctf * timeMultiplier(cv.value))
+ }
+ } else if w.customNumericFields[cv.name] {
+ m, ok := w.mx.ReqCustomNumericField[cv.name]
+ if !ok {
+ continue
+ }
+ v, err := strconv.ParseFloat(cv.value, 64)
+ if err != nil {
+ continue
+ }
+ v *= float64(m.multiplier)
+ m.Summary.Observe(v)
+ }
+ }
+}
+
+func (w *WebLog) addDimToVhostChart(vhost string) {
+ chart := w.Charts().Get(reqByVhost.ID)
+ if chart == nil {
+ w.Warningf("add dimension: no '%s' chart", reqByVhost.ID)
+ return
+ }
+ dim := &Dim{
+ ID: "req_vhost_" + vhost,
+ Name: vhost,
+ Algo: module.Incremental,
+ }
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToPortChart(port string) {
+ chart := w.Charts().Get(reqByPort.ID)
+ if chart == nil {
+ w.Warningf("add dimension: no '%s' chart", reqByPort.ID)
+ return
+ }
+ dim := &Dim{
+ ID: "req_port_" + port,
+ Name: port,
+ Algo: module.Incremental,
+ }
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToReqMethodChart(method string) {
+ chart := w.Charts().Get(reqByMethod.ID)
+ if chart == nil {
+ w.Warningf("add dimension: no '%s' chart", reqByMethod.ID)
+ return
+ }
+ dim := &Dim{
+ ID: "req_method_" + method,
+ Name: method,
+ Algo: module.Incremental,
+ }
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToReqVersionChart(version string) {
+ chart := w.Charts().Get(reqByVersion.ID)
+ if chart == nil {
+ w.Warningf("add dimension: no '%s' chart", reqByVersion.ID)
+ return
+ }
+ dim := &Dim{
+ ID: "req_version_" + version,
+ Name: version,
+ Algo: module.Incremental,
+ }
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToSSLProtoChart(proto string) {
+ chart := w.Charts().Get(reqBySSLProto.ID)
+ if chart == nil {
+ chart = reqBySSLProto.Copy()
+ if err := w.Charts().Add(chart); err != nil {
+ w.Warning(err)
+ return
+ }
+ }
+ dim := &Dim{
+ ID: "req_ssl_proto_" + proto,
+ Name: proto,
+ Algo: module.Incremental,
+ }
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToSSLCipherSuiteChart(cipher string) {
+ chart := w.Charts().Get(reqBySSLCipherSuite.ID)
+ if chart == nil {
+ chart = reqBySSLCipherSuite.Copy()
+ if err := w.Charts().Add(chart); err != nil {
+ w.Warning(err)
+ return
+ }
+ }
+ dim := &Dim{
+ ID: "req_ssl_cipher_suite_" + cipher,
+ Name: cipher,
+ Algo: module.Incremental,
+ }
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToRespCodesChart(code string) {
+ chart := w.findRespCodesChart(code)
+ if chart == nil {
+ w.Warning("add dimension: cant find resp codes chart")
+ return
+ }
+ dim := &Dim{
+ ID: "resp_code_" + code,
+ Name: code,
+ Algo: module.Incremental,
+ }
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToURLPatternRespCodesChart(name, code string) {
+ id := fmt.Sprintf(urlPatternRespCodes.ID, name)
+ chart := w.Charts().Get(id)
+ if chart == nil {
+ w.Warningf("add dimension: no '%s' chart", id)
+ return
+ }
+ dim := &Dim{
+ ID: fmt.Sprintf("url_ptn_%s_resp_code_%s", name, code),
+ Name: code,
+ Algo: module.Incremental,
+ }
+
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) addDimToURLPatternReqMethodsChart(name, method string) {
+ id := fmt.Sprintf(urlPatternReqMethods.ID, name)
+ chart := w.Charts().Get(id)
+ if chart == nil {
+ w.Warningf("add dimension: no '%s' chart", id)
+ return
+ }
+ dim := &Dim{
+ ID: fmt.Sprintf("url_ptn_%s_req_method_%s", name, method),
+ Name: method,
+ Algo: module.Incremental,
+ }
+
+ if err := chart.AddDim(dim); err != nil {
+ w.Warning(err)
+ return
+ }
+ chart.MarkNotCreated()
+}
+
+func (w *WebLog) findRespCodesChart(code string) *Chart {
+ if !w.GroupRespCodes {
+ return w.Charts().Get(respCodes.ID)
+ }
+
+ var id string
+ switch class := code[:1]; class {
+ case "1":
+ id = respCodes1xx.ID
+ case "2":
+ id = respCodes2xx.ID
+ case "3":
+ id = respCodes3xx.ID
+ case "4":
+ id = respCodes4xx.ID
+ case "5":
+ id = respCodes5xx.ID
+ default:
+ return nil
+ }
+ return w.Charts().Get(id)
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-17 00:11:29 +0000