diff options
Diffstat (limited to '')
-rw-r--r-- | src/go/collectors/go.d.plugin/modules/weblog/collect.go | 560 |
1 files changed, 560 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/modules/weblog/collect.go b/src/go/collectors/go.d.plugin/modules/weblog/collect.go new file mode 100644 index 000000000..d24620774 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/weblog/collect.go @@ -0,0 +1,560 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package weblog + +import ( + "fmt" + "io" + "runtime" + "strconv" + "strings" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/logs" + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +func (w *WebLog) logPanicStackIfAny() { + err := recover() + if err == nil { + return + } + w.Errorf("[ERROR] %s\n", err) + for depth := 0; ; depth++ { + _, file, line, ok := runtime.Caller(depth) + if !ok { + break + } + w.Errorf("======> %d: %v:%d", depth, file, line) + } + panic(err) +} + +func (w *WebLog) collect() (map[string]int64, error) { + defer w.logPanicStackIfAny() + w.mx.reset() + + var mx map[string]int64 + + n, err := w.collectLogLines() + + if n > 0 || err == nil { + mx = stm.ToMap(w.mx) + } + return mx, err +} + +func (w *WebLog) collectLogLines() (int, error) { + logOnce := true + var n int + for { + w.line.reset() + err := w.parser.ReadLine(w.line) + if err != nil { + if err == io.EOF { + return n, nil + } + if !logs.IsParseError(err) { + return n, err + } + n++ + if logOnce { + w.Infof("unmatched line: %v (parser: %s)", err, w.parser.Info()) + logOnce = false + } + w.collectUnmatched() + continue + } + n++ + if w.line.empty() { + w.collectUnmatched() + } else { + w.collectLogLine() + } + } +} + +func (w *WebLog) collectLogLine() { + w.mx.Requests.Inc() + w.collectVhost() + w.collectPort() + w.collectReqScheme() + w.collectReqClient() + w.collectReqMethod() + w.collectReqURL() + w.collectReqProto() + w.collectRespCode() + w.collectReqSize() + w.collectRespSize() + w.collectReqProcTime() + w.collectUpsRespTime() + w.collectSSLProto() + w.collectSSLCipherSuite() + w.collectCustomFields() +} + +func (w *WebLog) collectUnmatched() { + w.mx.Requests.Inc() + w.mx.ReqUnmatched.Inc() +} + +func (w *WebLog) collectVhost() { + if !w.line.hasVhost() { + return + } + c, ok := w.mx.ReqVhost.GetP(w.line.vhost) + if !ok { + w.addDimToVhostChart(w.line.vhost) + } + c.Inc() +} + +func (w *WebLog) collectPort() { + if !w.line.hasPort() { + return + } + c, ok := w.mx.ReqPort.GetP(w.line.port) + if !ok { + w.addDimToPortChart(w.line.port) + } + c.Inc() +} + +func (w *WebLog) collectReqClient() { + if !w.line.hasReqClient() { + return + } + if strings.ContainsRune(w.line.reqClient, ':') { + w.mx.ReqIPv6.Inc() + w.mx.UniqueIPv6.Insert(w.line.reqClient) + return + } + // NOTE: count hostname as IPv4 address + w.mx.ReqIPv4.Inc() + w.mx.UniqueIPv4.Insert(w.line.reqClient) +} + +func (w *WebLog) collectReqScheme() { + if !w.line.hasReqScheme() { + return + } + if w.line.reqScheme == "https" { + w.mx.ReqHTTPSScheme.Inc() + } else { + w.mx.ReqHTTPScheme.Inc() + } +} + +func (w *WebLog) collectReqMethod() { + if !w.line.hasReqMethod() { + return + } + c, ok := w.mx.ReqMethod.GetP(w.line.reqMethod) + if !ok { + w.addDimToReqMethodChart(w.line.reqMethod) + } + c.Inc() +} + +func (w *WebLog) collectReqURL() { + if !w.line.hasReqURL() { + return + } + for _, p := range w.urlPatterns { + if !p.MatchString(w.line.reqURL) { + continue + } + c, _ := w.mx.ReqURLPattern.GetP(p.name) + c.Inc() + + w.collectURLPatternStats(p.name) + return + } +} + +func (w *WebLog) collectReqProto() { + if !w.line.hasReqProto() { + return + } + c, ok := w.mx.ReqVersion.GetP(w.line.reqProto) + if !ok { + w.addDimToReqVersionChart(w.line.reqProto) + } + c.Inc() +} + +func (w *WebLog) collectRespCode() { + if !w.line.hasRespCode() { + return + } + + code := w.line.respCode + switch { + case code >= 100 && code < 300, code == 304, code == 401: + w.mx.ReqSuccess.Inc() + case code >= 300 && code < 400: + w.mx.ReqRedirect.Inc() + case code >= 400 && code < 500: + w.mx.ReqBad.Inc() + case code >= 500 && code < 600: + w.mx.ReqError.Inc() + } + + switch code / 100 { + case 1: + w.mx.Resp1xx.Inc() + case 2: + w.mx.Resp2xx.Inc() + case 3: + w.mx.Resp3xx.Inc() + case 4: + w.mx.Resp4xx.Inc() + case 5: + w.mx.Resp5xx.Inc() + } + + codeStr := strconv.Itoa(code) + c, ok := w.mx.RespCode.GetP(codeStr) + if !ok { + w.addDimToRespCodesChart(codeStr) + } + c.Inc() +} + +func (w *WebLog) collectReqSize() { + if !w.line.hasReqSize() { + return + } + w.mx.BytesReceived.Add(float64(w.line.reqSize)) +} + +func (w *WebLog) collectRespSize() { + if !w.line.hasRespSize() { + return + } + w.mx.BytesSent.Add(float64(w.line.respSize)) +} + +func (w *WebLog) collectReqProcTime() { + if !w.line.hasReqProcTime() { + return + } + w.mx.ReqProcTime.Observe(w.line.reqProcTime) + if w.mx.ReqProcTimeHist == nil { + return + } + w.mx.ReqProcTimeHist.Observe(w.line.reqProcTime) +} + +func (w *WebLog) collectUpsRespTime() { + if !w.line.hasUpsRespTime() { + return + } + w.mx.UpsRespTime.Observe(w.line.upsRespTime) + if w.mx.UpsRespTimeHist == nil { + return + } + w.mx.UpsRespTimeHist.Observe(w.line.upsRespTime) +} + +func (w *WebLog) collectSSLProto() { + if !w.line.hasSSLProto() { + return + } + c, ok := w.mx.ReqSSLProto.GetP(w.line.sslProto) + if !ok { + w.addDimToSSLProtoChart(w.line.sslProto) + } + c.Inc() +} + +func (w *WebLog) collectSSLCipherSuite() { + if !w.line.hasSSLCipherSuite() { + return + } + c, ok := w.mx.ReqSSLCipherSuite.GetP(w.line.sslCipherSuite) + if !ok { + w.addDimToSSLCipherSuiteChart(w.line.sslCipherSuite) + } + c.Inc() +} + +func (w *WebLog) collectURLPatternStats(name string) { + v, ok := w.mx.URLPatternStats[name] + if !ok { + return + } + if w.line.hasRespCode() { + status := strconv.Itoa(w.line.respCode) + c, ok := v.RespCode.GetP(status) + if !ok { + w.addDimToURLPatternRespCodesChart(name, status) + } + c.Inc() + } + + if w.line.hasReqMethod() { + c, ok := v.ReqMethod.GetP(w.line.reqMethod) + if !ok { + w.addDimToURLPatternReqMethodsChart(name, w.line.reqMethod) + } + c.Inc() + } + + if w.line.hasReqSize() { + v.BytesReceived.Add(float64(w.line.reqSize)) + } + + if w.line.hasRespSize() { + v.BytesSent.Add(float64(w.line.respSize)) + } + + if w.line.hasReqProcTime() { + v.ReqProcTime.Observe(w.line.reqProcTime) + } +} + +func (w *WebLog) collectCustomFields() { + if !w.line.hasCustomFields() { + return + } + + for _, cv := range w.line.custom.values { + _, _ = cv.name, cv.value + + if patterns, ok := w.customFields[cv.name]; ok { + for _, pattern := range patterns { + if !pattern.MatchString(cv.value) { + continue + } + v, ok := w.mx.ReqCustomField[cv.name] + if !ok { + break + } + c, _ := v.GetP(pattern.name) + c.Inc() + break + } + } else if histogram, ok := w.customTimeFields[cv.name]; ok { + v, ok := w.mx.ReqCustomTimeField[cv.name] + if !ok { + continue + } + ctf, err := strconv.ParseFloat(cv.value, 64) + if err != nil || !isTimeValid(ctf) { + continue + } + v.Time.Observe(ctf) + if histogram != nil { + v.TimeHist.Observe(ctf * timeMultiplier(cv.value)) + } + } else if w.customNumericFields[cv.name] { + m, ok := w.mx.ReqCustomNumericField[cv.name] + if !ok { + continue + } + v, err := strconv.ParseFloat(cv.value, 64) + if err != nil { + continue + } + v *= float64(m.multiplier) + m.Summary.Observe(v) + } + } +} + +func (w *WebLog) addDimToVhostChart(vhost string) { + chart := w.Charts().Get(reqByVhost.ID) + if chart == nil { + w.Warningf("add dimension: no '%s' chart", reqByVhost.ID) + return + } + dim := &Dim{ + ID: "req_vhost_" + vhost, + Name: vhost, + Algo: module.Incremental, + } + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToPortChart(port string) { + chart := w.Charts().Get(reqByPort.ID) + if chart == nil { + w.Warningf("add dimension: no '%s' chart", reqByPort.ID) + return + } + dim := &Dim{ + ID: "req_port_" + port, + Name: port, + Algo: module.Incremental, + } + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToReqMethodChart(method string) { + chart := w.Charts().Get(reqByMethod.ID) + if chart == nil { + w.Warningf("add dimension: no '%s' chart", reqByMethod.ID) + return + } + dim := &Dim{ + ID: "req_method_" + method, + Name: method, + Algo: module.Incremental, + } + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToReqVersionChart(version string) { + chart := w.Charts().Get(reqByVersion.ID) + if chart == nil { + w.Warningf("add dimension: no '%s' chart", reqByVersion.ID) + return + } + dim := &Dim{ + ID: "req_version_" + version, + Name: version, + Algo: module.Incremental, + } + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToSSLProtoChart(proto string) { + chart := w.Charts().Get(reqBySSLProto.ID) + if chart == nil { + chart = reqBySSLProto.Copy() + if err := w.Charts().Add(chart); err != nil { + w.Warning(err) + return + } + } + dim := &Dim{ + ID: "req_ssl_proto_" + proto, + Name: proto, + Algo: module.Incremental, + } + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToSSLCipherSuiteChart(cipher string) { + chart := w.Charts().Get(reqBySSLCipherSuite.ID) + if chart == nil { + chart = reqBySSLCipherSuite.Copy() + if err := w.Charts().Add(chart); err != nil { + w.Warning(err) + return + } + } + dim := &Dim{ + ID: "req_ssl_cipher_suite_" + cipher, + Name: cipher, + Algo: module.Incremental, + } + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToRespCodesChart(code string) { + chart := w.findRespCodesChart(code) + if chart == nil { + w.Warning("add dimension: cant find resp codes chart") + return + } + dim := &Dim{ + ID: "resp_code_" + code, + Name: code, + Algo: module.Incremental, + } + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToURLPatternRespCodesChart(name, code string) { + id := fmt.Sprintf(urlPatternRespCodes.ID, name) + chart := w.Charts().Get(id) + if chart == nil { + w.Warningf("add dimension: no '%s' chart", id) + return + } + dim := &Dim{ + ID: fmt.Sprintf("url_ptn_%s_resp_code_%s", name, code), + Name: code, + Algo: module.Incremental, + } + + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) addDimToURLPatternReqMethodsChart(name, method string) { + id := fmt.Sprintf(urlPatternReqMethods.ID, name) + chart := w.Charts().Get(id) + if chart == nil { + w.Warningf("add dimension: no '%s' chart", id) + return + } + dim := &Dim{ + ID: fmt.Sprintf("url_ptn_%s_req_method_%s", name, method), + Name: method, + Algo: module.Incremental, + } + + if err := chart.AddDim(dim); err != nil { + w.Warning(err) + return + } + chart.MarkNotCreated() +} + +func (w *WebLog) findRespCodesChart(code string) *Chart { + if !w.GroupRespCodes { + return w.Charts().Get(respCodes.ID) + } + + var id string + switch class := code[:1]; class { + case "1": + id = respCodes1xx.ID + case "2": + id = respCodes2xx.ID + case "3": + id = respCodes3xx.ID + case "4": + id = respCodes4xx.ID + case "5": + id = respCodes5xx.ID + default: + return nil + } + return w.Charts().Get(id) +} |