From ccd992355df7192993c666236047820244914598 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Tue, 16 Apr 2024 21:19:13 +0200 Subject: Adding upstream version 1.21.8. Signed-off-by: Daniel Baumann --- src/runtime/metrics/description.go | 453 ++++++++++++++++++++++++++++++++ src/runtime/metrics/description_test.go | 158 +++++++++++ src/runtime/metrics/doc.go | 400 ++++++++++++++++++++++++++++ src/runtime/metrics/example_test.go | 96 +++++++ src/runtime/metrics/histogram.go | 33 +++ src/runtime/metrics/sample.go | 47 ++++ src/runtime/metrics/value.go | 69 +++++ 7 files changed, 1256 insertions(+) create mode 100644 src/runtime/metrics/description.go create mode 100644 src/runtime/metrics/description_test.go create mode 100644 src/runtime/metrics/doc.go create mode 100644 src/runtime/metrics/example_test.go create mode 100644 src/runtime/metrics/histogram.go create mode 100644 src/runtime/metrics/sample.go create mode 100644 src/runtime/metrics/value.go (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go new file mode 100644 index 0000000..745691b --- /dev/null +++ b/src/runtime/metrics/description.go @@ -0,0 +1,453 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +import "internal/godebugs" + +// Description describes a runtime metric. +type Description struct { + // Name is the full name of the metric which includes the unit. + // + // The format of the metric may be described by the following regular expression. + // + // ^(?P/[^:]+):(?P[^:*/]+(?:[*/][^:*/]+)*)$ + // + // The format splits the name into two components, separated by a colon: a path which always + // starts with a /, and a machine-parseable unit. The name may contain any valid Unicode + // codepoint in between / characters, but by convention will try to stick to lowercase + // characters and hyphens. An example of such a path might be "/memory/heap/free". + // + // The unit is by convention a series of lowercase English unit names (singular or plural) + // without prefixes delimited by '*' or '/'. The unit names may contain any valid Unicode + // codepoint that is not a delimiter. + // Examples of units might be "seconds", "bytes", "bytes/second", "cpu-seconds", + // "byte*cpu-seconds", and "bytes/second/second". + // + // For histograms, multiple units may apply. For instance, the units of the buckets and + // the count. By convention, for histograms, the units of the count are always "samples" + // with the type of sample evident by the metric's name, while the unit in the name + // specifies the buckets' unit. + // + // A complete name might look like "/memory/heap/free:bytes". + Name string + + // Description is an English language sentence describing the metric. + Description string + + // Kind is the kind of value for this metric. + // + // The purpose of this field is to allow users to filter out metrics whose values are + // types which their application may not understand. + Kind ValueKind + + // Cumulative is whether or not the metric is cumulative. If a cumulative metric is just + // a single number, then it increases monotonically. If the metric is a distribution, + // then each bucket count increases monotonically. + // + // This flag thus indicates whether or not it's useful to compute a rate from this value. + Cumulative bool +} + +// The English language descriptions below must be kept in sync with the +// descriptions of each metric in doc.go by running 'go generate'. +var allDesc = []Description{ + { + Name: "/cgo/go-to-c-calls:calls", + Description: "Count of calls made from Go to C by the current process.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/cpu/classes/gc/mark/assist:cpu-seconds", + Description: "Estimated total CPU time goroutines spent performing GC tasks " + + "to assist the GC and prevent it from falling behind the application. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/gc/mark/dedicated:cpu-seconds", + Description: "Estimated total CPU time spent performing GC tasks on " + + "processors (as defined by GOMAXPROCS) dedicated to those tasks. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/gc/mark/idle:cpu-seconds", + Description: "Estimated total CPU time spent performing GC tasks on " + + "spare CPU resources that the Go scheduler could not otherwise find " + + "a use for. This should be subtracted from the total GC CPU time to " + + "obtain a measure of compulsory GC CPU time. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/gc/pause:cpu-seconds", + Description: "Estimated total CPU time spent with the application paused by " + + "the GC. Even if only one thread is running during the pause, this is " + + "computed as GOMAXPROCS times the pause latency because nothing else " + + "can be executing. This is the exact sum of samples in /gc/pause:seconds " + + "if each sample is multiplied by GOMAXPROCS at the time it is taken. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/gc/total:cpu-seconds", + Description: "Estimated total CPU time spent performing GC tasks. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics. Sum of all metrics in /cpu/classes/gc.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/idle:cpu-seconds", + Description: "Estimated total available CPU time not spent executing any Go or Go runtime code. " + + "In other words, the part of /cpu/classes/total:cpu-seconds that was unused. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/scavenge/assist:cpu-seconds", + Description: "Estimated total CPU time spent returning unused memory to the " + + "underlying platform in response eagerly in response to memory pressure. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/scavenge/background:cpu-seconds", + Description: "Estimated total CPU time spent performing background tasks " + + "to return unused memory to the underlying platform. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/scavenge/total:cpu-seconds", + Description: "Estimated total CPU time spent performing tasks that return " + + "unused memory to the underlying platform. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics. Sum of all metrics in /cpu/classes/scavenge.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/total:cpu-seconds", + Description: "Estimated total available CPU time for user Go code " + + "or the Go runtime, as defined by GOMAXPROCS. In other words, GOMAXPROCS " + + "integrated over the wall-clock duration this process has been executing for. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics. Sum of all metrics in /cpu/classes.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/cpu/classes/user:cpu-seconds", + Description: "Estimated total CPU time spent running user Go code. This may " + + "also include some small amount of time spent in the Go runtime. " + + "This metric is an overestimate, and not directly comparable to " + + "system CPU time measurements. Compare only with other /cpu/classes " + + "metrics.", + Kind: KindFloat64, + Cumulative: true, + }, + { + Name: "/gc/cycles/automatic:gc-cycles", + Description: "Count of completed GC cycles generated by the Go runtime.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/cycles/forced:gc-cycles", + Description: "Count of completed GC cycles forced by the application.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/cycles/total:gc-cycles", + Description: "Count of all completed GC cycles.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/gogc:percent", + Description: "Heap size target percentage configured by the user, otherwise 100. This " + + "value is set by the GOGC environment variable, and the runtime/debug.SetGCPercent " + + "function.", + Kind: KindUint64, + }, + { + Name: "/gc/gomemlimit:bytes", + Description: "Go runtime memory limit configured by the user, otherwise " + + "math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and " + + "the runtime/debug.SetMemoryLimit function.", + Kind: KindUint64, + }, + { + Name: "/gc/heap/allocs-by-size:bytes", + Description: "Distribution of heap allocations by approximate size. " + + "Bucket counts increase monotonically. " + + "Note that this does not include tiny objects as defined by " + + "/gc/heap/tiny/allocs:objects, only tiny blocks.", + Kind: KindFloat64Histogram, + Cumulative: true, + }, + { + Name: "/gc/heap/allocs:bytes", + Description: "Cumulative sum of memory allocated to the heap by the application.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/heap/allocs:objects", + Description: "Cumulative count of heap allocations triggered by the application. " + + "Note that this does not include tiny objects as defined by " + + "/gc/heap/tiny/allocs:objects, only tiny blocks.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/heap/frees-by-size:bytes", + Description: "Distribution of freed heap allocations by approximate size. " + + "Bucket counts increase monotonically. " + + "Note that this does not include tiny objects as defined by " + + "/gc/heap/tiny/allocs:objects, only tiny blocks.", + Kind: KindFloat64Histogram, + Cumulative: true, + }, + { + Name: "/gc/heap/frees:bytes", + Description: "Cumulative sum of heap memory freed by the garbage collector.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/heap/frees:objects", + Description: "Cumulative count of heap allocations whose storage was freed " + + "by the garbage collector. " + + "Note that this does not include tiny objects as defined by " + + "/gc/heap/tiny/allocs:objects, only tiny blocks.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/heap/goal:bytes", + Description: "Heap size target for the end of the GC cycle.", + Kind: KindUint64, + }, + { + Name: "/gc/heap/live:bytes", + Description: "Heap memory occupied by live objects that were marked by the previous GC.", + Kind: KindUint64, + }, + { + Name: "/gc/heap/objects:objects", + Description: "Number of objects, live or unswept, occupying heap memory.", + Kind: KindUint64, + }, + { + Name: "/gc/heap/tiny/allocs:objects", + Description: "Count of small allocations that are packed together into blocks. " + + "These allocations are counted separately from other allocations " + + "because each individual allocation is not tracked by the runtime, " + + "only their block. Each block is already accounted for in " + + "allocs-by-size and frees-by-size.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/limiter/last-enabled:gc-cycle", + Description: "GC cycle the last time the GC CPU limiter was enabled. " + + "This metric is useful for diagnosing the root cause of an out-of-memory " + + "error, because the limiter trades memory for CPU time when the GC's CPU " + + "time gets too high. This is most likely to occur with use of SetMemoryLimit. " + + "The first GC cycle is cycle 1, so a value of 0 indicates that it was never enabled.", + Kind: KindUint64, + }, + { + Name: "/gc/pauses:seconds", + Description: "Distribution of individual GC-related stop-the-world pause latencies. Bucket counts increase monotonically.", + Kind: KindFloat64Histogram, + Cumulative: true, + }, + { + Name: "/gc/scan/globals:bytes", + Description: "The total amount of global variable space that is scannable.", + Kind: KindUint64, + }, + { + Name: "/gc/scan/heap:bytes", + Description: "The total amount of heap space that is scannable.", + Kind: KindUint64, + }, + { + Name: "/gc/scan/stack:bytes", + Description: "The number of bytes of stack that were scanned last GC cycle.", + Kind: KindUint64, + }, + { + Name: "/gc/scan/total:bytes", + Description: "The total amount space that is scannable. Sum of all metrics in /gc/scan.", + Kind: KindUint64, + }, + { + Name: "/gc/stack/starting-size:bytes", + Description: "The stack size of new goroutines.", + Kind: KindUint64, + Cumulative: false, + }, + { + Name: "/memory/classes/heap/free:bytes", + Description: "Memory that is completely free and eligible to be returned to the underlying system, " + + "but has not been. This metric is the runtime's estimate of free address space that is backed by " + + "physical memory.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/objects:bytes", + Description: "Memory occupied by live objects and dead objects that have not yet been marked free by the garbage collector.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/released:bytes", + Description: "Memory that is completely free and has been returned to the underlying system. This " + + "metric is the runtime's estimate of free address space that is still mapped into the process, " + + "but is not backed by physical memory.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/stacks:bytes", + Description: "Memory allocated from the heap that is reserved for stack space, whether or not it is currently in-use. " + + "Currently, this represents all stack memory for goroutines. It also includes all OS thread stacks in non-cgo programs. " + + "Note that stacks may be allocated differently in the future, and this may change.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/unused:bytes", + Description: "Memory that is reserved for heap objects but is not currently used to hold heap objects.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mcache/free:bytes", + Description: "Memory that is reserved for runtime mcache structures, but not in-use.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mcache/inuse:bytes", + Description: "Memory that is occupied by runtime mcache structures that are currently being used.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mspan/free:bytes", + Description: "Memory that is reserved for runtime mspan structures, but not in-use.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mspan/inuse:bytes", + Description: "Memory that is occupied by runtime mspan structures that are currently being used.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/other:bytes", + Description: "Memory that is reserved for or used to hold runtime metadata.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/os-stacks:bytes", + Description: "Stack memory allocated by the underlying operating system. " + + "In non-cgo programs this metric is currently zero. This may change in the future." + + "In cgo programs this metric includes OS thread stacks allocated directly from the OS. " + + "Currently, this only accounts for one stack in c-shared and c-archive build modes, " + + "and other sources of stacks from the OS are not measured. This too may change in the future.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/other:bytes", + Description: "Memory used by execution trace buffers, structures for debugging the runtime, finalizer and profiler specials, and more.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/profiling/buckets:bytes", + Description: "Memory that is used by the stack trace hash map used for profiling.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/total:bytes", + Description: "All memory mapped by the Go runtime into the current process as read-write. Note that this does not include memory mapped by code called via cgo or via the syscall package. Sum of all metrics in /memory/classes.", + Kind: KindUint64, + }, + { + Name: "/sched/gomaxprocs:threads", + Description: "The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously.", + Kind: KindUint64, + }, + { + Name: "/sched/goroutines:goroutines", + Description: "Count of live goroutines.", + Kind: KindUint64, + }, + { + Name: "/sched/latencies:seconds", + Description: "Distribution of the time goroutines have spent in the scheduler in a runnable state before actually running. Bucket counts increase monotonically.", + Kind: KindFloat64Histogram, + Cumulative: true, + }, + { + Name: "/sync/mutex/wait/total:seconds", + Description: "Approximate cumulative time goroutines have spent blocked on a sync.Mutex or sync.RWMutex. This metric is useful for identifying global changes in lock contention. Collect a mutex or block profile using the runtime/pprof package for more detailed contention data.", + Kind: KindFloat64, + Cumulative: true, + }, +} + +func init() { + // Insert all the non-default-reporting GODEBUGs into the table, + // preserving the overall sort order. + i := 0 + for i < len(allDesc) && allDesc[i].Name < "/godebug/" { + i++ + } + more := make([]Description, i, len(allDesc)+len(godebugs.All)) + copy(more, allDesc) + for _, info := range godebugs.All { + if !info.Opaque { + more = append(more, Description{ + Name: "/godebug/non-default-behavior/" + info.Name + ":events", + Description: "The number of non-default behaviors executed by the " + + info.Package + " package " + "due to a non-default " + + "GODEBUG=" + info.Name + "=... setting.", + Kind: KindUint64, + Cumulative: true, + }) + } + } + allDesc = append(more, allDesc[i:]...) +} + +// All returns a slice of containing metric descriptions for all supported metrics. +func All() []Description { + return allDesc +} diff --git a/src/runtime/metrics/description_test.go b/src/runtime/metrics/description_test.go new file mode 100644 index 0000000..4fc6523 --- /dev/null +++ b/src/runtime/metrics/description_test.go @@ -0,0 +1,158 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics_test + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/doc" + "go/doc/comment" + "go/format" + "go/parser" + "go/token" + "internal/diff" + "os" + "regexp" + "runtime/metrics" + "sort" + "strings" + "testing" + _ "unsafe" +) + +// Implemented in the runtime. +// +//go:linkname runtime_readMetricNames +func runtime_readMetricNames() []string + +func TestNames(t *testing.T) { + // Note that this regexp is promised in the package docs for Description. Do not change. + r := regexp.MustCompile("^(?P/[^:]+):(?P[^:*/]+(?:[*/][^:*/]+)*)$") + all := metrics.All() + for i, d := range all { + if !r.MatchString(d.Name) { + t.Errorf("name %q does not match regexp %#q", d.Name, r) + } + if i > 0 && all[i-1].Name >= all[i].Name { + t.Fatalf("allDesc not sorted: %s ≥ %s", all[i-1].Name, all[i].Name) + } + } + + names := runtime_readMetricNames() + sort.Strings(names) + samples := make([]metrics.Sample, len(names)) + for i, name := range names { + samples[i].Name = name + } + metrics.Read(samples) + + for _, d := range all { + for len(samples) > 0 && samples[0].Name < d.Name { + t.Errorf("%s: reported by runtime but not listed in All", samples[0].Name) + samples = samples[1:] + } + if len(samples) == 0 || d.Name < samples[0].Name { + t.Errorf("%s: listed in All but not reported by runtime", d.Name) + continue + } + if samples[0].Value.Kind() != d.Kind { + t.Errorf("%s: runtime reports %v but All reports %v", d.Name, samples[0].Value.Kind(), d.Kind) + } + samples = samples[1:] + } +} + +func wrap(prefix, text string, width int) string { + doc := &comment.Doc{Content: []comment.Block{&comment.Paragraph{Text: []comment.Text{comment.Plain(text)}}}} + pr := &comment.Printer{TextPrefix: prefix, TextWidth: width} + return string(pr.Text(doc)) +} + +func formatDesc(t *testing.T) string { + var b strings.Builder + for i, d := range metrics.All() { + if i > 0 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "%s\n", d.Name) + fmt.Fprintf(&b, "%s", wrap("\t", d.Description, 80-2*8)) + } + return b.String() +} + +var generate = flag.Bool("generate", false, "update doc.go for go generate") + +func TestDocs(t *testing.T) { + want := formatDesc(t) + + src, err := os.ReadFile("doc.go") + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "doc.go", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + fdoc := f.Doc + if fdoc == nil { + t.Fatal("no doc comment in doc.go") + } + pkg, err := doc.NewFromFiles(fset, []*ast.File{f}, "runtime/metrics") + if err != nil { + t.Fatal(err) + } + if pkg.Doc == "" { + t.Fatal("doc.NewFromFiles lost doc comment") + } + doc := new(comment.Parser).Parse(pkg.Doc) + expectCode := false + foundCode := false + updated := false + for _, block := range doc.Content { + switch b := block.(type) { + case *comment.Heading: + expectCode = false + if b.Text[0] == comment.Plain("Supported metrics") { + expectCode = true + } + case *comment.Code: + if expectCode { + foundCode = true + if b.Text != want { + if !*generate { + t.Fatalf("doc comment out of date; use go generate to rebuild\n%s", diff.Diff("old", []byte(b.Text), "want", []byte(want))) + } + b.Text = want + updated = true + } + } + } + } + + if !foundCode { + t.Fatalf("did not find Supported metrics list in doc.go") + } + if updated { + fmt.Fprintf(os.Stderr, "go test -generate: writing new doc.go\n") + var buf bytes.Buffer + buf.Write(src[:fdoc.Pos()-f.FileStart]) + buf.WriteString("/*\n") + buf.Write(new(comment.Printer).Comment(doc)) + buf.WriteString("*/") + buf.Write(src[fdoc.End()-f.FileStart:]) + src, err := format.Source(buf.Bytes()) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile("doc.go", src, 0666); err != nil { + t.Fatal(err) + } + } else if *generate { + fmt.Fprintf(os.Stderr, "go test -generate: doc.go already up-to-date\n") + } +} diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go new file mode 100644 index 0000000..55d1f65 --- /dev/null +++ b/src/runtime/metrics/doc.go @@ -0,0 +1,400 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Note: run 'go generate' (which will run 'go test -generate') to update the "Supported metrics" list. +//go:generate go test -run=Docs -generate + +/* +Package metrics provides a stable interface to access implementation-defined +metrics exported by the Go runtime. This package is similar to existing functions +like [runtime.ReadMemStats] and [debug.ReadGCStats], but significantly more general. + +The set of metrics defined by this package may evolve as the runtime itself +evolves, and also enables variation across Go implementations, whose relevant +metric sets may not intersect. + +# Interface + +Metrics are designated by a string key, rather than, for example, a field name in +a struct. The full list of supported metrics is always available in the slice of +Descriptions returned by All. Each Description also includes useful information +about the metric. + +Thus, users of this API are encouraged to sample supported metrics defined by the +slice returned by All to remain compatible across Go versions. Of course, situations +arise where reading specific metrics is critical. For these cases, users are +encouraged to use build tags, and although metrics may be deprecated and removed, +users should consider this to be an exceptional and rare event, coinciding with a +very large change in a particular Go implementation. + +Each metric key also has a "kind" that describes the format of the metric's value. +In the interest of not breaking users of this package, the "kind" for a given metric +is guaranteed not to change. If it must change, then a new metric will be introduced +with a new key and a new "kind." + +# Metric key format + +As mentioned earlier, metric keys are strings. Their format is simple and well-defined, +designed to be both human and machine readable. It is split into two components, +separated by a colon: a rooted path and a unit. The choice to include the unit in +the key is motivated by compatibility: if a metric's unit changes, its semantics likely +did also, and a new key should be introduced. + +For more details on the precise definition of the metric key's path and unit formats, see +the documentation of the Name field of the Description struct. + +# A note about floats + +This package supports metrics whose values have a floating-point representation. In +order to improve ease-of-use, this package promises to never produce the following +classes of floating-point values: NaN, infinity. + +# Supported metrics + +Below is the full list of supported metrics, ordered lexicographically. + + /cgo/go-to-c-calls:calls + Count of calls made from Go to C by the current process. + + /cpu/classes/gc/mark/assist:cpu-seconds + Estimated total CPU time goroutines spent performing GC + tasks to assist the GC and prevent it from falling behind the + application. This metric is an overestimate, and not directly + comparable to system CPU time measurements. Compare only with + other /cpu/classes metrics. + + /cpu/classes/gc/mark/dedicated:cpu-seconds + Estimated total CPU time spent performing GC tasks on processors + (as defined by GOMAXPROCS) dedicated to those tasks. This metric + is an overestimate, and not directly comparable to system CPU + time measurements. Compare only with other /cpu/classes metrics. + + /cpu/classes/gc/mark/idle:cpu-seconds + Estimated total CPU time spent performing GC tasks on spare CPU + resources that the Go scheduler could not otherwise find a use + for. This should be subtracted from the total GC CPU time to + obtain a measure of compulsory GC CPU time. This metric is an + overestimate, and not directly comparable to system CPU time + measurements. Compare only with other /cpu/classes metrics. + + /cpu/classes/gc/pause:cpu-seconds + Estimated total CPU time spent with the application paused by + the GC. Even if only one thread is running during the pause, + this is computed as GOMAXPROCS times the pause latency because + nothing else can be executing. This is the exact sum of samples + in /gc/pause:seconds if each sample is multiplied by GOMAXPROCS + at the time it is taken. This metric is an overestimate, + and not directly comparable to system CPU time measurements. + Compare only with other /cpu/classes metrics. + + /cpu/classes/gc/total:cpu-seconds + Estimated total CPU time spent performing GC tasks. This metric + is an overestimate, and not directly comparable to system CPU + time measurements. Compare only with other /cpu/classes metrics. + Sum of all metrics in /cpu/classes/gc. + + /cpu/classes/idle:cpu-seconds + Estimated total available CPU time not spent executing + any Go or Go runtime code. In other words, the part of + /cpu/classes/total:cpu-seconds that was unused. This metric is + an overestimate, and not directly comparable to system CPU time + measurements. Compare only with other /cpu/classes metrics. + + /cpu/classes/scavenge/assist:cpu-seconds + Estimated total CPU time spent returning unused memory to the + underlying platform in response eagerly in response to memory + pressure. This metric is an overestimate, and not directly + comparable to system CPU time measurements. Compare only with + other /cpu/classes metrics. + + /cpu/classes/scavenge/background:cpu-seconds + Estimated total CPU time spent performing background tasks to + return unused memory to the underlying platform. This metric is + an overestimate, and not directly comparable to system CPU time + measurements. Compare only with other /cpu/classes metrics. + + /cpu/classes/scavenge/total:cpu-seconds + Estimated total CPU time spent performing tasks that return + unused memory to the underlying platform. This metric is an + overestimate, and not directly comparable to system CPU time + measurements. Compare only with other /cpu/classes metrics. + Sum of all metrics in /cpu/classes/scavenge. + + /cpu/classes/total:cpu-seconds + Estimated total available CPU time for user Go code or the Go + runtime, as defined by GOMAXPROCS. In other words, GOMAXPROCS + integrated over the wall-clock duration this process has been + executing for. This metric is an overestimate, and not directly + comparable to system CPU time measurements. Compare only with + other /cpu/classes metrics. Sum of all metrics in /cpu/classes. + + /cpu/classes/user:cpu-seconds + Estimated total CPU time spent running user Go code. This may + also include some small amount of time spent in the Go runtime. + This metric is an overestimate, and not directly comparable + to system CPU time measurements. Compare only with other + /cpu/classes metrics. + + /gc/cycles/automatic:gc-cycles + Count of completed GC cycles generated by the Go runtime. + + /gc/cycles/forced:gc-cycles + Count of completed GC cycles forced by the application. + + /gc/cycles/total:gc-cycles + Count of all completed GC cycles. + + /gc/gogc:percent + Heap size target percentage configured by the user, otherwise + 100. This value is set by the GOGC environment variable, and the + runtime/debug.SetGCPercent function. + + /gc/gomemlimit:bytes + Go runtime memory limit configured by the user, otherwise + math.MaxInt64. This value is set by the GOMEMLIMIT environment + variable, and the runtime/debug.SetMemoryLimit function. + + /gc/heap/allocs-by-size:bytes + Distribution of heap allocations by approximate size. + Bucket counts increase monotonically. Note that this does not + include tiny objects as defined by /gc/heap/tiny/allocs:objects, + only tiny blocks. + + /gc/heap/allocs:bytes + Cumulative sum of memory allocated to the heap by the + application. + + /gc/heap/allocs:objects + Cumulative count of heap allocations triggered by the + application. Note that this does not include tiny objects as + defined by /gc/heap/tiny/allocs:objects, only tiny blocks. + + /gc/heap/frees-by-size:bytes + Distribution of freed heap allocations by approximate size. + Bucket counts increase monotonically. Note that this does not + include tiny objects as defined by /gc/heap/tiny/allocs:objects, + only tiny blocks. + + /gc/heap/frees:bytes + Cumulative sum of heap memory freed by the garbage collector. + + /gc/heap/frees:objects + Cumulative count of heap allocations whose storage was freed + by the garbage collector. Note that this does not include tiny + objects as defined by /gc/heap/tiny/allocs:objects, only tiny + blocks. + + /gc/heap/goal:bytes + Heap size target for the end of the GC cycle. + + /gc/heap/live:bytes + Heap memory occupied by live objects that were marked by the + previous GC. + + /gc/heap/objects:objects + Number of objects, live or unswept, occupying heap memory. + + /gc/heap/tiny/allocs:objects + Count of small allocations that are packed together into blocks. + These allocations are counted separately from other allocations + because each individual allocation is not tracked by the + runtime, only their block. Each block is already accounted for + in allocs-by-size and frees-by-size. + + /gc/limiter/last-enabled:gc-cycle + GC cycle the last time the GC CPU limiter was enabled. + This metric is useful for diagnosing the root cause of an + out-of-memory error, because the limiter trades memory for CPU + time when the GC's CPU time gets too high. This is most likely + to occur with use of SetMemoryLimit. The first GC cycle is cycle + 1, so a value of 0 indicates that it was never enabled. + + /gc/pauses:seconds + Distribution of individual GC-related stop-the-world pause + latencies. Bucket counts increase monotonically. + + /gc/scan/globals:bytes + The total amount of global variable space that is scannable. + + /gc/scan/heap:bytes + The total amount of heap space that is scannable. + + /gc/scan/stack:bytes + The number of bytes of stack that were scanned last GC cycle. + + /gc/scan/total:bytes + The total amount space that is scannable. Sum of all metrics in + /gc/scan. + + /gc/stack/starting-size:bytes + The stack size of new goroutines. + + /godebug/non-default-behavior/execerrdot:events + The number of non-default behaviors executed by the os/exec + package due to a non-default GODEBUG=execerrdot=... setting. + + /godebug/non-default-behavior/gocachehash:events + The number of non-default behaviors executed by the cmd/go + package due to a non-default GODEBUG=gocachehash=... setting. + + /godebug/non-default-behavior/gocachetest:events + The number of non-default behaviors executed by the cmd/go + package due to a non-default GODEBUG=gocachetest=... setting. + + /godebug/non-default-behavior/gocacheverify:events + The number of non-default behaviors executed by the cmd/go + package due to a non-default GODEBUG=gocacheverify=... setting. + + /godebug/non-default-behavior/http2client:events + The number of non-default behaviors executed by the net/http + package due to a non-default GODEBUG=http2client=... setting. + + /godebug/non-default-behavior/http2server:events + The number of non-default behaviors executed by the net/http + package due to a non-default GODEBUG=http2server=... setting. + + /godebug/non-default-behavior/installgoroot:events + The number of non-default behaviors executed by the go/build + package due to a non-default GODEBUG=installgoroot=... setting. + + /godebug/non-default-behavior/jstmpllitinterp:events + The number of non-default behaviors executed by + the html/template package due to a non-default + GODEBUG=jstmpllitinterp=... setting. + + /godebug/non-default-behavior/multipartmaxheaders:events + The number of non-default behaviors executed by + the mime/multipart package due to a non-default + GODEBUG=multipartmaxheaders=... setting. + + /godebug/non-default-behavior/multipartmaxparts:events + The number of non-default behaviors executed by + the mime/multipart package due to a non-default + GODEBUG=multipartmaxparts=... setting. + + /godebug/non-default-behavior/multipathtcp:events + The number of non-default behaviors executed by the net package + due to a non-default GODEBUG=multipathtcp=... setting. + + /godebug/non-default-behavior/panicnil:events + The number of non-default behaviors executed by the runtime + package due to a non-default GODEBUG=panicnil=... setting. + + /godebug/non-default-behavior/randautoseed:events + The number of non-default behaviors executed by the math/rand + package due to a non-default GODEBUG=randautoseed=... setting. + + /godebug/non-default-behavior/tarinsecurepath:events + The number of non-default behaviors executed by the archive/tar + package due to a non-default GODEBUG=tarinsecurepath=... + setting. + + /godebug/non-default-behavior/tlsmaxrsasize:events + The number of non-default behaviors executed by the crypto/tls + package due to a non-default GODEBUG=tlsmaxrsasize=... setting. + + /godebug/non-default-behavior/x509sha1:events + The number of non-default behaviors executed by the crypto/x509 + package due to a non-default GODEBUG=x509sha1=... setting. + + /godebug/non-default-behavior/x509usefallbackroots:events + The number of non-default behaviors executed by the crypto/x509 + package due to a non-default GODEBUG=x509usefallbackroots=... + setting. + + /godebug/non-default-behavior/zipinsecurepath:events + The number of non-default behaviors executed by the archive/zip + package due to a non-default GODEBUG=zipinsecurepath=... + setting. + + /memory/classes/heap/free:bytes + Memory that is completely free and eligible to be returned to + the underlying system, but has not been. This metric is the + runtime's estimate of free address space that is backed by + physical memory. + + /memory/classes/heap/objects:bytes + Memory occupied by live objects and dead objects that have not + yet been marked free by the garbage collector. + + /memory/classes/heap/released:bytes + Memory that is completely free and has been returned to the + underlying system. This metric is the runtime's estimate of free + address space that is still mapped into the process, but is not + backed by physical memory. + + /memory/classes/heap/stacks:bytes + Memory allocated from the heap that is reserved for stack space, + whether or not it is currently in-use. Currently, this + represents all stack memory for goroutines. It also includes all + OS thread stacks in non-cgo programs. Note that stacks may be + allocated differently in the future, and this may change. + + /memory/classes/heap/unused:bytes + Memory that is reserved for heap objects but is not currently + used to hold heap objects. + + /memory/classes/metadata/mcache/free:bytes + Memory that is reserved for runtime mcache structures, but not + in-use. + + /memory/classes/metadata/mcache/inuse:bytes + Memory that is occupied by runtime mcache structures that are + currently being used. + + /memory/classes/metadata/mspan/free:bytes + Memory that is reserved for runtime mspan structures, but not + in-use. + + /memory/classes/metadata/mspan/inuse:bytes + Memory that is occupied by runtime mspan structures that are + currently being used. + + /memory/classes/metadata/other:bytes + Memory that is reserved for or used to hold runtime metadata. + + /memory/classes/os-stacks:bytes + Stack memory allocated by the underlying operating system. + In non-cgo programs this metric is currently zero. This may + change in the future.In cgo programs this metric includes + OS thread stacks allocated directly from the OS. Currently, + this only accounts for one stack in c-shared and c-archive build + modes, and other sources of stacks from the OS are not measured. + This too may change in the future. + + /memory/classes/other:bytes + Memory used by execution trace buffers, structures for debugging + the runtime, finalizer and profiler specials, and more. + + /memory/classes/profiling/buckets:bytes + Memory that is used by the stack trace hash map used for + profiling. + + /memory/classes/total:bytes + All memory mapped by the Go runtime into the current process + as read-write. Note that this does not include memory mapped + by code called via cgo or via the syscall package. Sum of all + metrics in /memory/classes. + + /sched/gomaxprocs:threads + The current runtime.GOMAXPROCS setting, or the number of + operating system threads that can execute user-level Go code + simultaneously. + + /sched/goroutines:goroutines + Count of live goroutines. + + /sched/latencies:seconds + Distribution of the time goroutines have spent in the scheduler + in a runnable state before actually running. Bucket counts + increase monotonically. + + /sync/mutex/wait/total:seconds + Approximate cumulative time goroutines have spent blocked + on a sync.Mutex or sync.RWMutex. This metric is useful for + identifying global changes in lock contention. Collect a mutex + or block profile using the runtime/pprof package for more + detailed contention data. +*/ +package metrics diff --git a/src/runtime/metrics/example_test.go b/src/runtime/metrics/example_test.go new file mode 100644 index 0000000..624d9d8 --- /dev/null +++ b/src/runtime/metrics/example_test.go @@ -0,0 +1,96 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics_test + +import ( + "fmt" + "runtime/metrics" +) + +func ExampleRead_readingOneMetric() { + // Name of the metric we want to read. + const myMetric = "/memory/classes/heap/free:bytes" + + // Create a sample for the metric. + sample := make([]metrics.Sample, 1) + sample[0].Name = myMetric + + // Sample the metric. + metrics.Read(sample) + + // Check if the metric is actually supported. + // If it's not, the resulting value will always have + // kind KindBad. + if sample[0].Value.Kind() == metrics.KindBad { + panic(fmt.Sprintf("metric %q no longer supported", myMetric)) + } + + // Handle the result. + // + // It's OK to assume a particular Kind for a metric; + // they're guaranteed not to change. + freeBytes := sample[0].Value.Uint64() + + fmt.Printf("free but not released memory: %d\n", freeBytes) +} + +func ExampleRead_readingAllMetrics() { + // Get descriptions for all supported metrics. + descs := metrics.All() + + // Create a sample for each metric. + samples := make([]metrics.Sample, len(descs)) + for i := range samples { + samples[i].Name = descs[i].Name + } + + // Sample the metrics. Re-use the samples slice if you can! + metrics.Read(samples) + + // Iterate over all results. + for _, sample := range samples { + // Pull out the name and value. + name, value := sample.Name, sample.Value + + // Handle each sample. + switch value.Kind() { + case metrics.KindUint64: + fmt.Printf("%s: %d\n", name, value.Uint64()) + case metrics.KindFloat64: + fmt.Printf("%s: %f\n", name, value.Float64()) + case metrics.KindFloat64Histogram: + // The histogram may be quite large, so let's just pull out + // a crude estimate for the median for the sake of this example. + fmt.Printf("%s: %f\n", name, medianBucket(value.Float64Histogram())) + case metrics.KindBad: + // This should never happen because all metrics are supported + // by construction. + panic("bug in runtime/metrics package!") + default: + // This may happen as new metrics get added. + // + // The safest thing to do here is to simply log it somewhere + // as something to look into, but ignore it for now. + // In the worst case, you might temporarily miss out on a new metric. + fmt.Printf("%s: unexpected metric Kind: %v\n", name, value.Kind()) + } + } +} + +func medianBucket(h *metrics.Float64Histogram) float64 { + total := uint64(0) + for _, count := range h.Counts { + total += count + } + thresh := total / 2 + total = 0 + for i, count := range h.Counts { + total += count + if total >= thresh { + return h.Buckets[i] + } + } + panic("should not happen") +} diff --git a/src/runtime/metrics/histogram.go b/src/runtime/metrics/histogram.go new file mode 100644 index 0000000..956422b --- /dev/null +++ b/src/runtime/metrics/histogram.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +// Float64Histogram represents a distribution of float64 values. +type Float64Histogram struct { + // Counts contains the weights for each histogram bucket. + // + // Given N buckets, Count[n] is the weight of the range + // [bucket[n], bucket[n+1]), for 0 <= n < N. + Counts []uint64 + + // Buckets contains the boundaries of the histogram buckets, in increasing order. + // + // Buckets[0] is the inclusive lower bound of the minimum bucket while + // Buckets[len(Buckets)-1] is the exclusive upper bound of the maximum bucket. + // Hence, there are len(Buckets)-1 counts. Furthermore, len(Buckets) != 1, always, + // since at least two boundaries are required to describe one bucket (and 0 + // boundaries are used to describe 0 buckets). + // + // Buckets[0] is permitted to have value -Inf and Buckets[len(Buckets)-1] is + // permitted to have value Inf. + // + // For a given metric name, the value of Buckets is guaranteed not to change + // between calls until program exit. + // + // This slice value is permitted to alias with other Float64Histograms' Buckets + // fields, so the values within should only ever be read. If they need to be + // modified, the user must make a copy. + Buckets []float64 +} diff --git a/src/runtime/metrics/sample.go b/src/runtime/metrics/sample.go new file mode 100644 index 0000000..4cf8cdf --- /dev/null +++ b/src/runtime/metrics/sample.go @@ -0,0 +1,47 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +import ( + _ "runtime" // depends on the runtime via a linkname'd function + "unsafe" +) + +// Sample captures a single metric sample. +type Sample struct { + // Name is the name of the metric sampled. + // + // It must correspond to a name in one of the metric descriptions + // returned by All. + Name string + + // Value is the value of the metric sample. + Value Value +} + +// Implemented in the runtime. +func runtime_readMetrics(unsafe.Pointer, int, int) + +// Read populates each Value field in the given slice of metric samples. +// +// Desired metrics should be present in the slice with the appropriate name. +// The user of this API is encouraged to re-use the same slice between calls for +// efficiency, but is not required to do so. +// +// Note that re-use has some caveats. Notably, Values should not be read or +// manipulated while a Read with that value is outstanding; that is a data race. +// This property includes pointer-typed Values (for example, Float64Histogram) +// whose underlying storage will be reused by Read when possible. To safely use +// such values in a concurrent setting, all data must be deep-copied. +// +// It is safe to execute multiple Read calls concurrently, but their arguments +// must share no underlying memory. When in doubt, create a new []Sample from +// scratch, which is always safe, though may be inefficient. +// +// Sample values with names not appearing in All will have their Value populated +// as KindBad to indicate that the name is unknown. +func Read(m []Sample) { + runtime_readMetrics(unsafe.Pointer(&m[0]), len(m), cap(m)) +} diff --git a/src/runtime/metrics/value.go b/src/runtime/metrics/value.go new file mode 100644 index 0000000..ed9a33d --- /dev/null +++ b/src/runtime/metrics/value.go @@ -0,0 +1,69 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +import ( + "math" + "unsafe" +) + +// ValueKind is a tag for a metric Value which indicates its type. +type ValueKind int + +const ( + // KindBad indicates that the Value has no type and should not be used. + KindBad ValueKind = iota + + // KindUint64 indicates that the type of the Value is a uint64. + KindUint64 + + // KindFloat64 indicates that the type of the Value is a float64. + KindFloat64 + + // KindFloat64Histogram indicates that the type of the Value is a *Float64Histogram. + KindFloat64Histogram +) + +// Value represents a metric value returned by the runtime. +type Value struct { + kind ValueKind + scalar uint64 // contains scalar values for scalar Kinds. + pointer unsafe.Pointer // contains non-scalar values. +} + +// Kind returns the tag representing the kind of value this is. +func (v Value) Kind() ValueKind { + return v.kind +} + +// Uint64 returns the internal uint64 value for the metric. +// +// If v.Kind() != KindUint64, this method panics. +func (v Value) Uint64() uint64 { + if v.kind != KindUint64 { + panic("called Uint64 on non-uint64 metric value") + } + return v.scalar +} + +// Float64 returns the internal float64 value for the metric. +// +// If v.Kind() != KindFloat64, this method panics. +func (v Value) Float64() float64 { + if v.kind != KindFloat64 { + panic("called Float64 on non-float64 metric value") + } + return math.Float64frombits(v.scalar) +} + +// Float64Histogram returns the internal *Float64Histogram value for the metric. +// +// If v.Kind() != KindFloat64Histogram, this method panics. +func (v Value) Float64Histogram() *Float64Histogram { + if v.kind != KindFloat64Histogram { + panic("called Float64Histogram on non-Float64Histogram metric value") + } + return (*Float64Histogram)(v.pointer) +} -- cgit v1.2.3