Adding upstream version 1.21.8.upstream/1.21.8

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 422 insertions, 0 deletions

diff --git a/src/cmd/covdata/metamerge.go b/src/cmd/covdata/metamerge.go
new file mode 100644
index 0000000..6c68e0c
--- /dev/null
+++ b/src/cmd/covdata/metamerge.go
@@ -0,0 +1,422 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// This file contains functions and apis that support merging of
+// meta-data information.  It helps implement the "merge", "subtract",
+// and "intersect" subcommands.
+
+import (
+	"crypto/md5"
+	"fmt"
+	"internal/coverage"
+	"internal/coverage/calloc"
+	"internal/coverage/cmerge"
+	"internal/coverage/decodecounter"
+	"internal/coverage/decodemeta"
+	"internal/coverage/encodecounter"
+	"internal/coverage/encodemeta"
+	"internal/coverage/slicewriter"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+	"unsafe"
+)
+
+// metaMerge provides state and methods to help manage the process
+// of selecting or merging meta data files. There are three cases
+// of interest here: the "-pcombine" flag provided by merge, the
+// "-pkg" option provided by all merge/subtract/intersect, and
+// a regular vanilla merge with no package selection
+//
+// In the -pcombine case, we're essentially glomming together all the
+// meta-data for all packages and all functions, meaning that
+// everything we see in a given package needs to be added into the
+// meta-data file builder; we emit a single meta-data file at the end
+// of the run.
+//
+// In the -pkg case, we will typically emit a single meta-data file
+// per input pod, where that new meta-data file contains entries for
+// just the selected packages.
+//
+// In the third case (vanilla merge with no combining or package
+// selection) we can carry over meta-data files without touching them
+// at all (only counter data files will be merged).
+type metaMerge struct {
+	calloc.BatchCounterAlloc
+	cmerge.Merger
+	// maps package import path to package state
+	pkm map[string]*pkstate
+	// list of packages
+	pkgs []*pkstate
+	// current package state
+	p *pkstate
+	// current pod state
+	pod *podstate
+	// counter data file osargs/goos/goarch state
+	astate *argstate
+}
+
+// pkstate
+type pkstate struct {
+	// index of package within meta-data file.
+	pkgIdx uint32
+	// this maps function index within the package to counter data payload
+	ctab map[uint32]decodecounter.FuncPayload
+	// pointer to meta-data blob for package
+	mdblob []byte
+	// filled in only for -pcombine merges
+	*pcombinestate
+}
+
+type podstate struct {
+	pmm      map[pkfunc]decodecounter.FuncPayload
+	mdf      string
+	mfr      *decodemeta.CoverageMetaFileReader
+	fileHash [16]byte
+}
+
+type pkfunc struct {
+	pk, fcn uint32
+}
+
+// pcombinestate
+type pcombinestate struct {
+	// Meta-data builder for the package.
+	cmdb *encodemeta.CoverageMetaDataBuilder
+	// Maps function meta-data hash to new function index in the
+	// new version of the package we're building.
+	ftab map[[16]byte]uint32
+}
+
+func newMetaMerge() *metaMerge {
+	return &metaMerge{
+		pkm:    make(map[string]*pkstate),
+		astate: &argstate{},
+	}
+}
+
+func (mm *metaMerge) visitMetaDataFile(mdf string, mfr *decodemeta.CoverageMetaFileReader) {
+	dbgtrace(2, "visitMetaDataFile(mdf=%s)", mdf)
+
+	// Record meta-data file name.
+	mm.pod.mdf = mdf
+	// Keep a pointer to the file-level reader.
+	mm.pod.mfr = mfr
+	// Record file hash.
+	mm.pod.fileHash = mfr.FileHash()
+	// Counter mode and granularity -- detect and record clashes here.
+	newgran := mfr.CounterGranularity()
+	newmode := mfr.CounterMode()
+	if err := mm.SetModeAndGranularity(mdf, newmode, newgran); err != nil {
+		fatal("%v", err)
+	}
+}
+
+func (mm *metaMerge) beginCounterDataFile(cdr *decodecounter.CounterDataReader) {
+	state := argvalues{
+		osargs: cdr.OsArgs(),
+		goos:   cdr.Goos(),
+		goarch: cdr.Goarch(),
+	}
+	mm.astate.Merge(state)
+}
+
+func copyMetaDataFile(inpath, outpath string) {
+	inf, err := os.Open(inpath)
+	if err != nil {
+		fatal("opening input meta-data file %s: %v", inpath, err)
+	}
+	defer inf.Close()
+
+	fi, err := inf.Stat()
+	if err != nil {
+		fatal("accessing input meta-data file %s: %v", inpath, err)
+	}
+
+	outf, err := os.OpenFile(outpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fi.Mode())
+	if err != nil {
+		fatal("opening output meta-data file %s: %v", outpath, err)
+	}
+
+	_, err = io.Copy(outf, inf)
+	outf.Close()
+	if err != nil {
+		fatal("writing output meta-data file %s: %v", outpath, err)
+	}
+}
+
+func (mm *metaMerge) beginPod() {
+	mm.pod = &podstate{
+		pmm: make(map[pkfunc]decodecounter.FuncPayload),
+	}
+}
+
+// metaEndPod handles actions needed when we're done visiting all of
+// the things in a pod -- counter files and meta-data file. There are
+// three cases of interest here:
+//
+// Case 1: in an unconditional merge (we're not selecting a specific set of
+// packages using "-pkg", and the "-pcombine" option is not in use),
+// we can simply copy over the meta-data file from input to output.
+//
+// Case 2: if this is a select merge (-pkg is in effect), then at
+// this point we write out a new smaller meta-data file that includes
+// only the packages of interest. At this point we also emit a merged
+// counter data file as well.
+//
+// Case 3: if "-pcombine" is in effect, we don't write anything at
+// this point (all writes will happen at the end of the run).
+func (mm *metaMerge) endPod(pcombine bool) {
+	if pcombine {
+		// Just clear out the pod data, we'll do all the
+		// heavy lifting at the end.
+		mm.pod = nil
+		return
+	}
+
+	finalHash := mm.pod.fileHash
+	if matchpkg != nil {
+		// Emit modified meta-data file for this pod.
+		finalHash = mm.emitMeta(*outdirflag, pcombine)
+	} else {
+		// Copy meta-data file for this pod to the output directory.
+		inpath := mm.pod.mdf
+		mdfbase := filepath.Base(mm.pod.mdf)
+		outpath := filepath.Join(*outdirflag, mdfbase)
+		copyMetaDataFile(inpath, outpath)
+	}
+
+	// Emit acccumulated counter data for this pod.
+	mm.emitCounters(*outdirflag, finalHash)
+
+	// Reset package state.
+	mm.pkm = make(map[string]*pkstate)
+	mm.pkgs = nil
+	mm.pod = nil
+
+	// Reset counter mode and granularity
+	mm.ResetModeAndGranularity()
+}
+
+// emitMeta encodes and writes out a new coverage meta-data file as
+// part of a merge operation, specifically a merge with the
+// "-pcombine" flag.
+func (mm *metaMerge) emitMeta(outdir string, pcombine bool) [16]byte {
+	fh := md5.New()
+	blobs := [][]byte{}
+	tlen := uint64(unsafe.Sizeof(coverage.MetaFileHeader{}))
+	for _, p := range mm.pkgs {
+		var blob []byte
+		if pcombine {
+			mdw := &slicewriter.WriteSeeker{}
+			p.cmdb.Emit(mdw)
+			blob = mdw.BytesWritten()
+		} else {
+			blob = p.mdblob
+		}
+		ph := md5.Sum(blob)
+		blobs = append(blobs, blob)
+		if _, err := fh.Write(ph[:]); err != nil {
+			panic(fmt.Sprintf("internal error: md5 sum failed: %v", err))
+		}
+		tlen += uint64(len(blob))
+	}
+	var finalHash [16]byte
+	fhh := fh.Sum(nil)
+	copy(finalHash[:], fhh)
+
+	// Open meta-file for writing.
+	fn := fmt.Sprintf("%s.%x", coverage.MetaFilePref, finalHash)
+	fpath := filepath.Join(outdir, fn)
+	mf, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		fatal("unable to open output meta-data file %s: %v", fpath, err)
+	}
+
+	// Encode and write.
+	mfw := encodemeta.NewCoverageMetaFileWriter(fpath, mf)
+	err = mfw.Write(finalHash, blobs, mm.Mode(), mm.Granularity())
+	if err != nil {
+		fatal("error writing %s: %v\n", fpath, err)
+	}
+	return finalHash
+}
+
+func (mm *metaMerge) emitCounters(outdir string, metaHash [16]byte) {
+	// Open output file. The file naming scheme is intended to mimic
+	// that used when running a coverage-instrumented binary, for
+	// consistency (however the process ID is not meaningful here, so
+	// use a value of zero).
+	var dummyPID int
+	fn := fmt.Sprintf(coverage.CounterFileTempl, coverage.CounterFilePref, metaHash, dummyPID, time.Now().UnixNano())
+	fpath := filepath.Join(outdir, fn)
+	cf, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		fatal("opening counter data file %s: %v", fpath, err)
+	}
+	defer func() {
+		if err := cf.Close(); err != nil {
+			fatal("error closing output meta-data file %s: %v", fpath, err)
+		}
+	}()
+
+	args := mm.astate.ArgsSummary()
+	cfw := encodecounter.NewCoverageDataWriter(cf, coverage.CtrULeb128)
+	if err := cfw.Write(metaHash, args, mm); err != nil {
+		fatal("counter file write failed: %v", err)
+	}
+	mm.astate = &argstate{}
+}
+
+// VisitFuncs is used while writing the counter data files; it
+// implements the 'VisitFuncs' method required by the interface
+// internal/coverage/encodecounter/CounterVisitor.
+func (mm *metaMerge) VisitFuncs(f encodecounter.CounterVisitorFn) error {
+	if *verbflag >= 4 {
+		fmt.Printf("counterVisitor invoked\n")
+	}
+	// For each package, for each function, construct counter
+	// array and then call "f" on it.
+	for pidx, p := range mm.pkgs {
+		fids := make([]int, 0, len(p.ctab))
+		for fid := range p.ctab {
+			fids = append(fids, int(fid))
+		}
+		sort.Ints(fids)
+		if *verbflag >= 4 {
+			fmt.Printf("fids for pk=%d: %+v\n", pidx, fids)
+		}
+		for _, fid := range fids {
+			fp := p.ctab[uint32(fid)]
+			if *verbflag >= 4 {
+				fmt.Printf("counter write for pk=%d fid=%d len(ctrs)=%d\n", pidx, fid, len(fp.Counters))
+			}
+			if err := f(uint32(pidx), uint32(fid), fp.Counters); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (mm *metaMerge) visitPackage(pd *decodemeta.CoverageMetaDataDecoder, pkgIdx uint32, pcombine bool) {
+	p, ok := mm.pkm[pd.PackagePath()]
+	if !ok {
+		p = &pkstate{
+			pkgIdx: uint32(len(mm.pkgs)),
+		}
+		mm.pkgs = append(mm.pkgs, p)
+		mm.pkm[pd.PackagePath()] = p
+		if pcombine {
+			p.pcombinestate = new(pcombinestate)
+			cmdb, err := encodemeta.NewCoverageMetaDataBuilder(pd.PackagePath(), pd.PackageName(), pd.ModulePath())
+			if err != nil {
+				fatal("fatal error creating meta-data builder: %v", err)
+			}
+			dbgtrace(2, "install new pkm entry for package %s pk=%d", pd.PackagePath(), pkgIdx)
+			p.cmdb = cmdb
+			p.ftab = make(map[[16]byte]uint32)
+		} else {
+			var err error
+			p.mdblob, err = mm.pod.mfr.GetPackagePayload(pkgIdx, nil)
+			if err != nil {
+				fatal("error extracting package %d payload from %s: %v",
+					pkgIdx, mm.pod.mdf, err)
+			}
+		}
+		p.ctab = make(map[uint32]decodecounter.FuncPayload)
+	}
+	mm.p = p
+}
+
+func (mm *metaMerge) visitFuncCounterData(data decodecounter.FuncPayload) {
+	key := pkfunc{pk: data.PkgIdx, fcn: data.FuncIdx}
+	val := mm.pod.pmm[key]
+	// FIXME: in theory either A) len(val.Counters) is zero, or B)
+	// the two lengths are equal. Assert if not? Of course, we could
+	// see odd stuff if there is source file skew.
+	if *verbflag > 4 {
+		fmt.Printf("visit pk=%d fid=%d len(counters)=%d\n", data.PkgIdx, data.FuncIdx, len(data.Counters))
+	}
+	if len(val.Counters) < len(data.Counters) {
+		t := val.Counters
+		val.Counters = mm.AllocateCounters(len(data.Counters))
+		copy(val.Counters, t)
+	}
+	err, overflow := mm.MergeCounters(val.Counters, data.Counters)
+	if err != nil {
+		fatal("%v", err)
+	}
+	if overflow {
+		warn("uint32 overflow during counter merge")
+	}
+	mm.pod.pmm[key] = val
+}
+
+func (mm *metaMerge) visitFunc(pkgIdx uint32, fnIdx uint32, fd *coverage.FuncDesc, verb string, pcombine bool) {
+	if *verbflag >= 3 {
+		fmt.Printf("visit pk=%d fid=%d func %s\n", pkgIdx, fnIdx, fd.Funcname)
+	}
+
+	var counters []uint32
+	key := pkfunc{pk: pkgIdx, fcn: fnIdx}
+	v, haveCounters := mm.pod.pmm[key]
+	if haveCounters {
+		counters = v.Counters
+	}
+
+	if pcombine {
+		// If the merge is running in "combine programs" mode, then hash
+		// the function and look it up in the package ftab to see if we've
+		// encountered it before. If we haven't, then register it with the
+		// meta-data builder.
+		fnhash := encodemeta.HashFuncDesc(fd)
+		gfidx, ok := mm.p.ftab[fnhash]
+		if !ok {
+			// We haven't seen this function before, need to add it to
+			// the meta data.
+			gfidx = uint32(mm.p.cmdb.AddFunc(*fd))
+			mm.p.ftab[fnhash] = gfidx
+			if *verbflag >= 3 {
+				fmt.Printf("new meta entry for fn %s fid=%d\n", fd.Funcname, gfidx)
+			}
+		}
+		fnIdx = gfidx
+	}
+	if !haveCounters {
+		return
+	}
+
+	// Install counters in package ctab.
+	gfp, ok := mm.p.ctab[fnIdx]
+	if ok {
+		if verb == "subtract" || verb == "intersect" {
+			panic("should never see this for intersect/subtract")
+		}
+		if *verbflag >= 3 {
+			fmt.Printf("counter merge for %s fidx=%d\n", fd.Funcname, fnIdx)
+		}
+		// Merge.
+		err, overflow := mm.MergeCounters(gfp.Counters, counters)
+		if err != nil {
+			fatal("%v", err)
+		}
+		if overflow {
+			warn("uint32 overflow during counter merge")
+		}
+		mm.p.ctab[fnIdx] = gfp
+	} else {
+		if *verbflag >= 3 {
+			fmt.Printf("null merge for %s fidx %d\n", fd.Funcname, fnIdx)
+		}
+		gfp := v
+		gfp.PkgIdx = mm.p.pkgIdx
+		gfp.FuncIdx = fnIdx
+		mm.p.ctab[fnIdx] = gfp
+	}
+}