1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package interleaved implements the interleaved devirtualization and
// inlining pass.
package interleaved
import (
"cmd/compile/internal/base"
"cmd/compile/internal/devirtualize"
"cmd/compile/internal/inline"
"cmd/compile/internal/inline/inlheur"
"cmd/compile/internal/ir"
"cmd/compile/internal/pgo"
"cmd/compile/internal/typecheck"
"fmt"
)
// DevirtualizeAndInlinePackage interleaves devirtualization and inlining on
// all functions within pkg.
func DevirtualizeAndInlinePackage(pkg *ir.Package, profile *pgo.Profile) {
if profile != nil && base.Debug.PGODevirtualize > 0 {
// TODO(mdempsky): Integrate into DevirtualizeAndInlineFunc below.
ir.VisitFuncsBottomUp(typecheck.Target.Funcs, func(list []*ir.Func, recursive bool) {
for _, fn := range list {
devirtualize.ProfileGuided(fn, profile)
}
})
ir.CurFunc = nil
}
if base.Flag.LowerL != 0 {
inlheur.SetupScoreAdjustments()
}
var inlProfile *pgo.Profile // copy of profile for inlining
if base.Debug.PGOInline != 0 {
inlProfile = profile
}
if inlProfile != nil {
inline.PGOInlinePrologue(inlProfile, pkg.Funcs)
}
ir.VisitFuncsBottomUp(pkg.Funcs, func(funcs []*ir.Func, recursive bool) {
// We visit functions within an SCC in fairly arbitrary order,
// so by computing inlinability for all functions in the SCC
// before performing any inlining, the results are less
// sensitive to the order within the SCC (see #58905 for an
// example).
// First compute inlinability for all functions in the SCC ...
inline.CanInlineSCC(funcs, recursive, inlProfile)
// ... then make a second pass to do devirtualization and inlining
// of calls.
for _, fn := range funcs {
DevirtualizeAndInlineFunc(fn, inlProfile)
}
})
if base.Flag.LowerL != 0 {
// Perform a garbage collection of hidden closures functions that
// are no longer reachable from top-level functions following
// inlining. See #59404 and #59638 for more context.
inline.GarbageCollectUnreferencedHiddenClosures()
if base.Debug.DumpInlFuncProps != "" {
inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps)
}
if inlheur.Enabled() {
inline.PostProcessCallSites(inlProfile)
inlheur.TearDown()
}
}
}
// DevirtualizeAndInlineFunc interleaves devirtualization and inlining
// on a single function.
func DevirtualizeAndInlineFunc(fn *ir.Func, profile *pgo.Profile) {
ir.WithFunc(fn, func() {
if base.Flag.LowerL != 0 {
if inlheur.Enabled() && !fn.Wrapper() {
inlheur.ScoreCalls(fn)
defer inlheur.ScoreCallsCleanup()
}
if base.Debug.DumpInlFuncProps != "" && !fn.Wrapper() {
inlheur.DumpFuncProps(fn, base.Debug.DumpInlFuncProps)
}
}
bigCaller := base.Flag.LowerL != 0 && inline.IsBigFunc(fn)
if bigCaller && base.Flag.LowerM > 1 {
fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn)
}
// Walk fn's body and apply devirtualization and inlining.
var inlCalls []*ir.InlinedCallExpr
var edit func(ir.Node) ir.Node
edit = func(n ir.Node) ir.Node {
switch n := n.(type) {
case *ir.TailCallStmt:
n.Call.NoInline = true // can't inline yet
}
ir.EditChildren(n, edit)
if call, ok := n.(*ir.CallExpr); ok {
devirtualize.StaticCall(call)
if inlCall := inline.TryInlineCall(fn, call, bigCaller, profile); inlCall != nil {
inlCalls = append(inlCalls, inlCall)
n = inlCall
}
}
return n
}
ir.EditChildren(fn, edit)
// If we inlined any calls, we want to recursively visit their
// bodies for further devirtualization and inlining. However, we
// need to wait until *after* the original function body has been
// expanded, or else inlCallee can have false positives (e.g.,
// #54632).
for len(inlCalls) > 0 {
call := inlCalls[0]
inlCalls = inlCalls[1:]
ir.EditChildren(call, edit)
}
})
}
|
