1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
From 779a169144581438d9e24b8b46a86704f6335e35 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Sat, 16 Nov 2019 16:22:18 +0100
Subject: [PATCH] [LVI] Restructure caching
Variant on D70103. The caching is switched to always use a BB to
cache entry map, which then contains per-value caches. A separate
set contains value handles with a deletion callback. This allows us
to properly invalidate overdefined values.
A possible alternative would be to always cache by value first and
have per-BB maps/sets in the each cache entry. In that case we could
use a ValueMap and would avoid the separate value handle set. I went
with the BB indexing at the top level to make it easier to integrate
D69914, but possibly that's not the right choice.
Differential Revision: https://reviews.llvm.org/D70376
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index bad2de9e5f5..33406a75d80 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -136,12 +136,10 @@ namespace {
/// A callback value handle updates the cache when values are erased.
class LazyValueInfoCache;
struct LVIValueHandle final : public CallbackVH {
- // Needs to access getValPtr(), which is protected.
- friend struct DenseMapInfo<LVIValueHandle>;
LazyValueInfoCache *Parent;
- LVIValueHandle(Value *V, LazyValueInfoCache *P)
+ LVIValueHandle(Value *V, LazyValueInfoCache *P = nullptr)
: CallbackVH(V), Parent(P) { }
void deleted() override;
@@ -155,89 +153,63 @@ namespace {
/// This is the cache kept by LazyValueInfo which
/// maintains information about queries across the clients' queries.
class LazyValueInfoCache {
- /// This is all of the cached block information for exactly one Value*.
- /// The entries are sorted by the BasicBlock* of the
- /// entries, allowing us to do a lookup with a binary search.
- /// Over-defined lattice values are recorded in OverDefinedCache to reduce
- /// memory overhead.
- struct ValueCacheEntryTy {
- ValueCacheEntryTy(Value *V, LazyValueInfoCache *P) : Handle(V, P) {}
- LVIValueHandle Handle;
- SmallDenseMap<PoisoningVH<BasicBlock>, ValueLatticeElement, 4> BlockVals;
+ /// This is all of the cached information for one basic block. It contains
+ /// the per-value lattice elements, as well as a separate set for
+ /// overdefined values to reduce memory usage.
+ struct BlockCacheEntryTy {
+ SmallDenseMap<AssertingVH<Value>, ValueLatticeElement, 4> LatticeElements;
+ SmallDenseSet<AssertingVH<Value>, 4> OverDefined;
};
- /// This tracks, on a per-block basis, the set of values that are
- /// over-defined at the end of that block.
- typedef DenseMap<PoisoningVH<BasicBlock>, SmallPtrSet<Value *, 4>>
- OverDefinedCacheTy;
- /// Keep track of all blocks that we have ever seen, so we
- /// don't spend time removing unused blocks from our caches.
- DenseSet<PoisoningVH<BasicBlock> > SeenBlocks;
-
- /// This is all of the cached information for all values,
- /// mapped from Value* to key information.
- DenseMap<Value *, std::unique_ptr<ValueCacheEntryTy>> ValueCache;
- OverDefinedCacheTy OverDefinedCache;
-
+ /// Cached information per basic block.
+ DenseMap<PoisoningVH<BasicBlock>, BlockCacheEntryTy> BlockCache;
+ /// Set of value handles used to erase values from the cache on deletion.
+ DenseSet<LVIValueHandle, DenseMapInfo<Value *>> ValueHandles;
public:
void insertResult(Value *Val, BasicBlock *BB,
const ValueLatticeElement &Result) {
- SeenBlocks.insert(BB);
-
+ auto &CacheEntry = BlockCache.try_emplace(BB).first->second;
// Insert over-defined values into their own cache to reduce memory
// overhead.
if (Result.isOverdefined())
- OverDefinedCache[BB].insert(Val);
- else {
- auto It = ValueCache.find_as(Val);
- if (It == ValueCache.end()) {
- ValueCache[Val] = std::make_unique<ValueCacheEntryTy>(Val, this);
- It = ValueCache.find_as(Val);
- assert(It != ValueCache.end() && "Val was just added to the map!");
- }
- It->second->BlockVals[BB] = Result;
- }
- }
-
- bool isOverdefined(Value *V, BasicBlock *BB) const {
- auto ODI = OverDefinedCache.find(BB);
-
- if (ODI == OverDefinedCache.end())
- return false;
+ CacheEntry.OverDefined.insert(Val);
+ else
+ CacheEntry.LatticeElements.insert({ Val, Result });
- return ODI->second.count(V);
+ auto HandleIt = ValueHandles.find_as(Val);
+ if (HandleIt == ValueHandles.end())
+ ValueHandles.insert({ Val, this });
}
bool hasCachedValueInfo(Value *V, BasicBlock *BB) const {
- if (isOverdefined(V, BB))
- return true;
-
- auto I = ValueCache.find_as(V);
- if (I == ValueCache.end())
+ auto It = BlockCache.find(BB);
+ if (It == BlockCache.end())
return false;
- return I->second->BlockVals.count(BB);
+ return It->second.OverDefined.count(V) ||
+ It->second.LatticeElements.count(V);
}
ValueLatticeElement getCachedValueInfo(Value *V, BasicBlock *BB) const {
- if (isOverdefined(V, BB))
+ auto It = BlockCache.find(BB);
+ if (It == BlockCache.end())
+ return ValueLatticeElement();
+
+ if (It->second.OverDefined.count(V))
return ValueLatticeElement::getOverdefined();
- auto I = ValueCache.find_as(V);
- if (I == ValueCache.end())
- return ValueLatticeElement();
- auto BBI = I->second->BlockVals.find(BB);
- if (BBI == I->second->BlockVals.end())
+ auto LatticeIt = It->second.LatticeElements.find(V);
+ if (LatticeIt == It->second.LatticeElements.end())
return ValueLatticeElement();
- return BBI->second;
+
+ return LatticeIt->second;
}
/// clear - Empty the cache.
void clear() {
- SeenBlocks.clear();
- ValueCache.clear();
- OverDefinedCache.clear();
+ BlockCache.clear();
+ ValueHandles.clear();
}
/// Inform the cache that a given value has been deleted.
@@ -251,23 +223,18 @@ namespace {
/// OldSucc might have (unless also overdefined in NewSucc). This just
/// flushes elements from the cache and does not add any.
void threadEdgeImpl(BasicBlock *OldSucc,BasicBlock *NewSucc);
-
- friend struct LVIValueHandle;
};
}
void LazyValueInfoCache::eraseValue(Value *V) {
- for (auto I = OverDefinedCache.begin(), E = OverDefinedCache.end(); I != E;) {
- // Copy and increment the iterator immediately so we can erase behind
- // ourselves.
- auto Iter = I++;
- SmallPtrSetImpl<Value *> &ValueSet = Iter->second;
- ValueSet.erase(V);
- if (ValueSet.empty())
- OverDefinedCache.erase(Iter);
+ for (auto &Pair : BlockCache) {
+ Pair.second.LatticeElements.erase(V);
+ Pair.second.OverDefined.erase(V);
}
- ValueCache.erase(V);
+ auto HandleIt = ValueHandles.find_as(V);
+ if (HandleIt != ValueHandles.end())
+ ValueHandles.erase(HandleIt);
}
void LVIValueHandle::deleted() {
@@ -277,18 +244,7 @@ void LVIValueHandle::deleted() {
}
void LazyValueInfoCache::eraseBlock(BasicBlock *BB) {
- // Shortcut if we have never seen this block.
- DenseSet<PoisoningVH<BasicBlock> >::iterator I = SeenBlocks.find(BB);
- if (I == SeenBlocks.end())
- return;
- SeenBlocks.erase(I);
-
- auto ODI = OverDefinedCache.find(BB);
- if (ODI != OverDefinedCache.end())
- OverDefinedCache.erase(ODI);
-
- for (auto &I : ValueCache)
- I.second->BlockVals.erase(BB);
+ BlockCache.erase(BB);
}
void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,
@@ -306,10 +262,11 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,
std::vector<BasicBlock*> worklist;
worklist.push_back(OldSucc);
- auto I = OverDefinedCache.find(OldSucc);
- if (I == OverDefinedCache.end())
+ auto I = BlockCache.find(OldSucc);
+ if (I == BlockCache.end() || I->second.OverDefined.empty())
return; // Nothing to process here.
- SmallVector<Value *, 4> ValsToClear(I->second.begin(), I->second.end());
+ SmallVector<Value *, 4> ValsToClear(I->second.OverDefined.begin(),
+ I->second.OverDefined.end());
// Use a worklist to perform a depth-first search of OldSucc's successors.
// NOTE: We do not need a visited list since any blocks we have already
@@ -323,10 +280,10 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,
if (ToUpdate == NewSucc) continue;
// If a value was marked overdefined in OldSucc, and is here too...
- auto OI = OverDefinedCache.find(ToUpdate);
- if (OI == OverDefinedCache.end())
+ auto OI = BlockCache.find(ToUpdate);
+ if (OI == BlockCache.end() || OI->second.OverDefined.empty())
continue;
- SmallPtrSetImpl<Value *> &ValueSet = OI->second;
+ auto &ValueSet = OI->second.OverDefined;
bool changed = false;
for (Value *V : ValsToClear) {
@@ -336,11 +293,6 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc,
// If we removed anything, then we potentially need to update
// blocks successors too.
changed = true;
-
- if (ValueSet.empty()) {
- OverDefinedCache.erase(OI);
- break;
- }
}
if (!changed) continue;
|
