File: | src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Transforms/Utils/SimplifyCFG.cpp |
Warning: | line 4858, column 19 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // Peephole optimize the CFG. | ||||
10 | // | ||||
11 | //===----------------------------------------------------------------------===// | ||||
12 | |||||
13 | #include "llvm/ADT/APInt.h" | ||||
14 | #include "llvm/ADT/ArrayRef.h" | ||||
15 | #include "llvm/ADT/DenseMap.h" | ||||
16 | #include "llvm/ADT/MapVector.h" | ||||
17 | #include "llvm/ADT/Optional.h" | ||||
18 | #include "llvm/ADT/STLExtras.h" | ||||
19 | #include "llvm/ADT/ScopeExit.h" | ||||
20 | #include "llvm/ADT/Sequence.h" | ||||
21 | #include "llvm/ADT/SetOperations.h" | ||||
22 | #include "llvm/ADT/SetVector.h" | ||||
23 | #include "llvm/ADT/SmallPtrSet.h" | ||||
24 | #include "llvm/ADT/SmallVector.h" | ||||
25 | #include "llvm/ADT/Statistic.h" | ||||
26 | #include "llvm/ADT/StringRef.h" | ||||
27 | #include "llvm/Analysis/AssumptionCache.h" | ||||
28 | #include "llvm/Analysis/ConstantFolding.h" | ||||
29 | #include "llvm/Analysis/EHPersonalities.h" | ||||
30 | #include "llvm/Analysis/GuardUtils.h" | ||||
31 | #include "llvm/Analysis/InstructionSimplify.h" | ||||
32 | #include "llvm/Analysis/MemorySSA.h" | ||||
33 | #include "llvm/Analysis/MemorySSAUpdater.h" | ||||
34 | #include "llvm/Analysis/TargetTransformInfo.h" | ||||
35 | #include "llvm/Analysis/ValueTracking.h" | ||||
36 | #include "llvm/IR/Attributes.h" | ||||
37 | #include "llvm/IR/BasicBlock.h" | ||||
38 | #include "llvm/IR/CFG.h" | ||||
39 | #include "llvm/IR/Constant.h" | ||||
40 | #include "llvm/IR/ConstantRange.h" | ||||
41 | #include "llvm/IR/Constants.h" | ||||
42 | #include "llvm/IR/DataLayout.h" | ||||
43 | #include "llvm/IR/DerivedTypes.h" | ||||
44 | #include "llvm/IR/Function.h" | ||||
45 | #include "llvm/IR/GlobalValue.h" | ||||
46 | #include "llvm/IR/GlobalVariable.h" | ||||
47 | #include "llvm/IR/IRBuilder.h" | ||||
48 | #include "llvm/IR/InstrTypes.h" | ||||
49 | #include "llvm/IR/Instruction.h" | ||||
50 | #include "llvm/IR/Instructions.h" | ||||
51 | #include "llvm/IR/IntrinsicInst.h" | ||||
52 | #include "llvm/IR/Intrinsics.h" | ||||
53 | #include "llvm/IR/LLVMContext.h" | ||||
54 | #include "llvm/IR/MDBuilder.h" | ||||
55 | #include "llvm/IR/Metadata.h" | ||||
56 | #include "llvm/IR/Module.h" | ||||
57 | #include "llvm/IR/NoFolder.h" | ||||
58 | #include "llvm/IR/Operator.h" | ||||
59 | #include "llvm/IR/PatternMatch.h" | ||||
60 | #include "llvm/IR/PseudoProbe.h" | ||||
61 | #include "llvm/IR/Type.h" | ||||
62 | #include "llvm/IR/Use.h" | ||||
63 | #include "llvm/IR/User.h" | ||||
64 | #include "llvm/IR/Value.h" | ||||
65 | #include "llvm/IR/ValueHandle.h" | ||||
66 | #include "llvm/Support/BranchProbability.h" | ||||
67 | #include "llvm/Support/Casting.h" | ||||
68 | #include "llvm/Support/CommandLine.h" | ||||
69 | #include "llvm/Support/Debug.h" | ||||
70 | #include "llvm/Support/ErrorHandling.h" | ||||
71 | #include "llvm/Support/KnownBits.h" | ||||
72 | #include "llvm/Support/MathExtras.h" | ||||
73 | #include "llvm/Support/raw_ostream.h" | ||||
74 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | ||||
75 | #include "llvm/Transforms/Utils/Local.h" | ||||
76 | #include "llvm/Transforms/Utils/SSAUpdater.h" | ||||
77 | #include "llvm/Transforms/Utils/ValueMapper.h" | ||||
78 | #include <algorithm> | ||||
79 | #include <cassert> | ||||
80 | #include <climits> | ||||
81 | #include <cstddef> | ||||
82 | #include <cstdint> | ||||
83 | #include <iterator> | ||||
84 | #include <map> | ||||
85 | #include <set> | ||||
86 | #include <tuple> | ||||
87 | #include <utility> | ||||
88 | #include <vector> | ||||
89 | |||||
90 | using namespace llvm; | ||||
91 | using namespace PatternMatch; | ||||
92 | |||||
93 | #define DEBUG_TYPE"simplifycfg" "simplifycfg" | ||||
94 | |||||
95 | cl::opt<bool> llvm::RequireAndPreserveDomTree( | ||||
96 | "simplifycfg-require-and-preserve-domtree", cl::Hidden, cl::ZeroOrMore, | ||||
97 | cl::init(false), | ||||
98 | cl::desc("Temorary development switch used to gradually uplift SimplifyCFG " | ||||
99 | "into preserving DomTree,")); | ||||
100 | |||||
101 | // Chosen as 2 so as to be cheap, but still to have enough power to fold | ||||
102 | // a select, so the "clamp" idiom (of a min followed by a max) will be caught. | ||||
103 | // To catch this, we need to fold a compare and a select, hence '2' being the | ||||
104 | // minimum reasonable default. | ||||
105 | static cl::opt<unsigned> PHINodeFoldingThreshold( | ||||
106 | "phi-node-folding-threshold", cl::Hidden, cl::init(2), | ||||
107 | cl::desc( | ||||
108 | "Control the amount of phi node folding to perform (default = 2)")); | ||||
109 | |||||
110 | static cl::opt<unsigned> TwoEntryPHINodeFoldingThreshold( | ||||
111 | "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4), | ||||
112 | cl::desc("Control the maximal total instruction cost that we are willing " | ||||
113 | "to speculatively execute to fold a 2-entry PHI node into a " | ||||
114 | "select (default = 4)")); | ||||
115 | |||||
116 | static cl::opt<bool> | ||||
117 | HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true), | ||||
118 | cl::desc("Hoist common instructions up to the parent block")); | ||||
119 | |||||
120 | static cl::opt<bool> | ||||
121 | SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true), | ||||
122 | cl::desc("Sink common instructions down to the end block")); | ||||
123 | |||||
124 | static cl::opt<bool> HoistCondStores( | ||||
125 | "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true), | ||||
126 | cl::desc("Hoist conditional stores if an unconditional store precedes")); | ||||
127 | |||||
128 | static cl::opt<bool> MergeCondStores( | ||||
129 | "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true), | ||||
130 | cl::desc("Hoist conditional stores even if an unconditional store does not " | ||||
131 | "precede - hoist multiple conditional stores into a single " | ||||
132 | "predicated store")); | ||||
133 | |||||
134 | static cl::opt<bool> MergeCondStoresAggressively( | ||||
135 | "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false), | ||||
136 | cl::desc("When merging conditional stores, do so even if the resultant " | ||||
137 | "basic blocks are unlikely to be if-converted as a result")); | ||||
138 | |||||
139 | static cl::opt<bool> SpeculateOneExpensiveInst( | ||||
140 | "speculate-one-expensive-inst", cl::Hidden, cl::init(true), | ||||
141 | cl::desc("Allow exactly one expensive instruction to be speculatively " | ||||
142 | "executed")); | ||||
143 | |||||
144 | static cl::opt<unsigned> MaxSpeculationDepth( | ||||
145 | "max-speculation-depth", cl::Hidden, cl::init(10), | ||||
146 | cl::desc("Limit maximum recursion depth when calculating costs of " | ||||
147 | "speculatively executed instructions")); | ||||
148 | |||||
149 | static cl::opt<int> | ||||
150 | MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden, | ||||
151 | cl::init(10), | ||||
152 | cl::desc("Max size of a block which is still considered " | ||||
153 | "small enough to thread through")); | ||||
154 | |||||
155 | // Two is chosen to allow one negation and a logical combine. | ||||
156 | static cl::opt<unsigned> | ||||
157 | BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden, | ||||
158 | cl::init(2), | ||||
159 | cl::desc("Maximum cost of combining conditions when " | ||||
160 | "folding branches")); | ||||
161 | |||||
162 | STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps")static llvm::Statistic NumBitMaps = {"simplifycfg", "NumBitMaps" , "Number of switch instructions turned into bitmaps"}; | ||||
163 | STATISTIC(NumLinearMaps,static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps" , "Number of switch instructions turned into linear mapping"} | ||||
164 | "Number of switch instructions turned into linear mapping")static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps" , "Number of switch instructions turned into linear mapping"}; | ||||
165 | STATISTIC(NumLookupTables,static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables" , "Number of switch instructions turned into lookup tables"} | ||||
166 | "Number of switch instructions turned into lookup tables")static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables" , "Number of switch instructions turned into lookup tables"}; | ||||
167 | STATISTIC(static llvm::Statistic NumLookupTablesHoles = {"simplifycfg", "NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)" } | ||||
168 | NumLookupTablesHoles,static llvm::Statistic NumLookupTablesHoles = {"simplifycfg", "NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)" } | ||||
169 | "Number of switch instructions turned into lookup tables (holes checked)")static llvm::Statistic NumLookupTablesHoles = {"simplifycfg", "NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)" }; | ||||
170 | STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares")static llvm::Statistic NumTableCmpReuses = {"simplifycfg", "NumTableCmpReuses" , "Number of reused switch table lookup compares"}; | ||||
171 | STATISTIC(NumFoldValueComparisonIntoPredecessors,static llvm::Statistic NumFoldValueComparisonIntoPredecessors = {"simplifycfg", "NumFoldValueComparisonIntoPredecessors", "Number of value comparisons folded into predecessor basic blocks" } | ||||
172 | "Number of value comparisons folded into predecessor basic blocks")static llvm::Statistic NumFoldValueComparisonIntoPredecessors = {"simplifycfg", "NumFoldValueComparisonIntoPredecessors", "Number of value comparisons folded into predecessor basic blocks" }; | ||||
173 | STATISTIC(NumFoldBranchToCommonDest,static llvm::Statistic NumFoldBranchToCommonDest = {"simplifycfg" , "NumFoldBranchToCommonDest", "Number of branches folded into predecessor basic block" } | ||||
174 | "Number of branches folded into predecessor basic block")static llvm::Statistic NumFoldBranchToCommonDest = {"simplifycfg" , "NumFoldBranchToCommonDest", "Number of branches folded into predecessor basic block" }; | ||||
175 | STATISTIC(static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode" , "Number of common instruction 'blocks' hoisted up to the begin block" } | ||||
176 | NumHoistCommonCode,static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode" , "Number of common instruction 'blocks' hoisted up to the begin block" } | ||||
177 | "Number of common instruction 'blocks' hoisted up to the begin block")static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode" , "Number of common instruction 'blocks' hoisted up to the begin block" }; | ||||
178 | STATISTIC(NumHoistCommonInstrs,static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg", "NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block" } | ||||
179 | "Number of common instructions hoisted up to the begin block")static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg", "NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block" }; | ||||
180 | STATISTIC(NumSinkCommonCode,static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode" , "Number of common instruction 'blocks' sunk down to the end block" } | ||||
181 | "Number of common instruction 'blocks' sunk down to the end block")static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode" , "Number of common instruction 'blocks' sunk down to the end block" }; | ||||
182 | STATISTIC(NumSinkCommonInstrs,static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs" , "Number of common instructions sunk down to the end block"} | ||||
183 | "Number of common instructions sunk down to the end block")static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs" , "Number of common instructions sunk down to the end block"}; | ||||
184 | STATISTIC(NumSpeculations, "Number of speculative executed instructions")static llvm::Statistic NumSpeculations = {"simplifycfg", "NumSpeculations" , "Number of speculative executed instructions"}; | ||||
185 | STATISTIC(NumInvokes,static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes" , "Number of invokes with empty resume blocks simplified into calls" } | ||||
186 | "Number of invokes with empty resume blocks simplified into calls")static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes" , "Number of invokes with empty resume blocks simplified into calls" }; | ||||
187 | |||||
188 | namespace { | ||||
189 | |||||
190 | // The first field contains the value that the switch produces when a certain | ||||
191 | // case group is selected, and the second field is a vector containing the | ||||
192 | // cases composing the case group. | ||||
193 | using SwitchCaseResultVectorTy = | ||||
194 | SmallVector<std::pair<Constant *, SmallVector<ConstantInt *, 4>>, 2>; | ||||
195 | |||||
196 | // The first field contains the phi node that generates a result of the switch | ||||
197 | // and the second field contains the value generated for a certain case in the | ||||
198 | // switch for that PHI. | ||||
199 | using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>; | ||||
200 | |||||
201 | /// ValueEqualityComparisonCase - Represents a case of a switch. | ||||
202 | struct ValueEqualityComparisonCase { | ||||
203 | ConstantInt *Value; | ||||
204 | BasicBlock *Dest; | ||||
205 | |||||
206 | ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest) | ||||
207 | : Value(Value), Dest(Dest) {} | ||||
208 | |||||
209 | bool operator<(ValueEqualityComparisonCase RHS) const { | ||||
210 | // Comparing pointers is ok as we only rely on the order for uniquing. | ||||
211 | return Value < RHS.Value; | ||||
212 | } | ||||
213 | |||||
214 | bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; } | ||||
215 | }; | ||||
216 | |||||
217 | class SimplifyCFGOpt { | ||||
218 | const TargetTransformInfo &TTI; | ||||
219 | DomTreeUpdater *DTU; | ||||
220 | const DataLayout &DL; | ||||
221 | ArrayRef<WeakVH> LoopHeaders; | ||||
222 | const SimplifyCFGOptions &Options; | ||||
223 | bool Resimplify; | ||||
224 | |||||
225 | Value *isValueEqualityComparison(Instruction *TI); | ||||
226 | BasicBlock *GetValueEqualityComparisonCases( | ||||
227 | Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases); | ||||
228 | bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI, | ||||
229 | BasicBlock *Pred, | ||||
230 | IRBuilder<> &Builder); | ||||
231 | bool PerformValueComparisonIntoPredecessorFolding(Instruction *TI, Value *&CV, | ||||
232 | Instruction *PTI, | ||||
233 | IRBuilder<> &Builder); | ||||
234 | bool FoldValueComparisonIntoPredecessors(Instruction *TI, | ||||
235 | IRBuilder<> &Builder); | ||||
236 | |||||
237 | bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder); | ||||
238 | bool simplifySingleResume(ResumeInst *RI); | ||||
239 | bool simplifyCommonResume(ResumeInst *RI); | ||||
240 | bool simplifyCleanupReturn(CleanupReturnInst *RI); | ||||
241 | bool simplifyUnreachable(UnreachableInst *UI); | ||||
242 | bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder); | ||||
243 | bool simplifyIndirectBr(IndirectBrInst *IBI); | ||||
244 | bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder); | ||||
245 | bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder); | ||||
246 | bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder); | ||||
247 | |||||
248 | bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI, | ||||
249 | IRBuilder<> &Builder); | ||||
250 | |||||
251 | bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI, | ||||
252 | bool EqTermsOnly); | ||||
253 | bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB, | ||||
254 | const TargetTransformInfo &TTI); | ||||
255 | bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond, | ||||
256 | BasicBlock *TrueBB, BasicBlock *FalseBB, | ||||
257 | uint32_t TrueWeight, uint32_t FalseWeight); | ||||
258 | bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder, | ||||
259 | const DataLayout &DL); | ||||
260 | bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select); | ||||
261 | bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI); | ||||
262 | bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder); | ||||
263 | |||||
264 | public: | ||||
265 | SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU, | ||||
266 | const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders, | ||||
267 | const SimplifyCFGOptions &Opts) | ||||
268 | : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) { | ||||
269 | assert((!DTU || !DTU->hasPostDomTree()) &&((void)0) | ||||
270 | "SimplifyCFG is not yet capable of maintaining validity of a "((void)0) | ||||
271 | "PostDomTree, so don't ask for it.")((void)0); | ||||
272 | } | ||||
273 | |||||
274 | bool simplifyOnce(BasicBlock *BB); | ||||
275 | bool simplifyOnceImpl(BasicBlock *BB); | ||||
276 | bool run(BasicBlock *BB); | ||||
277 | |||||
278 | // Helper to set Resimplify and return change indication. | ||||
279 | bool requestResimplify() { | ||||
280 | Resimplify = true; | ||||
281 | return true; | ||||
282 | } | ||||
283 | }; | ||||
284 | |||||
285 | } // end anonymous namespace | ||||
286 | |||||
287 | /// Return true if it is safe to merge these two | ||||
288 | /// terminator instructions together. | ||||
289 | static bool | ||||
290 | SafeToMergeTerminators(Instruction *SI1, Instruction *SI2, | ||||
291 | SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) { | ||||
292 | if (SI1 == SI2) | ||||
293 | return false; // Can't merge with self! | ||||
294 | |||||
295 | // It is not safe to merge these two switch instructions if they have a common | ||||
296 | // successor, and if that successor has a PHI node, and if *that* PHI node has | ||||
297 | // conflicting incoming values from the two switch blocks. | ||||
298 | BasicBlock *SI1BB = SI1->getParent(); | ||||
299 | BasicBlock *SI2BB = SI2->getParent(); | ||||
300 | |||||
301 | SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB)); | ||||
302 | bool Fail = false; | ||||
303 | for (BasicBlock *Succ : successors(SI2BB)) | ||||
304 | if (SI1Succs.count(Succ)) | ||||
305 | for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) { | ||||
306 | PHINode *PN = cast<PHINode>(BBI); | ||||
307 | if (PN->getIncomingValueForBlock(SI1BB) != | ||||
308 | PN->getIncomingValueForBlock(SI2BB)) { | ||||
309 | if (FailBlocks) | ||||
310 | FailBlocks->insert(Succ); | ||||
311 | Fail = true; | ||||
312 | } | ||||
313 | } | ||||
314 | |||||
315 | return !Fail; | ||||
316 | } | ||||
317 | |||||
318 | /// Update PHI nodes in Succ to indicate that there will now be entries in it | ||||
319 | /// from the 'NewPred' block. The values that will be flowing into the PHI nodes | ||||
320 | /// will be the same as those coming in from ExistPred, an existing predecessor | ||||
321 | /// of Succ. | ||||
322 | static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred, | ||||
323 | BasicBlock *ExistPred, | ||||
324 | MemorySSAUpdater *MSSAU = nullptr) { | ||||
325 | for (PHINode &PN : Succ->phis()) | ||||
326 | PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred); | ||||
327 | if (MSSAU) | ||||
328 | if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ)) | ||||
329 | MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred); | ||||
330 | } | ||||
331 | |||||
332 | /// Compute an abstract "cost" of speculating the given instruction, | ||||
333 | /// which is assumed to be safe to speculate. TCC_Free means cheap, | ||||
334 | /// TCC_Basic means less cheap, and TCC_Expensive means prohibitively | ||||
335 | /// expensive. | ||||
336 | static InstructionCost computeSpeculationCost(const User *I, | ||||
337 | const TargetTransformInfo &TTI) { | ||||
338 | assert(isSafeToSpeculativelyExecute(I) &&((void)0) | ||||
339 | "Instruction is not safe to speculatively execute!")((void)0); | ||||
340 | return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency); | ||||
341 | } | ||||
342 | |||||
343 | /// If we have a merge point of an "if condition" as accepted above, | ||||
344 | /// return true if the specified value dominates the block. We | ||||
345 | /// don't handle the true generality of domination here, just a special case | ||||
346 | /// which works well enough for us. | ||||
347 | /// | ||||
348 | /// If AggressiveInsts is non-null, and if V does not dominate BB, we check to | ||||
349 | /// see if V (which must be an instruction) and its recursive operands | ||||
350 | /// that do not dominate BB have a combined cost lower than Budget and | ||||
351 | /// are non-trapping. If both are true, the instruction is inserted into the | ||||
352 | /// set and true is returned. | ||||
353 | /// | ||||
354 | /// The cost for most non-trapping instructions is defined as 1 except for | ||||
355 | /// Select whose cost is 2. | ||||
356 | /// | ||||
357 | /// After this function returns, Cost is increased by the cost of | ||||
358 | /// V plus its non-dominating operands. If that cost is greater than | ||||
359 | /// Budget, false is returned and Cost is undefined. | ||||
360 | static bool dominatesMergePoint(Value *V, BasicBlock *BB, | ||||
361 | SmallPtrSetImpl<Instruction *> &AggressiveInsts, | ||||
362 | InstructionCost &Cost, | ||||
363 | InstructionCost Budget, | ||||
364 | const TargetTransformInfo &TTI, | ||||
365 | unsigned Depth = 0) { | ||||
366 | // It is possible to hit a zero-cost cycle (phi/gep instructions for example), | ||||
367 | // so limit the recursion depth. | ||||
368 | // TODO: While this recursion limit does prevent pathological behavior, it | ||||
369 | // would be better to track visited instructions to avoid cycles. | ||||
370 | if (Depth == MaxSpeculationDepth) | ||||
371 | return false; | ||||
372 | |||||
373 | Instruction *I = dyn_cast<Instruction>(V); | ||||
374 | if (!I) { | ||||
375 | // Non-instructions all dominate instructions, but not all constantexprs | ||||
376 | // can be executed unconditionally. | ||||
377 | if (ConstantExpr *C = dyn_cast<ConstantExpr>(V)) | ||||
378 | if (C->canTrap()) | ||||
379 | return false; | ||||
380 | return true; | ||||
381 | } | ||||
382 | BasicBlock *PBB = I->getParent(); | ||||
383 | |||||
384 | // We don't want to allow weird loops that might have the "if condition" in | ||||
385 | // the bottom of this block. | ||||
386 | if (PBB == BB) | ||||
387 | return false; | ||||
388 | |||||
389 | // If this instruction is defined in a block that contains an unconditional | ||||
390 | // branch to BB, then it must be in the 'conditional' part of the "if | ||||
391 | // statement". If not, it definitely dominates the region. | ||||
392 | BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator()); | ||||
393 | if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB) | ||||
394 | return true; | ||||
395 | |||||
396 | // If we have seen this instruction before, don't count it again. | ||||
397 | if (AggressiveInsts.count(I)) | ||||
398 | return true; | ||||
399 | |||||
400 | // Okay, it looks like the instruction IS in the "condition". Check to | ||||
401 | // see if it's a cheap instruction to unconditionally compute, and if it | ||||
402 | // only uses stuff defined outside of the condition. If so, hoist it out. | ||||
403 | if (!isSafeToSpeculativelyExecute(I)) | ||||
404 | return false; | ||||
405 | |||||
406 | Cost += computeSpeculationCost(I, TTI); | ||||
407 | |||||
408 | // Allow exactly one instruction to be speculated regardless of its cost | ||||
409 | // (as long as it is safe to do so). | ||||
410 | // This is intended to flatten the CFG even if the instruction is a division | ||||
411 | // or other expensive operation. The speculation of an expensive instruction | ||||
412 | // is expected to be undone in CodeGenPrepare if the speculation has not | ||||
413 | // enabled further IR optimizations. | ||||
414 | if (Cost > Budget && | ||||
415 | (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 || | ||||
416 | !Cost.isValid())) | ||||
417 | return false; | ||||
418 | |||||
419 | // Okay, we can only really hoist these out if their operands do | ||||
420 | // not take us over the cost threshold. | ||||
421 | for (Use &Op : I->operands()) | ||||
422 | if (!dominatesMergePoint(Op, BB, AggressiveInsts, Cost, Budget, TTI, | ||||
423 | Depth + 1)) | ||||
424 | return false; | ||||
425 | // Okay, it's safe to do this! Remember this instruction. | ||||
426 | AggressiveInsts.insert(I); | ||||
427 | return true; | ||||
428 | } | ||||
429 | |||||
430 | /// Extract ConstantInt from value, looking through IntToPtr | ||||
431 | /// and PointerNullValue. Return NULL if value is not a constant int. | ||||
432 | static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) { | ||||
433 | // Normal constant int. | ||||
434 | ConstantInt *CI = dyn_cast<ConstantInt>(V); | ||||
435 | if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy()) | ||||
436 | return CI; | ||||
437 | |||||
438 | // This is some kind of pointer constant. Turn it into a pointer-sized | ||||
439 | // ConstantInt if possible. | ||||
440 | IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType())); | ||||
441 | |||||
442 | // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*). | ||||
443 | if (isa<ConstantPointerNull>(V)) | ||||
444 | return ConstantInt::get(PtrTy, 0); | ||||
445 | |||||
446 | // IntToPtr const int. | ||||
447 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) | ||||
448 | if (CE->getOpcode() == Instruction::IntToPtr) | ||||
449 | if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) { | ||||
450 | // The constant is very likely to have the right type already. | ||||
451 | if (CI->getType() == PtrTy) | ||||
452 | return CI; | ||||
453 | else | ||||
454 | return cast<ConstantInt>( | ||||
455 | ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false)); | ||||
456 | } | ||||
457 | return nullptr; | ||||
458 | } | ||||
459 | |||||
460 | namespace { | ||||
461 | |||||
462 | /// Given a chain of or (||) or and (&&) comparison of a value against a | ||||
463 | /// constant, this will try to recover the information required for a switch | ||||
464 | /// structure. | ||||
465 | /// It will depth-first traverse the chain of comparison, seeking for patterns | ||||
466 | /// like %a == 12 or %a < 4 and combine them to produce a set of integer | ||||
467 | /// representing the different cases for the switch. | ||||
468 | /// Note that if the chain is composed of '||' it will build the set of elements | ||||
469 | /// that matches the comparisons (i.e. any of this value validate the chain) | ||||
470 | /// while for a chain of '&&' it will build the set elements that make the test | ||||
471 | /// fail. | ||||
472 | struct ConstantComparesGatherer { | ||||
473 | const DataLayout &DL; | ||||
474 | |||||
475 | /// Value found for the switch comparison | ||||
476 | Value *CompValue = nullptr; | ||||
477 | |||||
478 | /// Extra clause to be checked before the switch | ||||
479 | Value *Extra = nullptr; | ||||
480 | |||||
481 | /// Set of integers to match in switch | ||||
482 | SmallVector<ConstantInt *, 8> Vals; | ||||
483 | |||||
484 | /// Number of comparisons matched in the and/or chain | ||||
485 | unsigned UsedICmps = 0; | ||||
486 | |||||
487 | /// Construct and compute the result for the comparison instruction Cond | ||||
488 | ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) { | ||||
489 | gather(Cond); | ||||
490 | } | ||||
491 | |||||
492 | ConstantComparesGatherer(const ConstantComparesGatherer &) = delete; | ||||
493 | ConstantComparesGatherer & | ||||
494 | operator=(const ConstantComparesGatherer &) = delete; | ||||
495 | |||||
496 | private: | ||||
497 | /// Try to set the current value used for the comparison, it succeeds only if | ||||
498 | /// it wasn't set before or if the new value is the same as the old one | ||||
499 | bool setValueOnce(Value *NewVal) { | ||||
500 | if (CompValue && CompValue != NewVal) | ||||
501 | return false; | ||||
502 | CompValue = NewVal; | ||||
503 | return (CompValue != nullptr); | ||||
504 | } | ||||
505 | |||||
506 | /// Try to match Instruction "I" as a comparison against a constant and | ||||
507 | /// populates the array Vals with the set of values that match (or do not | ||||
508 | /// match depending on isEQ). | ||||
509 | /// Return false on failure. On success, the Value the comparison matched | ||||
510 | /// against is placed in CompValue. | ||||
511 | /// If CompValue is already set, the function is expected to fail if a match | ||||
512 | /// is found but the value compared to is different. | ||||
513 | bool matchInstruction(Instruction *I, bool isEQ) { | ||||
514 | // If this is an icmp against a constant, handle this as one of the cases. | ||||
515 | ICmpInst *ICI; | ||||
516 | ConstantInt *C; | ||||
517 | if (!((ICI = dyn_cast<ICmpInst>(I)) && | ||||
518 | (C = GetConstantInt(I->getOperand(1), DL)))) { | ||||
519 | return false; | ||||
520 | } | ||||
521 | |||||
522 | Value *RHSVal; | ||||
523 | const APInt *RHSC; | ||||
524 | |||||
525 | // Pattern match a special case | ||||
526 | // (x & ~2^z) == y --> x == y || x == y|2^z | ||||
527 | // This undoes a transformation done by instcombine to fuse 2 compares. | ||||
528 | if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) { | ||||
529 | // It's a little bit hard to see why the following transformations are | ||||
530 | // correct. Here is a CVC3 program to verify them for 64-bit values: | ||||
531 | |||||
532 | /* | ||||
533 | ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63); | ||||
534 | x : BITVECTOR(64); | ||||
535 | y : BITVECTOR(64); | ||||
536 | z : BITVECTOR(64); | ||||
537 | mask : BITVECTOR(64) = BVSHL(ONE, z); | ||||
538 | QUERY( (y & ~mask = y) => | ||||
539 | ((x & ~mask = y) <=> (x = y OR x = (y | mask))) | ||||
540 | ); | ||||
541 | QUERY( (y | mask = y) => | ||||
542 | ((x | mask = y) <=> (x = y OR x = (y & ~mask))) | ||||
543 | ); | ||||
544 | */ | ||||
545 | |||||
546 | // Please note that each pattern must be a dual implication (<--> or | ||||
547 | // iff). One directional implication can create spurious matches. If the | ||||
548 | // implication is only one-way, an unsatisfiable condition on the left | ||||
549 | // side can imply a satisfiable condition on the right side. Dual | ||||
550 | // implication ensures that satisfiable conditions are transformed to | ||||
551 | // other satisfiable conditions and unsatisfiable conditions are | ||||
552 | // transformed to other unsatisfiable conditions. | ||||
553 | |||||
554 | // Here is a concrete example of a unsatisfiable condition on the left | ||||
555 | // implying a satisfiable condition on the right: | ||||
556 | // | ||||
557 | // mask = (1 << z) | ||||
558 | // (x & ~mask) == y --> (x == y || x == (y | mask)) | ||||
559 | // | ||||
560 | // Substituting y = 3, z = 0 yields: | ||||
561 | // (x & -2) == 3 --> (x == 3 || x == 2) | ||||
562 | |||||
563 | // Pattern match a special case: | ||||
564 | /* | ||||
565 | QUERY( (y & ~mask = y) => | ||||
566 | ((x & ~mask = y) <=> (x = y OR x = (y | mask))) | ||||
567 | ); | ||||
568 | */ | ||||
569 | if (match(ICI->getOperand(0), | ||||
570 | m_And(m_Value(RHSVal), m_APInt(RHSC)))) { | ||||
571 | APInt Mask = ~*RHSC; | ||||
572 | if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) { | ||||
573 | // If we already have a value for the switch, it has to match! | ||||
574 | if (!setValueOnce(RHSVal)) | ||||
575 | return false; | ||||
576 | |||||
577 | Vals.push_back(C); | ||||
578 | Vals.push_back( | ||||
579 | ConstantInt::get(C->getContext(), | ||||
580 | C->getValue() | Mask)); | ||||
581 | UsedICmps++; | ||||
582 | return true; | ||||
583 | } | ||||
584 | } | ||||
585 | |||||
586 | // Pattern match a special case: | ||||
587 | /* | ||||
588 | QUERY( (y | mask = y) => | ||||
589 | ((x | mask = y) <=> (x = y OR x = (y & ~mask))) | ||||
590 | ); | ||||
591 | */ | ||||
592 | if (match(ICI->getOperand(0), | ||||
593 | m_Or(m_Value(RHSVal), m_APInt(RHSC)))) { | ||||
594 | APInt Mask = *RHSC; | ||||
595 | if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) { | ||||
596 | // If we already have a value for the switch, it has to match! | ||||
597 | if (!setValueOnce(RHSVal)) | ||||
598 | return false; | ||||
599 | |||||
600 | Vals.push_back(C); | ||||
601 | Vals.push_back(ConstantInt::get(C->getContext(), | ||||
602 | C->getValue() & ~Mask)); | ||||
603 | UsedICmps++; | ||||
604 | return true; | ||||
605 | } | ||||
606 | } | ||||
607 | |||||
608 | // If we already have a value for the switch, it has to match! | ||||
609 | if (!setValueOnce(ICI->getOperand(0))) | ||||
610 | return false; | ||||
611 | |||||
612 | UsedICmps++; | ||||
613 | Vals.push_back(C); | ||||
614 | return ICI->getOperand(0); | ||||
615 | } | ||||
616 | |||||
617 | // If we have "x ult 3", for example, then we can add 0,1,2 to the set. | ||||
618 | ConstantRange Span = | ||||
619 | ConstantRange::makeExactICmpRegion(ICI->getPredicate(), C->getValue()); | ||||
620 | |||||
621 | // Shift the range if the compare is fed by an add. This is the range | ||||
622 | // compare idiom as emitted by instcombine. | ||||
623 | Value *CandidateVal = I->getOperand(0); | ||||
624 | if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) { | ||||
625 | Span = Span.subtract(*RHSC); | ||||
626 | CandidateVal = RHSVal; | ||||
627 | } | ||||
628 | |||||
629 | // If this is an and/!= check, then we are looking to build the set of | ||||
630 | // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into | ||||
631 | // x != 0 && x != 1. | ||||
632 | if (!isEQ) | ||||
633 | Span = Span.inverse(); | ||||
634 | |||||
635 | // If there are a ton of values, we don't want to make a ginormous switch. | ||||
636 | if (Span.isSizeLargerThan(8) || Span.isEmptySet()) { | ||||
637 | return false; | ||||
638 | } | ||||
639 | |||||
640 | // If we already have a value for the switch, it has to match! | ||||
641 | if (!setValueOnce(CandidateVal)) | ||||
642 | return false; | ||||
643 | |||||
644 | // Add all values from the range to the set | ||||
645 | for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp) | ||||
646 | Vals.push_back(ConstantInt::get(I->getContext(), Tmp)); | ||||
647 | |||||
648 | UsedICmps++; | ||||
649 | return true; | ||||
650 | } | ||||
651 | |||||
652 | /// Given a potentially 'or'd or 'and'd together collection of icmp | ||||
653 | /// eq/ne/lt/gt instructions that compare a value against a constant, extract | ||||
654 | /// the value being compared, and stick the list constants into the Vals | ||||
655 | /// vector. | ||||
656 | /// One "Extra" case is allowed to differ from the other. | ||||
657 | void gather(Value *V) { | ||||
658 | bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value())); | ||||
659 | |||||
660 | // Keep a stack (SmallVector for efficiency) for depth-first traversal | ||||
661 | SmallVector<Value *, 8> DFT; | ||||
662 | SmallPtrSet<Value *, 8> Visited; | ||||
663 | |||||
664 | // Initialize | ||||
665 | Visited.insert(V); | ||||
666 | DFT.push_back(V); | ||||
667 | |||||
668 | while (!DFT.empty()) { | ||||
669 | V = DFT.pop_back_val(); | ||||
670 | |||||
671 | if (Instruction *I = dyn_cast<Instruction>(V)) { | ||||
672 | // If it is a || (or && depending on isEQ), process the operands. | ||||
673 | Value *Op0, *Op1; | ||||
674 | if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))) | ||||
675 | : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) { | ||||
676 | if (Visited.insert(Op1).second) | ||||
677 | DFT.push_back(Op1); | ||||
678 | if (Visited.insert(Op0).second) | ||||
679 | DFT.push_back(Op0); | ||||
680 | |||||
681 | continue; | ||||
682 | } | ||||
683 | |||||
684 | // Try to match the current instruction | ||||
685 | if (matchInstruction(I, isEQ)) | ||||
686 | // Match succeed, continue the loop | ||||
687 | continue; | ||||
688 | } | ||||
689 | |||||
690 | // One element of the sequence of || (or &&) could not be match as a | ||||
691 | // comparison against the same value as the others. | ||||
692 | // We allow only one "Extra" case to be checked before the switch | ||||
693 | if (!Extra) { | ||||
694 | Extra = V; | ||||
695 | continue; | ||||
696 | } | ||||
697 | // Failed to parse a proper sequence, abort now | ||||
698 | CompValue = nullptr; | ||||
699 | break; | ||||
700 | } | ||||
701 | } | ||||
702 | }; | ||||
703 | |||||
704 | } // end anonymous namespace | ||||
705 | |||||
706 | static void EraseTerminatorAndDCECond(Instruction *TI, | ||||
707 | MemorySSAUpdater *MSSAU = nullptr) { | ||||
708 | Instruction *Cond = nullptr; | ||||
709 | if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { | ||||
710 | Cond = dyn_cast<Instruction>(SI->getCondition()); | ||||
711 | } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { | ||||
712 | if (BI->isConditional()) | ||||
713 | Cond = dyn_cast<Instruction>(BI->getCondition()); | ||||
714 | } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) { | ||||
715 | Cond = dyn_cast<Instruction>(IBI->getAddress()); | ||||
716 | } | ||||
717 | |||||
718 | TI->eraseFromParent(); | ||||
719 | if (Cond) | ||||
720 | RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU); | ||||
721 | } | ||||
722 | |||||
723 | /// Return true if the specified terminator checks | ||||
724 | /// to see if a value is equal to constant integer value. | ||||
725 | Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) { | ||||
726 | Value *CV = nullptr; | ||||
727 | if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { | ||||
728 | // Do not permit merging of large switch instructions into their | ||||
729 | // predecessors unless there is only one predecessor. | ||||
730 | if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors())) | ||||
731 | CV = SI->getCondition(); | ||||
732 | } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) | ||||
733 | if (BI->isConditional() && BI->getCondition()->hasOneUse()) | ||||
734 | if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) { | ||||
735 | if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL)) | ||||
736 | CV = ICI->getOperand(0); | ||||
737 | } | ||||
738 | |||||
739 | // Unwrap any lossless ptrtoint cast. | ||||
740 | if (CV) { | ||||
741 | if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) { | ||||
742 | Value *Ptr = PTII->getPointerOperand(); | ||||
743 | if (PTII->getType() == DL.getIntPtrType(Ptr->getType())) | ||||
744 | CV = Ptr; | ||||
745 | } | ||||
746 | } | ||||
747 | return CV; | ||||
748 | } | ||||
749 | |||||
750 | /// Given a value comparison instruction, | ||||
751 | /// decode all of the 'cases' that it represents and return the 'default' block. | ||||
752 | BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases( | ||||
753 | Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) { | ||||
754 | if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { | ||||
755 | Cases.reserve(SI->getNumCases()); | ||||
756 | for (auto Case : SI->cases()) | ||||
757 | Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(), | ||||
758 | Case.getCaseSuccessor())); | ||||
759 | return SI->getDefaultDest(); | ||||
760 | } | ||||
761 | |||||
762 | BranchInst *BI = cast<BranchInst>(TI); | ||||
763 | ICmpInst *ICI = cast<ICmpInst>(BI->getCondition()); | ||||
764 | BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE); | ||||
765 | Cases.push_back(ValueEqualityComparisonCase( | ||||
766 | GetConstantInt(ICI->getOperand(1), DL), Succ)); | ||||
767 | return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ); | ||||
768 | } | ||||
769 | |||||
770 | /// Given a vector of bb/value pairs, remove any entries | ||||
771 | /// in the list that match the specified block. | ||||
772 | static void | ||||
773 | EliminateBlockCases(BasicBlock *BB, | ||||
774 | std::vector<ValueEqualityComparisonCase> &Cases) { | ||||
775 | llvm::erase_value(Cases, BB); | ||||
776 | } | ||||
777 | |||||
778 | /// Return true if there are any keys in C1 that exist in C2 as well. | ||||
779 | static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1, | ||||
780 | std::vector<ValueEqualityComparisonCase> &C2) { | ||||
781 | std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2; | ||||
782 | |||||
783 | // Make V1 be smaller than V2. | ||||
784 | if (V1->size() > V2->size()) | ||||
785 | std::swap(V1, V2); | ||||
786 | |||||
787 | if (V1->empty()) | ||||
788 | return false; | ||||
789 | if (V1->size() == 1) { | ||||
790 | // Just scan V2. | ||||
791 | ConstantInt *TheVal = (*V1)[0].Value; | ||||
792 | for (unsigned i = 0, e = V2->size(); i != e; ++i) | ||||
793 | if (TheVal == (*V2)[i].Value) | ||||
794 | return true; | ||||
795 | } | ||||
796 | |||||
797 | // Otherwise, just sort both lists and compare element by element. | ||||
798 | array_pod_sort(V1->begin(), V1->end()); | ||||
799 | array_pod_sort(V2->begin(), V2->end()); | ||||
800 | unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size(); | ||||
801 | while (i1 != e1 && i2 != e2) { | ||||
802 | if ((*V1)[i1].Value == (*V2)[i2].Value) | ||||
803 | return true; | ||||
804 | if ((*V1)[i1].Value < (*V2)[i2].Value) | ||||
805 | ++i1; | ||||
806 | else | ||||
807 | ++i2; | ||||
808 | } | ||||
809 | return false; | ||||
810 | } | ||||
811 | |||||
812 | // Set branch weights on SwitchInst. This sets the metadata if there is at | ||||
813 | // least one non-zero weight. | ||||
814 | static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) { | ||||
815 | // Check that there is at least one non-zero weight. Otherwise, pass | ||||
816 | // nullptr to setMetadata which will erase the existing metadata. | ||||
817 | MDNode *N = nullptr; | ||||
818 | if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; })) | ||||
819 | N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights); | ||||
820 | SI->setMetadata(LLVMContext::MD_prof, N); | ||||
821 | } | ||||
822 | |||||
823 | // Similar to the above, but for branch and select instructions that take | ||||
824 | // exactly 2 weights. | ||||
825 | static void setBranchWeights(Instruction *I, uint32_t TrueWeight, | ||||
826 | uint32_t FalseWeight) { | ||||
827 | assert(isa<BranchInst>(I) || isa<SelectInst>(I))((void)0); | ||||
828 | // Check that there is at least one non-zero weight. Otherwise, pass | ||||
829 | // nullptr to setMetadata which will erase the existing metadata. | ||||
830 | MDNode *N = nullptr; | ||||
831 | if (TrueWeight || FalseWeight) | ||||
832 | N = MDBuilder(I->getParent()->getContext()) | ||||
833 | .createBranchWeights(TrueWeight, FalseWeight); | ||||
834 | I->setMetadata(LLVMContext::MD_prof, N); | ||||
835 | } | ||||
836 | |||||
837 | /// If TI is known to be a terminator instruction and its block is known to | ||||
838 | /// only have a single predecessor block, check to see if that predecessor is | ||||
839 | /// also a value comparison with the same value, and if that comparison | ||||
840 | /// determines the outcome of this comparison. If so, simplify TI. This does a | ||||
841 | /// very limited form of jump threading. | ||||
842 | bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor( | ||||
843 | Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) { | ||||
844 | Value *PredVal = isValueEqualityComparison(Pred->getTerminator()); | ||||
845 | if (!PredVal) | ||||
846 | return false; // Not a value comparison in predecessor. | ||||
847 | |||||
848 | Value *ThisVal = isValueEqualityComparison(TI); | ||||
849 | assert(ThisVal && "This isn't a value comparison!!")((void)0); | ||||
850 | if (ThisVal != PredVal) | ||||
851 | return false; // Different predicates. | ||||
852 | |||||
853 | // TODO: Preserve branch weight metadata, similarly to how | ||||
854 | // FoldValueComparisonIntoPredecessors preserves it. | ||||
855 | |||||
856 | // Find out information about when control will move from Pred to TI's block. | ||||
857 | std::vector<ValueEqualityComparisonCase> PredCases; | ||||
858 | BasicBlock *PredDef = | ||||
859 | GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases); | ||||
860 | EliminateBlockCases(PredDef, PredCases); // Remove default from cases. | ||||
861 | |||||
862 | // Find information about how control leaves this block. | ||||
863 | std::vector<ValueEqualityComparisonCase> ThisCases; | ||||
864 | BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases); | ||||
865 | EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases. | ||||
866 | |||||
867 | // If TI's block is the default block from Pred's comparison, potentially | ||||
868 | // simplify TI based on this knowledge. | ||||
869 | if (PredDef == TI->getParent()) { | ||||
870 | // If we are here, we know that the value is none of those cases listed in | ||||
871 | // PredCases. If there are any cases in ThisCases that are in PredCases, we | ||||
872 | // can simplify TI. | ||||
873 | if (!ValuesOverlap(PredCases, ThisCases)) | ||||
874 | return false; | ||||
875 | |||||
876 | if (isa<BranchInst>(TI)) { | ||||
877 | // Okay, one of the successors of this condbr is dead. Convert it to a | ||||
878 | // uncond br. | ||||
879 | assert(ThisCases.size() == 1 && "Branch can only have one case!")((void)0); | ||||
880 | // Insert the new branch. | ||||
881 | Instruction *NI = Builder.CreateBr(ThisDef); | ||||
882 | (void)NI; | ||||
883 | |||||
884 | // Remove PHI node entries for the dead edge. | ||||
885 | ThisCases[0].Dest->removePredecessor(PredDef); | ||||
886 | |||||
887 | LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { } while (false) | ||||
888 | << "Through successor TI: " << *TI << "Leaving: " << *NIdo { } while (false) | ||||
889 | << "\n")do { } while (false); | ||||
890 | |||||
891 | EraseTerminatorAndDCECond(TI); | ||||
892 | |||||
893 | if (DTU) | ||||
894 | DTU->applyUpdates( | ||||
895 | {{DominatorTree::Delete, PredDef, ThisCases[0].Dest}}); | ||||
896 | |||||
897 | return true; | ||||
898 | } | ||||
899 | |||||
900 | SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI); | ||||
901 | // Okay, TI has cases that are statically dead, prune them away. | ||||
902 | SmallPtrSet<Constant *, 16> DeadCases; | ||||
903 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | ||||
904 | DeadCases.insert(PredCases[i].Value); | ||||
905 | |||||
906 | LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { } while (false) | ||||
907 | << "Through successor TI: " << *TI)do { } while (false); | ||||
908 | |||||
909 | SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases; | ||||
910 | for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) { | ||||
911 | --i; | ||||
912 | auto *Successor = i->getCaseSuccessor(); | ||||
913 | if (DTU) | ||||
914 | ++NumPerSuccessorCases[Successor]; | ||||
915 | if (DeadCases.count(i->getCaseValue())) { | ||||
916 | Successor->removePredecessor(PredDef); | ||||
917 | SI.removeCase(i); | ||||
918 | if (DTU) | ||||
919 | --NumPerSuccessorCases[Successor]; | ||||
920 | } | ||||
921 | } | ||||
922 | |||||
923 | if (DTU) { | ||||
924 | std::vector<DominatorTree::UpdateType> Updates; | ||||
925 | for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases) | ||||
926 | if (I.second == 0) | ||||
927 | Updates.push_back({DominatorTree::Delete, PredDef, I.first}); | ||||
928 | DTU->applyUpdates(Updates); | ||||
929 | } | ||||
930 | |||||
931 | LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n")do { } while (false); | ||||
932 | return true; | ||||
933 | } | ||||
934 | |||||
935 | // Otherwise, TI's block must correspond to some matched value. Find out | ||||
936 | // which value (or set of values) this is. | ||||
937 | ConstantInt *TIV = nullptr; | ||||
938 | BasicBlock *TIBB = TI->getParent(); | ||||
939 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | ||||
940 | if (PredCases[i].Dest == TIBB) { | ||||
941 | if (TIV) | ||||
942 | return false; // Cannot handle multiple values coming to this block. | ||||
943 | TIV = PredCases[i].Value; | ||||
944 | } | ||||
945 | assert(TIV && "No edge from pred to succ?")((void)0); | ||||
946 | |||||
947 | // Okay, we found the one constant that our value can be if we get into TI's | ||||
948 | // BB. Find out which successor will unconditionally be branched to. | ||||
949 | BasicBlock *TheRealDest = nullptr; | ||||
950 | for (unsigned i = 0, e = ThisCases.size(); i != e; ++i) | ||||
951 | if (ThisCases[i].Value == TIV) { | ||||
952 | TheRealDest = ThisCases[i].Dest; | ||||
953 | break; | ||||
954 | } | ||||
955 | |||||
956 | // If not handled by any explicit cases, it is handled by the default case. | ||||
957 | if (!TheRealDest) | ||||
958 | TheRealDest = ThisDef; | ||||
959 | |||||
960 | SmallPtrSet<BasicBlock *, 2> RemovedSuccs; | ||||
961 | |||||
962 | // Remove PHI node entries for dead edges. | ||||
963 | BasicBlock *CheckEdge = TheRealDest; | ||||
964 | for (BasicBlock *Succ : successors(TIBB)) | ||||
965 | if (Succ != CheckEdge) { | ||||
966 | if (Succ != TheRealDest) | ||||
967 | RemovedSuccs.insert(Succ); | ||||
968 | Succ->removePredecessor(TIBB); | ||||
969 | } else | ||||
970 | CheckEdge = nullptr; | ||||
971 | |||||
972 | // Insert the new branch. | ||||
973 | Instruction *NI = Builder.CreateBr(TheRealDest); | ||||
974 | (void)NI; | ||||
975 | |||||
976 | LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { } while (false) | ||||
977 | << "Through successor TI: " << *TI << "Leaving: " << *NIdo { } while (false) | ||||
978 | << "\n")do { } while (false); | ||||
979 | |||||
980 | EraseTerminatorAndDCECond(TI); | ||||
981 | if (DTU) { | ||||
982 | SmallVector<DominatorTree::UpdateType, 2> Updates; | ||||
983 | Updates.reserve(RemovedSuccs.size()); | ||||
984 | for (auto *RemovedSucc : RemovedSuccs) | ||||
985 | Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc}); | ||||
986 | DTU->applyUpdates(Updates); | ||||
987 | } | ||||
988 | return true; | ||||
989 | } | ||||
990 | |||||
991 | namespace { | ||||
992 | |||||
993 | /// This class implements a stable ordering of constant | ||||
994 | /// integers that does not depend on their address. This is important for | ||||
995 | /// applications that sort ConstantInt's to ensure uniqueness. | ||||
996 | struct ConstantIntOrdering { | ||||
997 | bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const { | ||||
998 | return LHS->getValue().ult(RHS->getValue()); | ||||
999 | } | ||||
1000 | }; | ||||
1001 | |||||
1002 | } // end anonymous namespace | ||||
1003 | |||||
1004 | static int ConstantIntSortPredicate(ConstantInt *const *P1, | ||||
1005 | ConstantInt *const *P2) { | ||||
1006 | const ConstantInt *LHS = *P1; | ||||
1007 | const ConstantInt *RHS = *P2; | ||||
1008 | if (LHS == RHS) | ||||
1009 | return 0; | ||||
1010 | return LHS->getValue().ult(RHS->getValue()) ? 1 : -1; | ||||
1011 | } | ||||
1012 | |||||
1013 | static inline bool HasBranchWeights(const Instruction *I) { | ||||
1014 | MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof); | ||||
1015 | if (ProfMD && ProfMD->getOperand(0)) | ||||
1016 | if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0))) | ||||
1017 | return MDS->getString().equals("branch_weights"); | ||||
1018 | |||||
1019 | return false; | ||||
1020 | } | ||||
1021 | |||||
1022 | /// Get Weights of a given terminator, the default weight is at the front | ||||
1023 | /// of the vector. If TI is a conditional eq, we need to swap the branch-weight | ||||
1024 | /// metadata. | ||||
1025 | static void GetBranchWeights(Instruction *TI, | ||||
1026 | SmallVectorImpl<uint64_t> &Weights) { | ||||
1027 | MDNode *MD = TI->getMetadata(LLVMContext::MD_prof); | ||||
1028 | assert(MD)((void)0); | ||||
1029 | for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) { | ||||
1030 | ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i)); | ||||
1031 | Weights.push_back(CI->getValue().getZExtValue()); | ||||
1032 | } | ||||
1033 | |||||
1034 | // If TI is a conditional eq, the default case is the false case, | ||||
1035 | // and the corresponding branch-weight data is at index 2. We swap the | ||||
1036 | // default weight to be the first entry. | ||||
1037 | if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { | ||||
1038 | assert(Weights.size() == 2)((void)0); | ||||
1039 | ICmpInst *ICI = cast<ICmpInst>(BI->getCondition()); | ||||
1040 | if (ICI->getPredicate() == ICmpInst::ICMP_EQ) | ||||
1041 | std::swap(Weights.front(), Weights.back()); | ||||
1042 | } | ||||
1043 | } | ||||
1044 | |||||
1045 | /// Keep halving the weights until all can fit in uint32_t. | ||||
1046 | static void FitWeights(MutableArrayRef<uint64_t> Weights) { | ||||
1047 | uint64_t Max = *std::max_element(Weights.begin(), Weights.end()); | ||||
1048 | if (Max > UINT_MAX(2147483647 *2U +1U)) { | ||||
1049 | unsigned Offset = 32 - countLeadingZeros(Max); | ||||
1050 | for (uint64_t &I : Weights) | ||||
1051 | I >>= Offset; | ||||
1052 | } | ||||
1053 | } | ||||
1054 | |||||
1055 | static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses( | ||||
1056 | BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap) { | ||||
1057 | Instruction *PTI = PredBlock->getTerminator(); | ||||
1058 | |||||
1059 | // If we have bonus instructions, clone them into the predecessor block. | ||||
1060 | // Note that there may be multiple predecessor blocks, so we cannot move | ||||
1061 | // bonus instructions to a predecessor block. | ||||
1062 | for (Instruction &BonusInst : *BB) { | ||||
1063 | if (isa<DbgInfoIntrinsic>(BonusInst) || BonusInst.isTerminator()) | ||||
1064 | continue; | ||||
1065 | |||||
1066 | Instruction *NewBonusInst = BonusInst.clone(); | ||||
1067 | |||||
1068 | if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) { | ||||
1069 | // Unless the instruction has the same !dbg location as the original | ||||
1070 | // branch, drop it. When we fold the bonus instructions we want to make | ||||
1071 | // sure we reset their debug locations in order to avoid stepping on | ||||
1072 | // dead code caused by folding dead branches. | ||||
1073 | NewBonusInst->setDebugLoc(DebugLoc()); | ||||
1074 | } | ||||
1075 | |||||
1076 | RemapInstruction(NewBonusInst, VMap, | ||||
1077 | RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); | ||||
1078 | VMap[&BonusInst] = NewBonusInst; | ||||
1079 | |||||
1080 | // If we moved a load, we cannot any longer claim any knowledge about | ||||
1081 | // its potential value. The previous information might have been valid | ||||
1082 | // only given the branch precondition. | ||||
1083 | // For an analogous reason, we must also drop all the metadata whose | ||||
1084 | // semantics we don't understand. We *can* preserve !annotation, because | ||||
1085 | // it is tied to the instruction itself, not the value or position. | ||||
1086 | // Similarly strip attributes on call parameters that may cause UB in | ||||
1087 | // location the call is moved to. | ||||
1088 | NewBonusInst->dropUndefImplyingAttrsAndUnknownMetadata( | ||||
1089 | LLVMContext::MD_annotation); | ||||
1090 | |||||
1091 | PredBlock->getInstList().insert(PTI->getIterator(), NewBonusInst); | ||||
1092 | NewBonusInst->takeName(&BonusInst); | ||||
1093 | BonusInst.setName(NewBonusInst->getName() + ".old"); | ||||
1094 | |||||
1095 | // Update (liveout) uses of bonus instructions, | ||||
1096 | // now that the bonus instruction has been cloned into predecessor. | ||||
1097 | // Note that we expect to be in a block-closed SSA form for this to work! | ||||
1098 | for (Use &U : make_early_inc_range(BonusInst.uses())) { | ||||
1099 | auto *UI = cast<Instruction>(U.getUser()); | ||||
1100 | auto *PN = dyn_cast<PHINode>(UI); | ||||
1101 | if (!PN) { | ||||
1102 | assert(UI->getParent() == BB && BonusInst.comesBefore(UI) &&((void)0) | ||||
1103 | "If the user is not a PHI node, then it should be in the same "((void)0) | ||||
1104 | "block as, and come after, the original bonus instruction.")((void)0); | ||||
1105 | continue; // Keep using the original bonus instruction. | ||||
1106 | } | ||||
1107 | // Is this the block-closed SSA form PHI node? | ||||
1108 | if (PN->getIncomingBlock(U) == BB) | ||||
1109 | continue; // Great, keep using the original bonus instruction. | ||||
1110 | // The only other alternative is an "use" when coming from | ||||
1111 | // the predecessor block - here we should refer to the cloned bonus instr. | ||||
1112 | assert(PN->getIncomingBlock(U) == PredBlock &&((void)0) | ||||
1113 | "Not in block-closed SSA form?")((void)0); | ||||
1114 | U.set(NewBonusInst); | ||||
1115 | } | ||||
1116 | } | ||||
1117 | } | ||||
1118 | |||||
1119 | bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding( | ||||
1120 | Instruction *TI, Value *&CV, Instruction *PTI, IRBuilder<> &Builder) { | ||||
1121 | BasicBlock *BB = TI->getParent(); | ||||
1122 | BasicBlock *Pred = PTI->getParent(); | ||||
1123 | |||||
1124 | SmallVector<DominatorTree::UpdateType, 32> Updates; | ||||
1125 | |||||
1126 | // Figure out which 'cases' to copy from SI to PSI. | ||||
1127 | std::vector<ValueEqualityComparisonCase> BBCases; | ||||
1128 | BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases); | ||||
1129 | |||||
1130 | std::vector<ValueEqualityComparisonCase> PredCases; | ||||
1131 | BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases); | ||||
1132 | |||||
1133 | // Based on whether the default edge from PTI goes to BB or not, fill in | ||||
1134 | // PredCases and PredDefault with the new switch cases we would like to | ||||
1135 | // build. | ||||
1136 | SmallMapVector<BasicBlock *, int, 8> NewSuccessors; | ||||
1137 | |||||
1138 | // Update the branch weight metadata along the way | ||||
1139 | SmallVector<uint64_t, 8> Weights; | ||||
1140 | bool PredHasWeights = HasBranchWeights(PTI); | ||||
1141 | bool SuccHasWeights = HasBranchWeights(TI); | ||||
1142 | |||||
1143 | if (PredHasWeights) { | ||||
1144 | GetBranchWeights(PTI, Weights); | ||||
1145 | // branch-weight metadata is inconsistent here. | ||||
1146 | if (Weights.size() != 1 + PredCases.size()) | ||||
1147 | PredHasWeights = SuccHasWeights = false; | ||||
1148 | } else if (SuccHasWeights) | ||||
1149 | // If there are no predecessor weights but there are successor weights, | ||||
1150 | // populate Weights with 1, which will later be scaled to the sum of | ||||
1151 | // successor's weights | ||||
1152 | Weights.assign(1 + PredCases.size(), 1); | ||||
1153 | |||||
1154 | SmallVector<uint64_t, 8> SuccWeights; | ||||
1155 | if (SuccHasWeights) { | ||||
1156 | GetBranchWeights(TI, SuccWeights); | ||||
1157 | // branch-weight metadata is inconsistent here. | ||||
1158 | if (SuccWeights.size() != 1 + BBCases.size()) | ||||
1159 | PredHasWeights = SuccHasWeights = false; | ||||
1160 | } else if (PredHasWeights) | ||||
1161 | SuccWeights.assign(1 + BBCases.size(), 1); | ||||
1162 | |||||
1163 | if (PredDefault == BB) { | ||||
1164 | // If this is the default destination from PTI, only the edges in TI | ||||
1165 | // that don't occur in PTI, or that branch to BB will be activated. | ||||
1166 | std::set<ConstantInt *, ConstantIntOrdering> PTIHandled; | ||||
1167 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | ||||
1168 | if (PredCases[i].Dest != BB) | ||||
1169 | PTIHandled.insert(PredCases[i].Value); | ||||
1170 | else { | ||||
1171 | // The default destination is BB, we don't need explicit targets. | ||||
1172 | std::swap(PredCases[i], PredCases.back()); | ||||
1173 | |||||
1174 | if (PredHasWeights || SuccHasWeights) { | ||||
1175 | // Increase weight for the default case. | ||||
1176 | Weights[0] += Weights[i + 1]; | ||||
1177 | std::swap(Weights[i + 1], Weights.back()); | ||||
1178 | Weights.pop_back(); | ||||
1179 | } | ||||
1180 | |||||
1181 | PredCases.pop_back(); | ||||
1182 | --i; | ||||
1183 | --e; | ||||
1184 | } | ||||
1185 | |||||
1186 | // Reconstruct the new switch statement we will be building. | ||||
1187 | if (PredDefault != BBDefault) { | ||||
1188 | PredDefault->removePredecessor(Pred); | ||||
1189 | if (DTU && PredDefault != BB) | ||||
1190 | Updates.push_back({DominatorTree::Delete, Pred, PredDefault}); | ||||
1191 | PredDefault = BBDefault; | ||||
1192 | ++NewSuccessors[BBDefault]; | ||||
1193 | } | ||||
1194 | |||||
1195 | unsigned CasesFromPred = Weights.size(); | ||||
1196 | uint64_t ValidTotalSuccWeight = 0; | ||||
1197 | for (unsigned i = 0, e = BBCases.size(); i != e; ++i) | ||||
1198 | if (!PTIHandled.count(BBCases[i].Value) && BBCases[i].Dest != BBDefault) { | ||||
1199 | PredCases.push_back(BBCases[i]); | ||||
1200 | ++NewSuccessors[BBCases[i].Dest]; | ||||
1201 | if (SuccHasWeights || PredHasWeights) { | ||||
1202 | // The default weight is at index 0, so weight for the ith case | ||||
1203 | // should be at index i+1. Scale the cases from successor by | ||||
1204 | // PredDefaultWeight (Weights[0]). | ||||
1205 | Weights.push_back(Weights[0] * SuccWeights[i + 1]); | ||||
1206 | ValidTotalSuccWeight += SuccWeights[i + 1]; | ||||
1207 | } | ||||
1208 | } | ||||
1209 | |||||
1210 | if (SuccHasWeights || PredHasWeights) { | ||||
1211 | ValidTotalSuccWeight += SuccWeights[0]; | ||||
1212 | // Scale the cases from predecessor by ValidTotalSuccWeight. | ||||
1213 | for (unsigned i = 1; i < CasesFromPred; ++i) | ||||
1214 | Weights[i] *= ValidTotalSuccWeight; | ||||
1215 | // Scale the default weight by SuccDefaultWeight (SuccWeights[0]). | ||||
1216 | Weights[0] *= SuccWeights[0]; | ||||
1217 | } | ||||
1218 | } else { | ||||
1219 | // If this is not the default destination from PSI, only the edges | ||||
1220 | // in SI that occur in PSI with a destination of BB will be | ||||
1221 | // activated. | ||||
1222 | std::set<ConstantInt *, ConstantIntOrdering> PTIHandled; | ||||
1223 | std::map<ConstantInt *, uint64_t> WeightsForHandled; | ||||
1224 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | ||||
1225 | if (PredCases[i].Dest == BB) { | ||||
1226 | PTIHandled.insert(PredCases[i].Value); | ||||
1227 | |||||
1228 | if (PredHasWeights || SuccHasWeights) { | ||||
1229 | WeightsForHandled[PredCases[i].Value] = Weights[i + 1]; | ||||
1230 | std::swap(Weights[i + 1], Weights.back()); | ||||
1231 | Weights.pop_back(); | ||||
1232 | } | ||||
1233 | |||||
1234 | std::swap(PredCases[i], PredCases.back()); | ||||
1235 | PredCases.pop_back(); | ||||
1236 | --i; | ||||
1237 | --e; | ||||
1238 | } | ||||
1239 | |||||
1240 | // Okay, now we know which constants were sent to BB from the | ||||
1241 | // predecessor. Figure out where they will all go now. | ||||
1242 | for (unsigned i = 0, e = BBCases.size(); i != e; ++i) | ||||
1243 | if (PTIHandled.count(BBCases[i].Value)) { | ||||
1244 | // If this is one we are capable of getting... | ||||
1245 | if (PredHasWeights || SuccHasWeights) | ||||
1246 | Weights.push_back(WeightsForHandled[BBCases[i].Value]); | ||||
1247 | PredCases.push_back(BBCases[i]); | ||||
1248 | ++NewSuccessors[BBCases[i].Dest]; | ||||
1249 | PTIHandled.erase(BBCases[i].Value); // This constant is taken care of | ||||
1250 | } | ||||
1251 | |||||
1252 | // If there are any constants vectored to BB that TI doesn't handle, | ||||
1253 | // they must go to the default destination of TI. | ||||
1254 | for (ConstantInt *I : PTIHandled) { | ||||
1255 | if (PredHasWeights || SuccHasWeights) | ||||
1256 | Weights.push_back(WeightsForHandled[I]); | ||||
1257 | PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault)); | ||||
1258 | ++NewSuccessors[BBDefault]; | ||||
1259 | } | ||||
1260 | } | ||||
1261 | |||||
1262 | // Okay, at this point, we know which new successor Pred will get. Make | ||||
1263 | // sure we update the number of entries in the PHI nodes for these | ||||
1264 | // successors. | ||||
1265 | SmallPtrSet<BasicBlock *, 2> SuccsOfPred; | ||||
1266 | if (DTU) { | ||||
1267 | SuccsOfPred = {succ_begin(Pred), succ_end(Pred)}; | ||||
1268 | Updates.reserve(Updates.size() + NewSuccessors.size()); | ||||
1269 | } | ||||
1270 | for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor : | ||||
1271 | NewSuccessors) { | ||||
1272 | for (auto I : seq(0, NewSuccessor.second)) { | ||||
1273 | (void)I; | ||||
1274 | AddPredecessorToBlock(NewSuccessor.first, Pred, BB); | ||||
1275 | } | ||||
1276 | if (DTU && !SuccsOfPred.contains(NewSuccessor.first)) | ||||
1277 | Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first}); | ||||
1278 | } | ||||
1279 | |||||
1280 | Builder.SetInsertPoint(PTI); | ||||
1281 | // Convert pointer to int before we switch. | ||||
1282 | if (CV->getType()->isPointerTy()) { | ||||
1283 | CV = | ||||
1284 | Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), "magicptr"); | ||||
1285 | } | ||||
1286 | |||||
1287 | // Now that the successors are updated, create the new Switch instruction. | ||||
1288 | SwitchInst *NewSI = Builder.CreateSwitch(CV, PredDefault, PredCases.size()); | ||||
1289 | NewSI->setDebugLoc(PTI->getDebugLoc()); | ||||
1290 | for (ValueEqualityComparisonCase &V : PredCases) | ||||
1291 | NewSI->addCase(V.Value, V.Dest); | ||||
1292 | |||||
1293 | if (PredHasWeights || SuccHasWeights) { | ||||
1294 | // Halve the weights if any of them cannot fit in an uint32_t | ||||
1295 | FitWeights(Weights); | ||||
1296 | |||||
1297 | SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); | ||||
1298 | |||||
1299 | setBranchWeights(NewSI, MDWeights); | ||||
1300 | } | ||||
1301 | |||||
1302 | EraseTerminatorAndDCECond(PTI); | ||||
1303 | |||||
1304 | // Okay, last check. If BB is still a successor of PSI, then we must | ||||
1305 | // have an infinite loop case. If so, add an infinitely looping block | ||||
1306 | // to handle the case to preserve the behavior of the code. | ||||
1307 | BasicBlock *InfLoopBlock = nullptr; | ||||
1308 | for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i) | ||||
1309 | if (NewSI->getSuccessor(i) == BB) { | ||||
1310 | if (!InfLoopBlock) { | ||||
1311 | // Insert it at the end of the function, because it's either code, | ||||
1312 | // or it won't matter if it's hot. :) | ||||
1313 | InfLoopBlock = | ||||
1314 | BasicBlock::Create(BB->getContext(), "infloop", BB->getParent()); | ||||
1315 | BranchInst::Create(InfLoopBlock, InfLoopBlock); | ||||
1316 | if (DTU) | ||||
1317 | Updates.push_back( | ||||
1318 | {DominatorTree::Insert, InfLoopBlock, InfLoopBlock}); | ||||
1319 | } | ||||
1320 | NewSI->setSuccessor(i, InfLoopBlock); | ||||
1321 | } | ||||
1322 | |||||
1323 | if (DTU) { | ||||
1324 | if (InfLoopBlock) | ||||
1325 | Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock}); | ||||
1326 | |||||
1327 | Updates.push_back({DominatorTree::Delete, Pred, BB}); | ||||
1328 | |||||
1329 | DTU->applyUpdates(Updates); | ||||
1330 | } | ||||
1331 | |||||
1332 | ++NumFoldValueComparisonIntoPredecessors; | ||||
1333 | return true; | ||||
1334 | } | ||||
1335 | |||||
1336 | /// The specified terminator is a value equality comparison instruction | ||||
1337 | /// (either a switch or a branch on "X == c"). | ||||
1338 | /// See if any of the predecessors of the terminator block are value comparisons | ||||
1339 | /// on the same value. If so, and if safe to do so, fold them together. | ||||
1340 | bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI, | ||||
1341 | IRBuilder<> &Builder) { | ||||
1342 | BasicBlock *BB = TI->getParent(); | ||||
1343 | Value *CV = isValueEqualityComparison(TI); // CondVal | ||||
1344 | assert(CV && "Not a comparison?")((void)0); | ||||
1345 | |||||
1346 | bool Changed = false; | ||||
1347 | |||||
1348 | SmallSetVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB)); | ||||
1349 | while (!Preds.empty()) { | ||||
1350 | BasicBlock *Pred = Preds.pop_back_val(); | ||||
1351 | Instruction *PTI = Pred->getTerminator(); | ||||
1352 | |||||
1353 | // Don't try to fold into itself. | ||||
1354 | if (Pred == BB) | ||||
1355 | continue; | ||||
1356 | |||||
1357 | // See if the predecessor is a comparison with the same value. | ||||
1358 | Value *PCV = isValueEqualityComparison(PTI); // PredCondVal | ||||
1359 | if (PCV != CV) | ||||
1360 | continue; | ||||
1361 | |||||
1362 | SmallSetVector<BasicBlock *, 4> FailBlocks; | ||||
1363 | if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) { | ||||
1364 | for (auto *Succ : FailBlocks) { | ||||
1365 | if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split", DTU)) | ||||
1366 | return false; | ||||
1367 | } | ||||
1368 | } | ||||
1369 | |||||
1370 | PerformValueComparisonIntoPredecessorFolding(TI, CV, PTI, Builder); | ||||
1371 | Changed = true; | ||||
1372 | } | ||||
1373 | return Changed; | ||||
1374 | } | ||||
1375 | |||||
1376 | // If we would need to insert a select that uses the value of this invoke | ||||
1377 | // (comments in HoistThenElseCodeToIf explain why we would need to do this), we | ||||
1378 | // can't hoist the invoke, as there is nowhere to put the select in this case. | ||||
1379 | static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2, | ||||
1380 | Instruction *I1, Instruction *I2) { | ||||
1381 | for (BasicBlock *Succ : successors(BB1)) { | ||||
1382 | for (const PHINode &PN : Succ->phis()) { | ||||
1383 | Value *BB1V = PN.getIncomingValueForBlock(BB1); | ||||
1384 | Value *BB2V = PN.getIncomingValueForBlock(BB2); | ||||
1385 | if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) { | ||||
1386 | return false; | ||||
1387 | } | ||||
1388 | } | ||||
1389 | } | ||||
1390 | return true; | ||||
1391 | } | ||||
1392 | |||||
1393 | static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false); | ||||
1394 | |||||
1395 | /// Given a conditional branch that goes to BB1 and BB2, hoist any common code | ||||
1396 | /// in the two blocks up into the branch block. The caller of this function | ||||
1397 | /// guarantees that BI's block dominates BB1 and BB2. If EqTermsOnly is given, | ||||
1398 | /// only perform hoisting in case both blocks only contain a terminator. In that | ||||
1399 | /// case, only the original BI will be replaced and selects for PHIs are added. | ||||
1400 | bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI, | ||||
1401 | const TargetTransformInfo &TTI, | ||||
1402 | bool EqTermsOnly) { | ||||
1403 | // This does very trivial matching, with limited scanning, to find identical | ||||
1404 | // instructions in the two blocks. In particular, we don't want to get into | ||||
1405 | // O(M*N) situations here where M and N are the sizes of BB1 and BB2. As | ||||
1406 | // such, we currently just scan for obviously identical instructions in an | ||||
1407 | // identical order. | ||||
1408 | BasicBlock *BB1 = BI->getSuccessor(0); // The true destination. | ||||
1409 | BasicBlock *BB2 = BI->getSuccessor(1); // The false destination | ||||
1410 | |||||
1411 | // If either of the blocks has it's address taken, then we can't do this fold, | ||||
1412 | // because the code we'd hoist would no longer run when we jump into the block | ||||
1413 | // by it's address. | ||||
1414 | if (BB1->hasAddressTaken() || BB2->hasAddressTaken()) | ||||
1415 | return false; | ||||
1416 | |||||
1417 | BasicBlock::iterator BB1_Itr = BB1->begin(); | ||||
1418 | BasicBlock::iterator BB2_Itr = BB2->begin(); | ||||
1419 | |||||
1420 | Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++; | ||||
1421 | // Skip debug info if it is not identical. | ||||
1422 | DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1); | ||||
1423 | DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2); | ||||
1424 | if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) { | ||||
1425 | while (isa<DbgInfoIntrinsic>(I1)) | ||||
1426 | I1 = &*BB1_Itr++; | ||||
1427 | while (isa<DbgInfoIntrinsic>(I2)) | ||||
1428 | I2 = &*BB2_Itr++; | ||||
1429 | } | ||||
1430 | // FIXME: Can we define a safety predicate for CallBr? | ||||
1431 | if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) || | ||||
1432 | (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) || | ||||
1433 | isa<CallBrInst>(I1)) | ||||
1434 | return false; | ||||
1435 | |||||
1436 | BasicBlock *BIParent = BI->getParent(); | ||||
1437 | |||||
1438 | bool Changed = false; | ||||
1439 | |||||
1440 | auto _ = make_scope_exit([&]() { | ||||
1441 | if (Changed) | ||||
1442 | ++NumHoistCommonCode; | ||||
1443 | }); | ||||
1444 | |||||
1445 | // Check if only hoisting terminators is allowed. This does not add new | ||||
1446 | // instructions to the hoist location. | ||||
1447 | if (EqTermsOnly) { | ||||
1448 | // Skip any debug intrinsics, as they are free to hoist. | ||||
1449 | auto *I1NonDbg = &*skipDebugIntrinsics(I1->getIterator()); | ||||
1450 | auto *I2NonDbg = &*skipDebugIntrinsics(I2->getIterator()); | ||||
1451 | if (!I1NonDbg->isIdenticalToWhenDefined(I2NonDbg)) | ||||
1452 | return false; | ||||
1453 | if (!I1NonDbg->isTerminator()) | ||||
1454 | return false; | ||||
1455 | // Now we know that we only need to hoist debug instrinsics and the | ||||
1456 | // terminator. Let the loop below handle those 2 cases. | ||||
1457 | } | ||||
1458 | |||||
1459 | do { | ||||
1460 | // If we are hoisting the terminator instruction, don't move one (making a | ||||
1461 | // broken BB), instead clone it, and remove BI. | ||||
1462 | if (I1->isTerminator()) | ||||
1463 | goto HoistTerminator; | ||||
1464 | |||||
1465 | // If we're going to hoist a call, make sure that the two instructions we're | ||||
1466 | // commoning/hoisting are both marked with musttail, or neither of them is | ||||
1467 | // marked as such. Otherwise, we might end up in a situation where we hoist | ||||
1468 | // from a block where the terminator is a `ret` to a block where the terminator | ||||
1469 | // is a `br`, and `musttail` calls expect to be followed by a return. | ||||
1470 | auto *C1 = dyn_cast<CallInst>(I1); | ||||
1471 | auto *C2 = dyn_cast<CallInst>(I2); | ||||
1472 | if (C1 && C2) | ||||
1473 | if (C1->isMustTailCall() != C2->isMustTailCall()) | ||||
1474 | return Changed; | ||||
1475 | |||||
1476 | if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2)) | ||||
1477 | return Changed; | ||||
1478 | |||||
1479 | // If any of the two call sites has nomerge attribute, stop hoisting. | ||||
1480 | if (const auto *CB1 = dyn_cast<CallBase>(I1)) | ||||
1481 | if (CB1->cannotMerge()) | ||||
1482 | return Changed; | ||||
1483 | if (const auto *CB2 = dyn_cast<CallBase>(I2)) | ||||
1484 | if (CB2->cannotMerge()) | ||||
1485 | return Changed; | ||||
1486 | |||||
1487 | if (isa<DbgInfoIntrinsic>(I1) || isa<DbgInfoIntrinsic>(I2)) { | ||||
1488 | assert (isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2))((void)0); | ||||
1489 | // The debug location is an integral part of a debug info intrinsic | ||||
1490 | // and can't be separated from it or replaced. Instead of attempting | ||||
1491 | // to merge locations, simply hoist both copies of the intrinsic. | ||||
1492 | BIParent->getInstList().splice(BI->getIterator(), | ||||
1493 | BB1->getInstList(), I1); | ||||
1494 | BIParent->getInstList().splice(BI->getIterator(), | ||||
1495 | BB2->getInstList(), I2); | ||||
1496 | Changed = true; | ||||
1497 | } else { | ||||
1498 | // For a normal instruction, we just move one to right before the branch, | ||||
1499 | // then replace all uses of the other with the first. Finally, we remove | ||||
1500 | // the now redundant second instruction. | ||||
1501 | BIParent->getInstList().splice(BI->getIterator(), | ||||
1502 | BB1->getInstList(), I1); | ||||
1503 | if (!I2->use_empty()) | ||||
1504 | I2->replaceAllUsesWith(I1); | ||||
1505 | I1->andIRFlags(I2); | ||||
1506 | unsigned KnownIDs[] = {LLVMContext::MD_tbaa, | ||||
1507 | LLVMContext::MD_range, | ||||
1508 | LLVMContext::MD_fpmath, | ||||
1509 | LLVMContext::MD_invariant_load, | ||||
1510 | LLVMContext::MD_nonnull, | ||||
1511 | LLVMContext::MD_invariant_group, | ||||
1512 | LLVMContext::MD_align, | ||||
1513 | LLVMContext::MD_dereferenceable, | ||||
1514 | LLVMContext::MD_dereferenceable_or_null, | ||||
1515 | LLVMContext::MD_mem_parallel_loop_access, | ||||
1516 | LLVMContext::MD_access_group, | ||||
1517 | LLVMContext::MD_preserve_access_index}; | ||||
1518 | combineMetadata(I1, I2, KnownIDs, true); | ||||
1519 | |||||
1520 | // I1 and I2 are being combined into a single instruction. Its debug | ||||
1521 | // location is the merged locations of the original instructions. | ||||
1522 | I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc()); | ||||
1523 | |||||
1524 | I2->eraseFromParent(); | ||||
1525 | Changed = true; | ||||
1526 | } | ||||
1527 | ++NumHoistCommonInstrs; | ||||
1528 | |||||
1529 | I1 = &*BB1_Itr++; | ||||
1530 | I2 = &*BB2_Itr++; | ||||
1531 | // Skip debug info if it is not identical. | ||||
1532 | DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1); | ||||
1533 | DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2); | ||||
1534 | if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) { | ||||
1535 | while (isa<DbgInfoIntrinsic>(I1)) | ||||
1536 | I1 = &*BB1_Itr++; | ||||
1537 | while (isa<DbgInfoIntrinsic>(I2)) | ||||
1538 | I2 = &*BB2_Itr++; | ||||
1539 | } | ||||
1540 | } while (I1->isIdenticalToWhenDefined(I2)); | ||||
1541 | |||||
1542 | return true; | ||||
1543 | |||||
1544 | HoistTerminator: | ||||
1545 | // It may not be possible to hoist an invoke. | ||||
1546 | // FIXME: Can we define a safety predicate for CallBr? | ||||
1547 | if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) | ||||
1548 | return Changed; | ||||
1549 | |||||
1550 | // TODO: callbr hoisting currently disabled pending further study. | ||||
1551 | if (isa<CallBrInst>(I1)) | ||||
1552 | return Changed; | ||||
1553 | |||||
1554 | for (BasicBlock *Succ : successors(BB1)) { | ||||
1555 | for (PHINode &PN : Succ->phis()) { | ||||
1556 | Value *BB1V = PN.getIncomingValueForBlock(BB1); | ||||
1557 | Value *BB2V = PN.getIncomingValueForBlock(BB2); | ||||
1558 | if (BB1V == BB2V) | ||||
1559 | continue; | ||||
1560 | |||||
1561 | // Check for passingValueIsAlwaysUndefined here because we would rather | ||||
1562 | // eliminate undefined control flow then converting it to a select. | ||||
1563 | if (passingValueIsAlwaysUndefined(BB1V, &PN) || | ||||
1564 | passingValueIsAlwaysUndefined(BB2V, &PN)) | ||||
1565 | return Changed; | ||||
1566 | |||||
1567 | if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V)) | ||||
1568 | return Changed; | ||||
1569 | if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V)) | ||||
1570 | return Changed; | ||||
1571 | } | ||||
1572 | } | ||||
1573 | |||||
1574 | // Okay, it is safe to hoist the terminator. | ||||
1575 | Instruction *NT = I1->clone(); | ||||
1576 | BIParent->getInstList().insert(BI->getIterator(), NT); | ||||
1577 | if (!NT->getType()->isVoidTy()) { | ||||
1578 | I1->replaceAllUsesWith(NT); | ||||
1579 | I2->replaceAllUsesWith(NT); | ||||
1580 | NT->takeName(I1); | ||||
1581 | } | ||||
1582 | Changed = true; | ||||
1583 | ++NumHoistCommonInstrs; | ||||
1584 | |||||
1585 | // Ensure terminator gets a debug location, even an unknown one, in case | ||||
1586 | // it involves inlinable calls. | ||||
1587 | NT->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc()); | ||||
1588 | |||||
1589 | // PHIs created below will adopt NT's merged DebugLoc. | ||||
1590 | IRBuilder<NoFolder> Builder(NT); | ||||
1591 | |||||
1592 | // Hoisting one of the terminators from our successor is a great thing. | ||||
1593 | // Unfortunately, the successors of the if/else blocks may have PHI nodes in | ||||
1594 | // them. If they do, all PHI entries for BB1/BB2 must agree for all PHI | ||||
1595 | // nodes, so we insert select instruction to compute the final result. | ||||
1596 | std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects; | ||||
1597 | for (BasicBlock *Succ : successors(BB1)) { | ||||
1598 | for (PHINode &PN : Succ->phis()) { | ||||
1599 | Value *BB1V = PN.getIncomingValueForBlock(BB1); | ||||
1600 | Value *BB2V = PN.getIncomingValueForBlock(BB2); | ||||
1601 | if (BB1V == BB2V) | ||||
1602 | continue; | ||||
1603 | |||||
1604 | // These values do not agree. Insert a select instruction before NT | ||||
1605 | // that determines the right value. | ||||
1606 | SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)]; | ||||
1607 | if (!SI) { | ||||
1608 | // Propagate fast-math-flags from phi node to its replacement select. | ||||
1609 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | ||||
1610 | if (isa<FPMathOperator>(PN)) | ||||
1611 | Builder.setFastMathFlags(PN.getFastMathFlags()); | ||||
1612 | |||||
1613 | SI = cast<SelectInst>( | ||||
1614 | Builder.CreateSelect(BI->getCondition(), BB1V, BB2V, | ||||
1615 | BB1V->getName() + "." + BB2V->getName(), BI)); | ||||
1616 | } | ||||
1617 | |||||
1618 | // Make the PHI node use the select for all incoming values for BB1/BB2 | ||||
1619 | for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) | ||||
1620 | if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2) | ||||
1621 | PN.setIncomingValue(i, SI); | ||||
1622 | } | ||||
1623 | } | ||||
1624 | |||||
1625 | SmallVector<DominatorTree::UpdateType, 4> Updates; | ||||
1626 | |||||
1627 | // Update any PHI nodes in our new successors. | ||||
1628 | for (BasicBlock *Succ : successors(BB1)) { | ||||
1629 | AddPredecessorToBlock(Succ, BIParent, BB1); | ||||
1630 | if (DTU) | ||||
1631 | Updates.push_back({DominatorTree::Insert, BIParent, Succ}); | ||||
1632 | } | ||||
1633 | |||||
1634 | if (DTU) | ||||
1635 | for (BasicBlock *Succ : successors(BI)) | ||||
1636 | Updates.push_back({DominatorTree::Delete, BIParent, Succ}); | ||||
1637 | |||||
1638 | EraseTerminatorAndDCECond(BI); | ||||
1639 | if (DTU) | ||||
1640 | DTU->applyUpdates(Updates); | ||||
1641 | return Changed; | ||||
1642 | } | ||||
1643 | |||||
1644 | // Check lifetime markers. | ||||
1645 | static bool isLifeTimeMarker(const Instruction *I) { | ||||
1646 | if (auto II = dyn_cast<IntrinsicInst>(I)) { | ||||
1647 | switch (II->getIntrinsicID()) { | ||||
1648 | default: | ||||
1649 | break; | ||||
1650 | case Intrinsic::lifetime_start: | ||||
1651 | case Intrinsic::lifetime_end: | ||||
1652 | return true; | ||||
1653 | } | ||||
1654 | } | ||||
1655 | return false; | ||||
1656 | } | ||||
1657 | |||||
1658 | // TODO: Refine this. This should avoid cases like turning constant memcpy sizes | ||||
1659 | // into variables. | ||||
1660 | static bool replacingOperandWithVariableIsCheap(const Instruction *I, | ||||
1661 | int OpIdx) { | ||||
1662 | return !isa<IntrinsicInst>(I); | ||||
1663 | } | ||||
1664 | |||||
1665 | // All instructions in Insts belong to different blocks that all unconditionally | ||||
1666 | // branch to a common successor. Analyze each instruction and return true if it | ||||
1667 | // would be possible to sink them into their successor, creating one common | ||||
1668 | // instruction instead. For every value that would be required to be provided by | ||||
1669 | // PHI node (because an operand varies in each input block), add to PHIOperands. | ||||
1670 | static bool canSinkInstructions( | ||||
1671 | ArrayRef<Instruction *> Insts, | ||||
1672 | DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) { | ||||
1673 | // Prune out obviously bad instructions to move. Each instruction must have | ||||
1674 | // exactly zero or one use, and we check later that use is by a single, common | ||||
1675 | // PHI instruction in the successor. | ||||
1676 | bool HasUse = !Insts.front()->user_empty(); | ||||
1677 | for (auto *I : Insts) { | ||||
1678 | // These instructions may change or break semantics if moved. | ||||
1679 | if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) || | ||||
1680 | I->getType()->isTokenTy()) | ||||
1681 | return false; | ||||
1682 | |||||
1683 | // Do not try to sink an instruction in an infinite loop - it can cause | ||||
1684 | // this algorithm to infinite loop. | ||||
1685 | if (I->getParent()->getSingleSuccessor() == I->getParent()) | ||||
1686 | return false; | ||||
1687 | |||||
1688 | // Conservatively return false if I is an inline-asm instruction. Sinking | ||||
1689 | // and merging inline-asm instructions can potentially create arguments | ||||
1690 | // that cannot satisfy the inline-asm constraints. | ||||
1691 | // If the instruction has nomerge attribute, return false. | ||||
1692 | if (const auto *C = dyn_cast<CallBase>(I)) | ||||
1693 | if (C->isInlineAsm() || C->cannotMerge()) | ||||
1694 | return false; | ||||
1695 | |||||
1696 | // Each instruction must have zero or one use. | ||||
1697 | if (HasUse && !I->hasOneUse()) | ||||
1698 | return false; | ||||
1699 | if (!HasUse && !I->user_empty()) | ||||
1700 | return false; | ||||
1701 | } | ||||
1702 | |||||
1703 | const Instruction *I0 = Insts.front(); | ||||
1704 | for (auto *I : Insts) | ||||
1705 | if (!I->isSameOperationAs(I0)) | ||||
1706 | return false; | ||||
1707 | |||||
1708 | // All instructions in Insts are known to be the same opcode. If they have a | ||||
1709 | // use, check that the only user is a PHI or in the same block as the | ||||
1710 | // instruction, because if a user is in the same block as an instruction we're | ||||
1711 | // contemplating sinking, it must already be determined to be sinkable. | ||||
1712 | if (HasUse) { | ||||
1713 | auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); | ||||
1714 | auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0); | ||||
1715 | if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool { | ||||
1716 | auto *U = cast<Instruction>(*I->user_begin()); | ||||
1717 | return (PNUse && | ||||
1718 | PNUse->getParent() == Succ && | ||||
1719 | PNUse->getIncomingValueForBlock(I->getParent()) == I) || | ||||
1720 | U->getParent() == I->getParent(); | ||||
1721 | })) | ||||
1722 | return false; | ||||
1723 | } | ||||
1724 | |||||
1725 | // Because SROA can't handle speculating stores of selects, try not to sink | ||||
1726 | // loads, stores or lifetime markers of allocas when we'd have to create a | ||||
1727 | // PHI for the address operand. Also, because it is likely that loads or | ||||
1728 | // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink | ||||
1729 | // them. | ||||
1730 | // This can cause code churn which can have unintended consequences down | ||||
1731 | // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244. | ||||
1732 | // FIXME: This is a workaround for a deficiency in SROA - see | ||||
1733 | // https://llvm.org/bugs/show_bug.cgi?id=30188 | ||||
1734 | if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) { | ||||
1735 | return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts()); | ||||
1736 | })) | ||||
1737 | return false; | ||||
1738 | if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) { | ||||
1739 | return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts()); | ||||
1740 | })) | ||||
1741 | return false; | ||||
1742 | if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) { | ||||
1743 | return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts()); | ||||
1744 | })) | ||||
1745 | return false; | ||||
1746 | |||||
1747 | // For calls to be sinkable, they must all be indirect, or have same callee. | ||||
1748 | // I.e. if we have two direct calls to different callees, we don't want to | ||||
1749 | // turn that into an indirect call. Likewise, if we have an indirect call, | ||||
1750 | // and a direct call, we don't actually want to have a single indirect call. | ||||
1751 | if (isa<CallBase>(I0)) { | ||||
1752 | auto IsIndirectCall = [](const Instruction *I) { | ||||
1753 | return cast<CallBase>(I)->isIndirectCall(); | ||||
1754 | }; | ||||
1755 | bool HaveIndirectCalls = any_of(Insts, IsIndirectCall); | ||||
1756 | bool AllCallsAreIndirect = all_of(Insts, IsIndirectCall); | ||||
1757 | if (HaveIndirectCalls) { | ||||
1758 | if (!AllCallsAreIndirect) | ||||
1759 | return false; | ||||
1760 | } else { | ||||
1761 | // All callees must be identical. | ||||
1762 | Value *Callee = nullptr; | ||||
1763 | for (const Instruction *I : Insts) { | ||||
1764 | Value *CurrCallee = cast<CallBase>(I)->getCalledOperand(); | ||||
1765 | if (!Callee) | ||||
1766 | Callee = CurrCallee; | ||||
1767 | else if (Callee != CurrCallee) | ||||
1768 | return false; | ||||
1769 | } | ||||
1770 | } | ||||
1771 | } | ||||
1772 | |||||
1773 | for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) { | ||||
1774 | Value *Op = I0->getOperand(OI); | ||||
1775 | if (Op->getType()->isTokenTy()) | ||||
1776 | // Don't touch any operand of token type. | ||||
1777 | return false; | ||||
1778 | |||||
1779 | auto SameAsI0 = [&I0, OI](const Instruction *I) { | ||||
1780 | assert(I->getNumOperands() == I0->getNumOperands())((void)0); | ||||
1781 | return I->getOperand(OI) == I0->getOperand(OI); | ||||
1782 | }; | ||||
1783 | if (!all_of(Insts, SameAsI0)) { | ||||
1784 | if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) || | ||||
1785 | !canReplaceOperandWithVariable(I0, OI)) | ||||
1786 | // We can't create a PHI from this GEP. | ||||
1787 | return false; | ||||
1788 | for (auto *I : Insts) | ||||
1789 | PHIOperands[I].push_back(I->getOperand(OI)); | ||||
1790 | } | ||||
1791 | } | ||||
1792 | return true; | ||||
1793 | } | ||||
1794 | |||||
1795 | // Assuming canSinkInstructions(Blocks) has returned true, sink the last | ||||
1796 | // instruction of every block in Blocks to their common successor, commoning | ||||
1797 | // into one instruction. | ||||
1798 | static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { | ||||
1799 | auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0); | ||||
1800 | |||||
1801 | // canSinkInstructions returning true guarantees that every block has at | ||||
1802 | // least one non-terminator instruction. | ||||
1803 | SmallVector<Instruction*,4> Insts; | ||||
1804 | for (auto *BB : Blocks) { | ||||
1805 | Instruction *I = BB->getTerminator(); | ||||
1806 | do { | ||||
1807 | I = I->getPrevNode(); | ||||
1808 | } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front()); | ||||
1809 | if (!isa<DbgInfoIntrinsic>(I)) | ||||
1810 | Insts.push_back(I); | ||||
1811 | } | ||||
1812 | |||||
1813 | // The only checking we need to do now is that all users of all instructions | ||||
1814 | // are the same PHI node. canSinkInstructions should have checked this but | ||||
1815 | // it is slightly over-aggressive - it gets confused by commutative | ||||
1816 | // instructions so double-check it here. | ||||
1817 | Instruction *I0 = Insts.front(); | ||||
1818 | if (!I0->user_empty()) { | ||||
1819 | auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); | ||||
1820 | if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool { | ||||
1821 | auto *U = cast<Instruction>(*I->user_begin()); | ||||
1822 | return U == PNUse; | ||||
1823 | })) | ||||
1824 | return false; | ||||
1825 | } | ||||
1826 | |||||
1827 | // We don't need to do any more checking here; canSinkInstructions should | ||||
1828 | // have done it all for us. | ||||
1829 | SmallVector<Value*, 4> NewOperands; | ||||
1830 | for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) { | ||||
1831 | // This check is different to that in canSinkInstructions. There, we | ||||
1832 | // cared about the global view once simplifycfg (and instcombine) have | ||||
1833 | // completed - it takes into account PHIs that become trivially | ||||
1834 | // simplifiable. However here we need a more local view; if an operand | ||||
1835 | // differs we create a PHI and rely on instcombine to clean up the very | ||||
1836 | // small mess we may make. | ||||
1837 | bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) { | ||||
1838 | return I->getOperand(O) != I0->getOperand(O); | ||||
1839 | }); | ||||
1840 | if (!NeedPHI) { | ||||
1841 | NewOperands.push_back(I0->getOperand(O)); | ||||
1842 | continue; | ||||
1843 | } | ||||
1844 | |||||
1845 | // Create a new PHI in the successor block and populate it. | ||||
1846 | auto *Op = I0->getOperand(O); | ||||
1847 | assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!")((void)0); | ||||
1848 | auto *PN = PHINode::Create(Op->getType(), Insts.size(), | ||||
1849 | Op->getName() + ".sink", &BBEnd->front()); | ||||
1850 | for (auto *I : Insts) | ||||
1851 | PN->addIncoming(I->getOperand(O), I->getParent()); | ||||
1852 | NewOperands.push_back(PN); | ||||
1853 | } | ||||
1854 | |||||
1855 | // Arbitrarily use I0 as the new "common" instruction; remap its operands | ||||
1856 | // and move it to the start of the successor block. | ||||
1857 | for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) | ||||
1858 | I0->getOperandUse(O).set(NewOperands[O]); | ||||
1859 | I0->moveBefore(&*BBEnd->getFirstInsertionPt()); | ||||
1860 | |||||
1861 | // Update metadata and IR flags, and merge debug locations. | ||||
1862 | for (auto *I : Insts) | ||||
1863 | if (I != I0) { | ||||
1864 | // The debug location for the "common" instruction is the merged locations | ||||
1865 | // of all the commoned instructions. We start with the original location | ||||
1866 | // of the "common" instruction and iteratively merge each location in the | ||||
1867 | // loop below. | ||||
1868 | // This is an N-way merge, which will be inefficient if I0 is a CallInst. | ||||
1869 | // However, as N-way merge for CallInst is rare, so we use simplified API | ||||
1870 | // instead of using complex API for N-way merge. | ||||
1871 | I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc()); | ||||
1872 | combineMetadataForCSE(I0, I, true); | ||||
1873 | I0->andIRFlags(I); | ||||
1874 | } | ||||
1875 | |||||
1876 | if (!I0->user_empty()) { | ||||
1877 | // canSinkLastInstruction checked that all instructions were used by | ||||
1878 | // one and only one PHI node. Find that now, RAUW it to our common | ||||
1879 | // instruction and nuke it. | ||||
1880 | auto *PN = cast<PHINode>(*I0->user_begin()); | ||||
1881 | PN->replaceAllUsesWith(I0); | ||||
1882 | PN->eraseFromParent(); | ||||
1883 | } | ||||
1884 | |||||
1885 | // Finally nuke all instructions apart from the common instruction. | ||||
1886 | for (auto *I : Insts) | ||||
1887 | if (I != I0) | ||||
1888 | I->eraseFromParent(); | ||||
1889 | |||||
1890 | return true; | ||||
1891 | } | ||||
1892 | |||||
1893 | namespace { | ||||
1894 | |||||
1895 | // LockstepReverseIterator - Iterates through instructions | ||||
1896 | // in a set of blocks in reverse order from the first non-terminator. | ||||
1897 | // For example (assume all blocks have size n): | ||||
1898 | // LockstepReverseIterator I([B1, B2, B3]); | ||||
1899 | // *I-- = [B1[n], B2[n], B3[n]]; | ||||
1900 | // *I-- = [B1[n-1], B2[n-1], B3[n-1]]; | ||||
1901 | // *I-- = [B1[n-2], B2[n-2], B3[n-2]]; | ||||
1902 | // ... | ||||
1903 | class LockstepReverseIterator { | ||||
1904 | ArrayRef<BasicBlock*> Blocks; | ||||
1905 | SmallVector<Instruction*,4> Insts; | ||||
1906 | bool Fail; | ||||
1907 | |||||
1908 | public: | ||||
1909 | LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) { | ||||
1910 | reset(); | ||||
1911 | } | ||||
1912 | |||||
1913 | void reset() { | ||||
1914 | Fail = false; | ||||
1915 | Insts.clear(); | ||||
1916 | for (auto *BB : Blocks) { | ||||
1917 | Instruction *Inst = BB->getTerminator(); | ||||
1918 | for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) | ||||
1919 | Inst = Inst->getPrevNode(); | ||||
1920 | if (!Inst) { | ||||
1921 | // Block wasn't big enough. | ||||
1922 | Fail = true; | ||||
1923 | return; | ||||
1924 | } | ||||
1925 | Insts.push_back(Inst); | ||||
1926 | } | ||||
1927 | } | ||||
1928 | |||||
1929 | bool isValid() const { | ||||
1930 | return !Fail; | ||||
1931 | } | ||||
1932 | |||||
1933 | void operator--() { | ||||
1934 | if (Fail) | ||||
1935 | return; | ||||
1936 | for (auto *&Inst : Insts) { | ||||
1937 | for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) | ||||
1938 | Inst = Inst->getPrevNode(); | ||||
1939 | // Already at beginning of block. | ||||
1940 | if (!Inst) { | ||||
1941 | Fail = true; | ||||
1942 | return; | ||||
1943 | } | ||||
1944 | } | ||||
1945 | } | ||||
1946 | |||||
1947 | void operator++() { | ||||
1948 | if (Fail) | ||||
1949 | return; | ||||
1950 | for (auto *&Inst : Insts) { | ||||
1951 | for (Inst = Inst->getNextNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) | ||||
1952 | Inst = Inst->getNextNode(); | ||||
1953 | // Already at end of block. | ||||
1954 | if (!Inst) { | ||||
1955 | Fail = true; | ||||
1956 | return; | ||||
1957 | } | ||||
1958 | } | ||||
1959 | } | ||||
1960 | |||||
1961 | ArrayRef<Instruction*> operator * () const { | ||||
1962 | return Insts; | ||||
1963 | } | ||||
1964 | }; | ||||
1965 | |||||
1966 | } // end anonymous namespace | ||||
1967 | |||||
1968 | /// Check whether BB's predecessors end with unconditional branches. If it is | ||||
1969 | /// true, sink any common code from the predecessors to BB. | ||||
1970 | static bool SinkCommonCodeFromPredecessors(BasicBlock *BB, | ||||
1971 | DomTreeUpdater *DTU) { | ||||
1972 | // We support two situations: | ||||
1973 | // (1) all incoming arcs are unconditional | ||||
1974 | // (2) there are non-unconditional incoming arcs | ||||
1975 | // | ||||
1976 | // (2) is very common in switch defaults and | ||||
1977 | // else-if patterns; | ||||
1978 | // | ||||
1979 | // if (a) f(1); | ||||
1980 | // else if (b) f(2); | ||||
1981 | // | ||||
1982 | // produces: | ||||
1983 | // | ||||
1984 | // [if] | ||||
1985 | // / \ | ||||
1986 | // [f(1)] [if] | ||||
1987 | // | | \ | ||||
1988 | // | | | | ||||
1989 | // | [f(2)]| | ||||
1990 | // \ | / | ||||
1991 | // [ end ] | ||||
1992 | // | ||||
1993 | // [end] has two unconditional predecessor arcs and one conditional. The | ||||
1994 | // conditional refers to the implicit empty 'else' arc. This conditional | ||||
1995 | // arc can also be caused by an empty default block in a switch. | ||||
1996 | // | ||||
1997 | // In this case, we attempt to sink code from all *unconditional* arcs. | ||||
1998 | // If we can sink instructions from these arcs (determined during the scan | ||||
1999 | // phase below) we insert a common successor for all unconditional arcs and | ||||
2000 | // connect that to [end], to enable sinking: | ||||
2001 | // | ||||
2002 | // [if] | ||||
2003 | // / \ | ||||
2004 | // [x(1)] [if] | ||||
2005 | // | | \ | ||||
2006 | // | | \ | ||||
2007 | // | [x(2)] | | ||||
2008 | // \ / | | ||||
2009 | // [sink.split] | | ||||
2010 | // \ / | ||||
2011 | // [ end ] | ||||
2012 | // | ||||
2013 | SmallVector<BasicBlock*,4> UnconditionalPreds; | ||||
2014 | bool HaveNonUnconditionalPredecessors = false; | ||||
2015 | for (auto *PredBB : predecessors(BB)) { | ||||
2016 | auto *PredBr = dyn_cast<BranchInst>(PredBB->getTerminator()); | ||||
2017 | if (PredBr && PredBr->isUnconditional()) | ||||
2018 | UnconditionalPreds.push_back(PredBB); | ||||
2019 | else | ||||
2020 | HaveNonUnconditionalPredecessors = true; | ||||
2021 | } | ||||
2022 | if (UnconditionalPreds.size() < 2) | ||||
2023 | return false; | ||||
2024 | |||||
2025 | // We take a two-step approach to tail sinking. First we scan from the end of | ||||
2026 | // each block upwards in lockstep. If the n'th instruction from the end of each | ||||
2027 | // block can be sunk, those instructions are added to ValuesToSink and we | ||||
2028 | // carry on. If we can sink an instruction but need to PHI-merge some operands | ||||
2029 | // (because they're not identical in each instruction) we add these to | ||||
2030 | // PHIOperands. | ||||
2031 | int ScanIdx = 0; | ||||
2032 | SmallPtrSet<Value*,4> InstructionsToSink; | ||||
2033 | DenseMap<Instruction*, SmallVector<Value*,4>> PHIOperands; | ||||
2034 | LockstepReverseIterator LRI(UnconditionalPreds); | ||||
2035 | while (LRI.isValid() && | ||||
2036 | canSinkInstructions(*LRI, PHIOperands)) { | ||||
2037 | LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]do { } while (false) | ||||
2038 | << "\n")do { } while (false); | ||||
2039 | InstructionsToSink.insert((*LRI).begin(), (*LRI).end()); | ||||
2040 | ++ScanIdx; | ||||
2041 | --LRI; | ||||
2042 | } | ||||
2043 | |||||
2044 | // If no instructions can be sunk, early-return. | ||||
2045 | if (ScanIdx == 0) | ||||
2046 | return false; | ||||
2047 | |||||
2048 | // Okay, we *could* sink last ScanIdx instructions. But how many can we | ||||
2049 | // actually sink before encountering instruction that is unprofitable to sink? | ||||
2050 | auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) { | ||||
2051 | unsigned NumPHIdValues = 0; | ||||
2052 | for (auto *I : *LRI) | ||||
2053 | for (auto *V : PHIOperands[I]) { | ||||
2054 | if (InstructionsToSink.count(V) == 0) | ||||
2055 | ++NumPHIdValues; | ||||
2056 | // FIXME: this check is overly optimistic. We may end up not sinking | ||||
2057 | // said instruction, due to the very same profitability check. | ||||
2058 | // See @creating_too_many_phis in sink-common-code.ll. | ||||
2059 | } | ||||
2060 | LLVM_DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n")do { } while (false); | ||||
2061 | unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size(); | ||||
2062 | if ((NumPHIdValues % UnconditionalPreds.size()) != 0) | ||||
2063 | NumPHIInsts++; | ||||
2064 | |||||
2065 | return NumPHIInsts <= 1; | ||||
2066 | }; | ||||
2067 | |||||
2068 | // We've determined that we are going to sink last ScanIdx instructions, | ||||
2069 | // and recorded them in InstructionsToSink. Now, some instructions may be | ||||
2070 | // unprofitable to sink. But that determination depends on the instructions | ||||
2071 | // that we are going to sink. | ||||
2072 | |||||
2073 | // First, forward scan: find the first instruction unprofitable to sink, | ||||
2074 | // recording all the ones that are profitable to sink. | ||||
2075 | // FIXME: would it be better, after we detect that not all are profitable. | ||||
2076 | // to either record the profitable ones, or erase the unprofitable ones? | ||||
2077 | // Maybe we need to choose (at runtime) the one that will touch least instrs? | ||||
2078 | LRI.reset(); | ||||
2079 | int Idx = 0; | ||||
2080 | SmallPtrSet<Value *, 4> InstructionsProfitableToSink; | ||||
2081 | while (Idx < ScanIdx) { | ||||
2082 | if (!ProfitableToSinkInstruction(LRI)) { | ||||
2083 | // Too many PHIs would be created. | ||||
2084 | LLVM_DEBUG(do { } while (false) | ||||
2085 | dbgs() << "SINK: stopping here, too many PHIs would be created!\n")do { } while (false); | ||||
2086 | break; | ||||
2087 | } | ||||
2088 | InstructionsProfitableToSink.insert((*LRI).begin(), (*LRI).end()); | ||||
2089 | --LRI; | ||||
2090 | ++Idx; | ||||
2091 | } | ||||
2092 | |||||
2093 | // If no instructions can be sunk, early-return. | ||||
2094 | if (Idx == 0) | ||||
2095 | return false; | ||||
2096 | |||||
2097 | // Did we determine that (only) some instructions are unprofitable to sink? | ||||
2098 | if (Idx < ScanIdx) { | ||||
2099 | // Okay, some instructions are unprofitable. | ||||
2100 | ScanIdx = Idx; | ||||
2101 | InstructionsToSink = InstructionsProfitableToSink; | ||||
2102 | |||||
2103 | // But, that may make other instructions unprofitable, too. | ||||
2104 | // So, do a backward scan, do any earlier instructions become unprofitable? | ||||
2105 | assert(!ProfitableToSinkInstruction(LRI) &&((void)0) | ||||
2106 | "We already know that the last instruction is unprofitable to sink")((void)0); | ||||
2107 | ++LRI; | ||||
2108 | --Idx; | ||||
2109 | while (Idx >= 0) { | ||||
2110 | // If we detect that an instruction becomes unprofitable to sink, | ||||
2111 | // all earlier instructions won't be sunk either, | ||||
2112 | // so preemptively keep InstructionsProfitableToSink in sync. | ||||
2113 | // FIXME: is this the most performant approach? | ||||
2114 | for (auto *I : *LRI) | ||||
2115 | InstructionsProfitableToSink.erase(I); | ||||
2116 | if (!ProfitableToSinkInstruction(LRI)) { | ||||
2117 | // Everything starting with this instruction won't be sunk. | ||||
2118 | ScanIdx = Idx; | ||||
2119 | InstructionsToSink = InstructionsProfitableToSink; | ||||
2120 | } | ||||
2121 | ++LRI; | ||||
2122 | --Idx; | ||||
2123 | } | ||||
2124 | } | ||||
2125 | |||||
2126 | // If no instructions can be sunk, early-return. | ||||
2127 | if (ScanIdx == 0) | ||||
2128 | return false; | ||||
2129 | |||||
2130 | bool Changed = false; | ||||
2131 | |||||
2132 | if (HaveNonUnconditionalPredecessors) { | ||||
2133 | // It is always legal to sink common instructions from unconditional | ||||
2134 | // predecessors. However, if not all predecessors are unconditional, | ||||
2135 | // this transformation might be pessimizing. So as a rule of thumb, | ||||
2136 | // don't do it unless we'd sink at least one non-speculatable instruction. | ||||
2137 | // See https://bugs.llvm.org/show_bug.cgi?id=30244 | ||||
2138 | LRI.reset(); | ||||
2139 | int Idx = 0; | ||||
2140 | bool Profitable = false; | ||||
2141 | while (Idx < ScanIdx) { | ||||
2142 | if (!isSafeToSpeculativelyExecute((*LRI)[0])) { | ||||
2143 | Profitable = true; | ||||
2144 | break; | ||||
2145 | } | ||||
2146 | --LRI; | ||||
2147 | ++Idx; | ||||
2148 | } | ||||
2149 | if (!Profitable) | ||||
2150 | return false; | ||||
2151 | |||||
2152 | LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n")do { } while (false); | ||||
2153 | // We have a conditional edge and we're going to sink some instructions. | ||||
2154 | // Insert a new block postdominating all blocks we're going to sink from. | ||||
2155 | if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU)) | ||||
2156 | // Edges couldn't be split. | ||||
2157 | return false; | ||||
2158 | Changed = true; | ||||
2159 | } | ||||
2160 | |||||
2161 | // Now that we've analyzed all potential sinking candidates, perform the | ||||
2162 | // actual sink. We iteratively sink the last non-terminator of the source | ||||
2163 | // blocks into their common successor unless doing so would require too | ||||
2164 | // many PHI instructions to be generated (currently only one PHI is allowed | ||||
2165 | // per sunk instruction). | ||||
2166 | // | ||||
2167 | // We can use InstructionsToSink to discount values needing PHI-merging that will | ||||
2168 | // actually be sunk in a later iteration. This allows us to be more | ||||
2169 | // aggressive in what we sink. This does allow a false positive where we | ||||
2170 | // sink presuming a later value will also be sunk, but stop half way through | ||||
2171 | // and never actually sink it which means we produce more PHIs than intended. | ||||
2172 | // This is unlikely in practice though. | ||||
2173 | int SinkIdx = 0; | ||||
2174 | for (; SinkIdx != ScanIdx; ++SinkIdx) { | ||||
2175 | LLVM_DEBUG(dbgs() << "SINK: Sink: "do { } while (false) | ||||
2176 | << *UnconditionalPreds[0]->getTerminator()->getPrevNode()do { } while (false) | ||||
2177 | << "\n")do { } while (false); | ||||
2178 | |||||
2179 | // Because we've sunk every instruction in turn, the current instruction to | ||||
2180 | // sink is always at index 0. | ||||
2181 | LRI.reset(); | ||||
2182 | |||||
2183 | if (!sinkLastInstruction(UnconditionalPreds)) { | ||||
2184 | LLVM_DEBUG(do { } while (false) | ||||
2185 | dbgs()do { } while (false) | ||||
2186 | << "SINK: stopping here, failed to actually sink instruction!\n")do { } while (false); | ||||
2187 | break; | ||||
2188 | } | ||||
2189 | |||||
2190 | NumSinkCommonInstrs++; | ||||
2191 | Changed = true; | ||||
2192 | } | ||||
2193 | if (SinkIdx != 0) | ||||
2194 | ++NumSinkCommonCode; | ||||
2195 | return Changed; | ||||
2196 | } | ||||
2197 | |||||
2198 | /// Determine if we can hoist sink a sole store instruction out of a | ||||
2199 | /// conditional block. | ||||
2200 | /// | ||||
2201 | /// We are looking for code like the following: | ||||
2202 | /// BrBB: | ||||
2203 | /// store i32 %add, i32* %arrayidx2 | ||||
2204 | /// ... // No other stores or function calls (we could be calling a memory | ||||
2205 | /// ... // function). | ||||
2206 | /// %cmp = icmp ult %x, %y | ||||
2207 | /// br i1 %cmp, label %EndBB, label %ThenBB | ||||
2208 | /// ThenBB: | ||||
2209 | /// store i32 %add5, i32* %arrayidx2 | ||||
2210 | /// br label EndBB | ||||
2211 | /// EndBB: | ||||
2212 | /// ... | ||||
2213 | /// We are going to transform this into: | ||||
2214 | /// BrBB: | ||||
2215 | /// store i32 %add, i32* %arrayidx2 | ||||
2216 | /// ... // | ||||
2217 | /// %cmp = icmp ult %x, %y | ||||
2218 | /// %add.add5 = select i1 %cmp, i32 %add, %add5 | ||||
2219 | /// store i32 %add.add5, i32* %arrayidx2 | ||||
2220 | /// ... | ||||
2221 | /// | ||||
2222 | /// \return The pointer to the value of the previous store if the store can be | ||||
2223 | /// hoisted into the predecessor block. 0 otherwise. | ||||
2224 | static Value *isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB, | ||||
2225 | BasicBlock *StoreBB, BasicBlock *EndBB) { | ||||
2226 | StoreInst *StoreToHoist = dyn_cast<StoreInst>(I); | ||||
2227 | if (!StoreToHoist) | ||||
2228 | return nullptr; | ||||
2229 | |||||
2230 | // Volatile or atomic. | ||||
2231 | if (!StoreToHoist->isSimple()) | ||||
2232 | return nullptr; | ||||
2233 | |||||
2234 | Value *StorePtr = StoreToHoist->getPointerOperand(); | ||||
2235 | Type *StoreTy = StoreToHoist->getValueOperand()->getType(); | ||||
2236 | |||||
2237 | // Look for a store to the same pointer in BrBB. | ||||
2238 | unsigned MaxNumInstToLookAt = 9; | ||||
2239 | // Skip pseudo probe intrinsic calls which are not really killing any memory | ||||
2240 | // accesses. | ||||
2241 | for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug(true))) { | ||||
2242 | if (!MaxNumInstToLookAt) | ||||
2243 | break; | ||||
2244 | --MaxNumInstToLookAt; | ||||
2245 | |||||
2246 | // Could be calling an instruction that affects memory like free(). | ||||
2247 | if (CurI.mayWriteToMemory() && !isa<StoreInst>(CurI)) | ||||
2248 | return nullptr; | ||||
2249 | |||||
2250 | if (auto *SI = dyn_cast<StoreInst>(&CurI)) { | ||||
2251 | // Found the previous store to same location and type. Make sure it is | ||||
2252 | // simple, to avoid introducing a spurious non-atomic write after an | ||||
2253 | // atomic write. | ||||
2254 | if (SI->getPointerOperand() == StorePtr && | ||||
2255 | SI->getValueOperand()->getType() == StoreTy && SI->isSimple()) | ||||
2256 | // Found the previous store, return its value operand. | ||||
2257 | return SI->getValueOperand(); | ||||
2258 | return nullptr; // Unknown store. | ||||
2259 | } | ||||
2260 | } | ||||
2261 | |||||
2262 | return nullptr; | ||||
2263 | } | ||||
2264 | |||||
2265 | /// Estimate the cost of the insertion(s) and check that the PHI nodes can be | ||||
2266 | /// converted to selects. | ||||
2267 | static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB, | ||||
2268 | BasicBlock *EndBB, | ||||
2269 | unsigned &SpeculatedInstructions, | ||||
2270 | InstructionCost &Cost, | ||||
2271 | const TargetTransformInfo &TTI) { | ||||
2272 | TargetTransformInfo::TargetCostKind CostKind = | ||||
2273 | BB->getParent()->hasMinSize() | ||||
2274 | ? TargetTransformInfo::TCK_CodeSize | ||||
2275 | : TargetTransformInfo::TCK_SizeAndLatency; | ||||
2276 | |||||
2277 | bool HaveRewritablePHIs = false; | ||||
2278 | for (PHINode &PN : EndBB->phis()) { | ||||
2279 | Value *OrigV = PN.getIncomingValueForBlock(BB); | ||||
2280 | Value *ThenV = PN.getIncomingValueForBlock(ThenBB); | ||||
2281 | |||||
2282 | // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf. | ||||
2283 | // Skip PHIs which are trivial. | ||||
2284 | if (ThenV == OrigV) | ||||
2285 | continue; | ||||
2286 | |||||
2287 | Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr, | ||||
2288 | CmpInst::BAD_ICMP_PREDICATE, CostKind); | ||||
2289 | |||||
2290 | // Don't convert to selects if we could remove undefined behavior instead. | ||||
2291 | if (passingValueIsAlwaysUndefined(OrigV, &PN) || | ||||
2292 | passingValueIsAlwaysUndefined(ThenV, &PN)) | ||||
2293 | return false; | ||||
2294 | |||||
2295 | HaveRewritablePHIs = true; | ||||
2296 | ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV); | ||||
2297 | ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV); | ||||
2298 | if (!OrigCE && !ThenCE) | ||||
2299 | continue; // Known safe and cheap. | ||||
2300 | |||||
2301 | if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) || | ||||
2302 | (OrigCE && !isSafeToSpeculativelyExecute(OrigCE))) | ||||
2303 | return false; | ||||
2304 | InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0; | ||||
2305 | InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0; | ||||
2306 | InstructionCost MaxCost = | ||||
2307 | 2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; | ||||
2308 | if (OrigCost + ThenCost > MaxCost) | ||||
2309 | return false; | ||||
2310 | |||||
2311 | // Account for the cost of an unfolded ConstantExpr which could end up | ||||
2312 | // getting expanded into Instructions. | ||||
2313 | // FIXME: This doesn't account for how many operations are combined in the | ||||
2314 | // constant expression. | ||||
2315 | ++SpeculatedInstructions; | ||||
2316 | if (SpeculatedInstructions > 1) | ||||
2317 | return false; | ||||
2318 | } | ||||
2319 | |||||
2320 | return HaveRewritablePHIs; | ||||
2321 | } | ||||
2322 | |||||
2323 | /// Speculate a conditional basic block flattening the CFG. | ||||
2324 | /// | ||||
2325 | /// Note that this is a very risky transform currently. Speculating | ||||
2326 | /// instructions like this is most often not desirable. Instead, there is an MI | ||||
2327 | /// pass which can do it with full awareness of the resource constraints. | ||||
2328 | /// However, some cases are "obvious" and we should do directly. An example of | ||||
2329 | /// this is speculating a single, reasonably cheap instruction. | ||||
2330 | /// | ||||
2331 | /// There is only one distinct advantage to flattening the CFG at the IR level: | ||||
2332 | /// it makes very common but simplistic optimizations such as are common in | ||||
2333 | /// instcombine and the DAG combiner more powerful by removing CFG edges and | ||||
2334 | /// modeling their effects with easier to reason about SSA value graphs. | ||||
2335 | /// | ||||
2336 | /// | ||||
2337 | /// An illustration of this transform is turning this IR: | ||||
2338 | /// \code | ||||
2339 | /// BB: | ||||
2340 | /// %cmp = icmp ult %x, %y | ||||
2341 | /// br i1 %cmp, label %EndBB, label %ThenBB | ||||
2342 | /// ThenBB: | ||||
2343 | /// %sub = sub %x, %y | ||||
2344 | /// br label BB2 | ||||
2345 | /// EndBB: | ||||
2346 | /// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ] | ||||
2347 | /// ... | ||||
2348 | /// \endcode | ||||
2349 | /// | ||||
2350 | /// Into this IR: | ||||
2351 | /// \code | ||||
2352 | /// BB: | ||||
2353 | /// %cmp = icmp ult %x, %y | ||||
2354 | /// %sub = sub %x, %y | ||||
2355 | /// %cond = select i1 %cmp, 0, %sub | ||||
2356 | /// ... | ||||
2357 | /// \endcode | ||||
2358 | /// | ||||
2359 | /// \returns true if the conditional block is removed. | ||||
2360 | bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB, | ||||
2361 | const TargetTransformInfo &TTI) { | ||||
2362 | // Be conservative for now. FP select instruction can often be expensive. | ||||
2363 | Value *BrCond = BI->getCondition(); | ||||
2364 | if (isa<FCmpInst>(BrCond)) | ||||
2365 | return false; | ||||
2366 | |||||
2367 | BasicBlock *BB = BI->getParent(); | ||||
2368 | BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0); | ||||
2369 | InstructionCost Budget = | ||||
2370 | PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; | ||||
2371 | |||||
2372 | // If ThenBB is actually on the false edge of the conditional branch, remember | ||||
2373 | // to swap the select operands later. | ||||
2374 | bool Invert = false; | ||||
2375 | if (ThenBB != BI->getSuccessor(0)) { | ||||
2376 | assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?")((void)0); | ||||
2377 | Invert = true; | ||||
2378 | } | ||||
2379 | assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block")((void)0); | ||||
2380 | |||||
2381 | // If the branch is non-unpredictable, and is predicted to *not* branch to | ||||
2382 | // the `then` block, then avoid speculating it. | ||||
2383 | if (!BI->getMetadata(LLVMContext::MD_unpredictable)) { | ||||
2384 | uint64_t TWeight, FWeight; | ||||
2385 | if (BI->extractProfMetadata(TWeight, FWeight) && (TWeight + FWeight) != 0) { | ||||
2386 | uint64_t EndWeight = Invert ? TWeight : FWeight; | ||||
2387 | BranchProbability BIEndProb = | ||||
2388 | BranchProbability::getBranchProbability(EndWeight, TWeight + FWeight); | ||||
2389 | BranchProbability Likely = TTI.getPredictableBranchThreshold(); | ||||
2390 | if (BIEndProb >= Likely) | ||||
2391 | return false; | ||||
2392 | } | ||||
2393 | } | ||||
2394 | |||||
2395 | // Keep a count of how many times instructions are used within ThenBB when | ||||
2396 | // they are candidates for sinking into ThenBB. Specifically: | ||||
2397 | // - They are defined in BB, and | ||||
2398 | // - They have no side effects, and | ||||
2399 | // - All of their uses are in ThenBB. | ||||
2400 | SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts; | ||||
2401 | |||||
2402 | SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics; | ||||
2403 | |||||
2404 | unsigned SpeculatedInstructions = 0; | ||||
2405 | Value *SpeculatedStoreValue = nullptr; | ||||
2406 | StoreInst *SpeculatedStore = nullptr; | ||||
2407 | for (BasicBlock::iterator BBI = ThenBB->begin(), | ||||
2408 | BBE = std::prev(ThenBB->end()); | ||||
2409 | BBI != BBE; ++BBI) { | ||||
2410 | Instruction *I = &*BBI; | ||||
2411 | // Skip debug info. | ||||
2412 | if (isa<DbgInfoIntrinsic>(I)) { | ||||
2413 | SpeculatedDbgIntrinsics.push_back(I); | ||||
2414 | continue; | ||||
2415 | } | ||||
2416 | |||||
2417 | // Skip pseudo probes. The consequence is we lose track of the branch | ||||
2418 | // probability for ThenBB, which is fine since the optimization here takes | ||||
2419 | // place regardless of the branch probability. | ||||
2420 | if (isa<PseudoProbeInst>(I)) { | ||||
2421 | // The probe should be deleted so that it will not be over-counted when | ||||
2422 | // the samples collected on the non-conditional path are counted towards | ||||
2423 | // the conditional path. We leave it for the counts inference algorithm to | ||||
2424 | // figure out a proper count for an unknown probe. | ||||
2425 | SpeculatedDbgIntrinsics.push_back(I); | ||||
2426 | continue; | ||||
2427 | } | ||||
2428 | |||||
2429 | // Only speculatively execute a single instruction (not counting the | ||||
2430 | // terminator) for now. | ||||
2431 | ++SpeculatedInstructions; | ||||
2432 | if (SpeculatedInstructions > 1) | ||||
2433 | return false; | ||||
2434 | |||||
2435 | // Don't hoist the instruction if it's unsafe or expensive. | ||||
2436 | if (!isSafeToSpeculativelyExecute(I) && | ||||
2437 | !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore( | ||||
2438 | I, BB, ThenBB, EndBB)))) | ||||
2439 | return false; | ||||
2440 | if (!SpeculatedStoreValue && | ||||
2441 | computeSpeculationCost(I, TTI) > | ||||
2442 | PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic) | ||||
2443 | return false; | ||||
2444 | |||||
2445 | // Store the store speculation candidate. | ||||
2446 | if (SpeculatedStoreValue) | ||||
2447 | SpeculatedStore = cast<StoreInst>(I); | ||||
2448 | |||||
2449 | // Do not hoist the instruction if any of its operands are defined but not | ||||
2450 | // used in BB. The transformation will prevent the operand from | ||||
2451 | // being sunk into the use block. | ||||
2452 | for (Use &Op : I->operands()) { | ||||
2453 | Instruction *OpI = dyn_cast<Instruction>(Op); | ||||
2454 | if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects()) | ||||
2455 | continue; // Not a candidate for sinking. | ||||
2456 | |||||
2457 | ++SinkCandidateUseCounts[OpI]; | ||||
2458 | } | ||||
2459 | } | ||||
2460 | |||||
2461 | // Consider any sink candidates which are only used in ThenBB as costs for | ||||
2462 | // speculation. Note, while we iterate over a DenseMap here, we are summing | ||||
2463 | // and so iteration order isn't significant. | ||||
2464 | for (SmallDenseMap<Instruction *, unsigned, 4>::iterator | ||||
2465 | I = SinkCandidateUseCounts.begin(), | ||||
2466 | E = SinkCandidateUseCounts.end(); | ||||
2467 | I != E; ++I) | ||||
2468 | if (I->first->hasNUses(I->second)) { | ||||
2469 | ++SpeculatedInstructions; | ||||
2470 | if (SpeculatedInstructions > 1) | ||||
2471 | return false; | ||||
2472 | } | ||||
2473 | |||||
2474 | // Check that we can insert the selects and that it's not too expensive to do | ||||
2475 | // so. | ||||
2476 | bool Convert = SpeculatedStore != nullptr; | ||||
2477 | InstructionCost Cost = 0; | ||||
2478 | Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB, | ||||
2479 | SpeculatedInstructions, | ||||
2480 | Cost, TTI); | ||||
2481 | if (!Convert || Cost > Budget) | ||||
2482 | return false; | ||||
2483 | |||||
2484 | // If we get here, we can hoist the instruction and if-convert. | ||||
2485 | LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";)do { } while (false); | ||||
2486 | |||||
2487 | // Insert a select of the value of the speculated store. | ||||
2488 | if (SpeculatedStoreValue) { | ||||
2489 | IRBuilder<NoFolder> Builder(BI); | ||||
2490 | Value *TrueV = SpeculatedStore->getValueOperand(); | ||||
2491 | Value *FalseV = SpeculatedStoreValue; | ||||
2492 | if (Invert) | ||||
2493 | std::swap(TrueV, FalseV); | ||||
2494 | Value *S = Builder.CreateSelect( | ||||
2495 | BrCond, TrueV, FalseV, "spec.store.select", BI); | ||||
2496 | SpeculatedStore->setOperand(0, S); | ||||
2497 | SpeculatedStore->applyMergedLocation(BI->getDebugLoc(), | ||||
2498 | SpeculatedStore->getDebugLoc()); | ||||
2499 | } | ||||
2500 | |||||
2501 | // Metadata can be dependent on the condition we are hoisting above. | ||||
2502 | // Conservatively strip all metadata on the instruction. Drop the debug loc | ||||
2503 | // to avoid making it appear as if the condition is a constant, which would | ||||
2504 | // be misleading while debugging. | ||||
2505 | // Similarly strip attributes that maybe dependent on condition we are | ||||
2506 | // hoisting above. | ||||
2507 | for (auto &I : *ThenBB) { | ||||
2508 | if (!SpeculatedStoreValue || &I != SpeculatedStore) | ||||
2509 | I.setDebugLoc(DebugLoc()); | ||||
2510 | I.dropUndefImplyingAttrsAndUnknownMetadata(); | ||||
2511 | } | ||||
2512 | |||||
2513 | // Hoist the instructions. | ||||
2514 | BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(), | ||||
2515 | ThenBB->begin(), std::prev(ThenBB->end())); | ||||
2516 | |||||
2517 | // Insert selects and rewrite the PHI operands. | ||||
2518 | IRBuilder<NoFolder> Builder(BI); | ||||
2519 | for (PHINode &PN : EndBB->phis()) { | ||||
2520 | unsigned OrigI = PN.getBasicBlockIndex(BB); | ||||
2521 | unsigned ThenI = PN.getBasicBlockIndex(ThenBB); | ||||
2522 | Value *OrigV = PN.getIncomingValue(OrigI); | ||||
2523 | Value *ThenV = PN.getIncomingValue(ThenI); | ||||
2524 | |||||
2525 | // Skip PHIs which are trivial. | ||||
2526 | if (OrigV == ThenV) | ||||
2527 | continue; | ||||
2528 | |||||
2529 | // Create a select whose true value is the speculatively executed value and | ||||
2530 | // false value is the pre-existing value. Swap them if the branch | ||||
2531 | // destinations were inverted. | ||||
2532 | Value *TrueV = ThenV, *FalseV = OrigV; | ||||
2533 | if (Invert) | ||||
2534 | std::swap(TrueV, FalseV); | ||||
2535 | Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI); | ||||
2536 | PN.setIncomingValue(OrigI, V); | ||||
2537 | PN.setIncomingValue(ThenI, V); | ||||
2538 | } | ||||
2539 | |||||
2540 | // Remove speculated dbg intrinsics. | ||||
2541 | // FIXME: Is it possible to do this in a more elegant way? Moving/merging the | ||||
2542 | // dbg value for the different flows and inserting it after the select. | ||||
2543 | for (Instruction *I : SpeculatedDbgIntrinsics) | ||||
2544 | I->eraseFromParent(); | ||||
2545 | |||||
2546 | ++NumSpeculations; | ||||
2547 | return true; | ||||
2548 | } | ||||
2549 | |||||
2550 | /// Return true if we can thread a branch across this block. | ||||
2551 | static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) { | ||||
2552 | int Size = 0; | ||||
2553 | |||||
2554 | SmallPtrSet<const Value *, 32> EphValues; | ||||
2555 | auto IsEphemeral = [&](const Value *V) { | ||||
2556 | if (isa<AssumeInst>(V)) | ||||
2557 | return true; | ||||
2558 | return isSafeToSpeculativelyExecute(V) && | ||||
2559 | all_of(V->users(), | ||||
2560 | [&](const User *U) { return EphValues.count(U); }); | ||||
2561 | }; | ||||
2562 | |||||
2563 | // Walk the loop in reverse so that we can identify ephemeral values properly | ||||
2564 | // (values only feeding assumes). | ||||
2565 | for (Instruction &I : reverse(BB->instructionsWithoutDebug())) { | ||||
2566 | // Can't fold blocks that contain noduplicate or convergent calls. | ||||
2567 | if (CallInst *CI = dyn_cast<CallInst>(&I)) | ||||
2568 | if (CI->cannotDuplicate() || CI->isConvergent()) | ||||
2569 | return false; | ||||
2570 | |||||
2571 | // Ignore ephemeral values which are deleted during codegen. | ||||
2572 | if (IsEphemeral(&I)) | ||||
2573 | EphValues.insert(&I); | ||||
2574 | // We will delete Phis while threading, so Phis should not be accounted in | ||||
2575 | // block's size. | ||||
2576 | else if (!isa<PHINode>(I)) { | ||||
2577 | if (Size++ > MaxSmallBlockSize) | ||||
2578 | return false; // Don't clone large BB's. | ||||
2579 | } | ||||
2580 | |||||
2581 | // We can only support instructions that do not define values that are | ||||
2582 | // live outside of the current basic block. | ||||
2583 | for (User *U : I.users()) { | ||||
2584 | Instruction *UI = cast<Instruction>(U); | ||||
2585 | if (UI->getParent() != BB || isa<PHINode>(UI)) | ||||
2586 | return false; | ||||
2587 | } | ||||
2588 | |||||
2589 | // Looks ok, continue checking. | ||||
2590 | } | ||||
2591 | |||||
2592 | return true; | ||||
2593 | } | ||||
2594 | |||||
2595 | /// If we have a conditional branch on a PHI node value that is defined in the | ||||
2596 | /// same block as the branch and if any PHI entries are constants, thread edges | ||||
2597 | /// corresponding to that entry to be branches to their ultimate destination. | ||||
2598 | static bool FoldCondBranchOnPHI(BranchInst *BI, DomTreeUpdater *DTU, | ||||
2599 | const DataLayout &DL, AssumptionCache *AC) { | ||||
2600 | BasicBlock *BB = BI->getParent(); | ||||
2601 | PHINode *PN = dyn_cast<PHINode>(BI->getCondition()); | ||||
2602 | // NOTE: we currently cannot transform this case if the PHI node is used | ||||
2603 | // outside of the block. | ||||
2604 | if (!PN || PN->getParent() != BB || !PN->hasOneUse()) | ||||
2605 | return false; | ||||
2606 | |||||
2607 | // Degenerate case of a single entry PHI. | ||||
2608 | if (PN->getNumIncomingValues() == 1) { | ||||
2609 | FoldSingleEntryPHINodes(PN->getParent()); | ||||
2610 | return true; | ||||
2611 | } | ||||
2612 | |||||
2613 | // Now we know that this block has multiple preds and two succs. | ||||
2614 | if (!BlockIsSimpleEnoughToThreadThrough(BB)) | ||||
2615 | return false; | ||||
2616 | |||||
2617 | // Okay, this is a simple enough basic block. See if any phi values are | ||||
2618 | // constants. | ||||
2619 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { | ||||
2620 | ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i)); | ||||
2621 | if (!CB || !CB->getType()->isIntegerTy(1)) | ||||
2622 | continue; | ||||
2623 | |||||
2624 | // Okay, we now know that all edges from PredBB should be revectored to | ||||
2625 | // branch to RealDest. | ||||
2626 | BasicBlock *PredBB = PN->getIncomingBlock(i); | ||||
2627 | BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue()); | ||||
2628 | |||||
2629 | if (RealDest == BB) | ||||
2630 | continue; // Skip self loops. | ||||
2631 | // Skip if the predecessor's terminator is an indirect branch. | ||||
2632 | if (isa<IndirectBrInst>(PredBB->getTerminator())) | ||||
2633 | continue; | ||||
2634 | |||||
2635 | SmallVector<DominatorTree::UpdateType, 3> Updates; | ||||
2636 | |||||
2637 | // The dest block might have PHI nodes, other predecessors and other | ||||
2638 | // difficult cases. Instead of being smart about this, just insert a new | ||||
2639 | // block that jumps to the destination block, effectively splitting | ||||
2640 | // the edge we are about to create. | ||||
2641 | BasicBlock *EdgeBB = | ||||
2642 | BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge", | ||||
2643 | RealDest->getParent(), RealDest); | ||||
2644 | BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB); | ||||
2645 | if (DTU) | ||||
2646 | Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest}); | ||||
2647 | CritEdgeBranch->setDebugLoc(BI->getDebugLoc()); | ||||
2648 | |||||
2649 | // Update PHI nodes. | ||||
2650 | AddPredecessorToBlock(RealDest, EdgeBB, BB); | ||||
2651 | |||||
2652 | // BB may have instructions that are being threaded over. Clone these | ||||
2653 | // instructions into EdgeBB. We know that there will be no uses of the | ||||
2654 | // cloned instructions outside of EdgeBB. | ||||
2655 | BasicBlock::iterator InsertPt = EdgeBB->begin(); | ||||
2656 | DenseMap<Value *, Value *> TranslateMap; // Track translated values. | ||||
2657 | for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) { | ||||
2658 | if (PHINode *PN = dyn_cast<PHINode>(BBI)) { | ||||
2659 | TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB); | ||||
2660 | continue; | ||||
2661 | } | ||||
2662 | // Clone the instruction. | ||||
2663 | Instruction *N = BBI->clone(); | ||||
2664 | if (BBI->hasName()) | ||||
2665 | N->setName(BBI->getName() + ".c"); | ||||
2666 | |||||
2667 | // Update operands due to translation. | ||||
2668 | for (Use &Op : N->operands()) { | ||||
2669 | DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(Op); | ||||
2670 | if (PI != TranslateMap.end()) | ||||
2671 | Op = PI->second; | ||||
2672 | } | ||||
2673 | |||||
2674 | // Check for trivial simplification. | ||||
2675 | if (Value *V = SimplifyInstruction(N, {DL, nullptr, nullptr, AC})) { | ||||
2676 | if (!BBI->use_empty()) | ||||
2677 | TranslateMap[&*BBI] = V; | ||||
2678 | if (!N->mayHaveSideEffects()) { | ||||
2679 | N->deleteValue(); // Instruction folded away, don't need actual inst | ||||
2680 | N = nullptr; | ||||
2681 | } | ||||
2682 | } else { | ||||
2683 | if (!BBI->use_empty()) | ||||
2684 | TranslateMap[&*BBI] = N; | ||||
2685 | } | ||||
2686 | if (N) { | ||||
2687 | // Insert the new instruction into its new home. | ||||
2688 | EdgeBB->getInstList().insert(InsertPt, N); | ||||
2689 | |||||
2690 | // Register the new instruction with the assumption cache if necessary. | ||||
2691 | if (auto *Assume = dyn_cast<AssumeInst>(N)) | ||||
2692 | if (AC) | ||||
2693 | AC->registerAssumption(Assume); | ||||
2694 | } | ||||
2695 | } | ||||
2696 | |||||
2697 | // Loop over all of the edges from PredBB to BB, changing them to branch | ||||
2698 | // to EdgeBB instead. | ||||
2699 | Instruction *PredBBTI = PredBB->getTerminator(); | ||||
2700 | for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i) | ||||
2701 | if (PredBBTI->getSuccessor(i) == BB) { | ||||
2702 | BB->removePredecessor(PredBB); | ||||
2703 | PredBBTI->setSuccessor(i, EdgeBB); | ||||
2704 | } | ||||
2705 | |||||
2706 | if (DTU) { | ||||
2707 | Updates.push_back({DominatorTree::Insert, PredBB, EdgeBB}); | ||||
2708 | Updates.push_back({DominatorTree::Delete, PredBB, BB}); | ||||
2709 | |||||
2710 | DTU->applyUpdates(Updates); | ||||
2711 | } | ||||
2712 | |||||
2713 | // Recurse, simplifying any other constants. | ||||
2714 | return FoldCondBranchOnPHI(BI, DTU, DL, AC) || true; | ||||
2715 | } | ||||
2716 | |||||
2717 | return false; | ||||
2718 | } | ||||
2719 | |||||
2720 | /// Given a BB that starts with the specified two-entry PHI node, | ||||
2721 | /// see if we can eliminate it. | ||||
2722 | static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI, | ||||
2723 | DomTreeUpdater *DTU, const DataLayout &DL) { | ||||
2724 | // Ok, this is a two entry PHI node. Check to see if this is a simple "if | ||||
2725 | // statement", which has a very simple dominance structure. Basically, we | ||||
2726 | // are trying to find the condition that is being branched on, which | ||||
2727 | // subsequently causes this merge to happen. We really want control | ||||
2728 | // dependence information for this check, but simplifycfg can't keep it up | ||||
2729 | // to date, and this catches most of the cases we care about anyway. | ||||
2730 | BasicBlock *BB = PN->getParent(); | ||||
2731 | |||||
2732 | BasicBlock *IfTrue, *IfFalse; | ||||
2733 | BranchInst *DomBI = GetIfCondition(BB, IfTrue, IfFalse); | ||||
2734 | if (!DomBI) | ||||
2735 | return false; | ||||
2736 | Value *IfCond = DomBI->getCondition(); | ||||
2737 | // Don't bother if the branch will be constant folded trivially. | ||||
2738 | if (isa<ConstantInt>(IfCond)) | ||||
2739 | return false; | ||||
2740 | |||||
2741 | BasicBlock *DomBlock = DomBI->getParent(); | ||||
2742 | SmallVector<BasicBlock *, 2> IfBlocks; | ||||
2743 | llvm::copy_if( | ||||
2744 | PN->blocks(), std::back_inserter(IfBlocks), [](BasicBlock *IfBlock) { | ||||
2745 | return cast<BranchInst>(IfBlock->getTerminator())->isUnconditional(); | ||||
2746 | }); | ||||
2747 | assert((IfBlocks.size() == 1 || IfBlocks.size() == 2) &&((void)0) | ||||
2748 | "Will have either one or two blocks to speculate.")((void)0); | ||||
2749 | |||||
2750 | // If the branch is non-unpredictable, see if we either predictably jump to | ||||
2751 | // the merge bb (if we have only a single 'then' block), or if we predictably | ||||
2752 | // jump to one specific 'then' block (if we have two of them). | ||||
2753 | // It isn't beneficial to speculatively execute the code | ||||
2754 | // from the block that we know is predictably not entered. | ||||
2755 | if (!DomBI->getMetadata(LLVMContext::MD_unpredictable)) { | ||||
2756 | uint64_t TWeight, FWeight; | ||||
2757 | if (DomBI->extractProfMetadata(TWeight, FWeight) && | ||||
2758 | (TWeight + FWeight) != 0) { | ||||
2759 | BranchProbability BITrueProb = | ||||
2760 | BranchProbability::getBranchProbability(TWeight, TWeight + FWeight); | ||||
2761 | BranchProbability Likely = TTI.getPredictableBranchThreshold(); | ||||
2762 | BranchProbability BIFalseProb = BITrueProb.getCompl(); | ||||
2763 | if (IfBlocks.size() == 1) { | ||||
2764 | BranchProbability BIBBProb = | ||||
2765 | DomBI->getSuccessor(0) == BB ? BITrueProb : BIFalseProb; | ||||
2766 | if (BIBBProb >= Likely) | ||||
2767 | return false; | ||||
2768 | } else { | ||||
2769 | if (BITrueProb >= Likely || BIFalseProb >= Likely) | ||||
2770 | return false; | ||||
2771 | } | ||||
2772 | } | ||||
2773 | } | ||||
2774 | |||||
2775 | // Don't try to fold an unreachable block. For example, the phi node itself | ||||
2776 | // can't be the candidate if-condition for a select that we want to form. | ||||
2777 | if (auto *IfCondPhiInst = dyn_cast<PHINode>(IfCond)) | ||||
2778 | if (IfCondPhiInst->getParent() == BB) | ||||
2779 | return false; | ||||
2780 | |||||
2781 | // Okay, we found that we can merge this two-entry phi node into a select. | ||||
2782 | // Doing so would require us to fold *all* two entry phi nodes in this block. | ||||
2783 | // At some point this becomes non-profitable (particularly if the target | ||||
2784 | // doesn't support cmov's). Only do this transformation if there are two or | ||||
2785 | // fewer PHI nodes in this block. | ||||
2786 | unsigned NumPhis = 0; | ||||
2787 | for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I) | ||||
2788 | if (NumPhis > 2) | ||||
2789 | return false; | ||||
2790 | |||||
2791 | // Loop over the PHI's seeing if we can promote them all to select | ||||
2792 | // instructions. While we are at it, keep track of the instructions | ||||
2793 | // that need to be moved to the dominating block. | ||||
2794 | SmallPtrSet<Instruction *, 4> AggressiveInsts; | ||||
2795 | InstructionCost Cost = 0; | ||||
2796 | InstructionCost Budget = | ||||
2797 | TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; | ||||
2798 | |||||
2799 | bool Changed = false; | ||||
2800 | for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) { | ||||
2801 | PHINode *PN = cast<PHINode>(II++); | ||||
2802 | if (Value *V = SimplifyInstruction(PN, {DL, PN})) { | ||||
2803 | PN->replaceAllUsesWith(V); | ||||
2804 | PN->eraseFromParent(); | ||||
2805 | Changed = true; | ||||
2806 | continue; | ||||
2807 | } | ||||
2808 | |||||
2809 | if (!dominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts, | ||||
2810 | Cost, Budget, TTI) || | ||||
2811 | !dominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts, | ||||
2812 | Cost, Budget, TTI)) | ||||
2813 | return Changed; | ||||
2814 | } | ||||
2815 | |||||
2816 | // If we folded the first phi, PN dangles at this point. Refresh it. If | ||||
2817 | // we ran out of PHIs then we simplified them all. | ||||
2818 | PN = dyn_cast<PHINode>(BB->begin()); | ||||
2819 | if (!PN) | ||||
2820 | return true; | ||||
2821 | |||||
2822 | // Return true if at least one of these is a 'not', and another is either | ||||
2823 | // a 'not' too, or a constant. | ||||
2824 | auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) { | ||||
2825 | if (!match(V0, m_Not(m_Value()))) | ||||
2826 | std::swap(V0, V1); | ||||
2827 | auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant()); | ||||
2828 | return match(V0, m_Not(m_Value())) && match(V1, Invertible); | ||||
2829 | }; | ||||
2830 | |||||
2831 | // Don't fold i1 branches on PHIs which contain binary operators or | ||||
2832 | // (possibly inverted) select form of or/ands, unless one of | ||||
2833 | // the incoming values is an 'not' and another one is freely invertible. | ||||
2834 | // These can often be turned into switches and other things. | ||||
2835 | auto IsBinOpOrAnd = [](Value *V) { | ||||
2836 | return match( | ||||
2837 | V, m_CombineOr( | ||||
2838 | m_BinOp(), | ||||
2839 | m_CombineOr(m_Select(m_Value(), m_ImmConstant(), m_Value()), | ||||
2840 | m_Select(m_Value(), m_Value(), m_ImmConstant())))); | ||||
2841 | }; | ||||
2842 | if (PN->getType()->isIntegerTy(1) && | ||||
2843 | (IsBinOpOrAnd(PN->getIncomingValue(0)) || | ||||
2844 | IsBinOpOrAnd(PN->getIncomingValue(1)) || IsBinOpOrAnd(IfCond)) && | ||||
2845 | !CanHoistNotFromBothValues(PN->getIncomingValue(0), | ||||
2846 | PN->getIncomingValue(1))) | ||||
2847 | return Changed; | ||||
2848 | |||||
2849 | // If all PHI nodes are promotable, check to make sure that all instructions | ||||
2850 | // in the predecessor blocks can be promoted as well. If not, we won't be able | ||||
2851 | // to get rid of the control flow, so it's not worth promoting to select | ||||
2852 | // instructions. | ||||
2853 | for (BasicBlock *IfBlock : IfBlocks) | ||||
2854 | for (BasicBlock::iterator I = IfBlock->begin(); !I->isTerminator(); ++I) | ||||
2855 | if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) && | ||||
2856 | !isa<PseudoProbeInst>(I)) { | ||||
2857 | // This is not an aggressive instruction that we can promote. | ||||
2858 | // Because of this, we won't be able to get rid of the control flow, so | ||||
2859 | // the xform is not worth it. | ||||
2860 | return Changed; | ||||
2861 | } | ||||
2862 | |||||
2863 | // If either of the blocks has it's address taken, we can't do this fold. | ||||
2864 | if (any_of(IfBlocks, | ||||
2865 | [](BasicBlock *IfBlock) { return IfBlock->hasAddressTaken(); })) | ||||
2866 | return Changed; | ||||
2867 | |||||
2868 | LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfConddo { } while (false) | ||||
2869 | << " T: " << IfTrue->getName()do { } while (false) | ||||
2870 | << " F: " << IfFalse->getName() << "\n")do { } while (false); | ||||
2871 | |||||
2872 | // If we can still promote the PHI nodes after this gauntlet of tests, | ||||
2873 | // do all of the PHI's now. | ||||
2874 | |||||
2875 | // Move all 'aggressive' instructions, which are defined in the | ||||
2876 | // conditional parts of the if's up to the dominating block. | ||||
2877 | for (BasicBlock *IfBlock : IfBlocks) | ||||
2878 | hoistAllInstructionsInto(DomBlock, DomBI, IfBlock); | ||||
2879 | |||||
2880 | IRBuilder<NoFolder> Builder(DomBI); | ||||
2881 | // Propagate fast-math-flags from phi nodes to replacement selects. | ||||
2882 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | ||||
2883 | while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) { | ||||
2884 | if (isa<FPMathOperator>(PN)) | ||||
2885 | Builder.setFastMathFlags(PN->getFastMathFlags()); | ||||
2886 | |||||
2887 | // Change the PHI node into a select instruction. | ||||
2888 | Value *TrueVal = PN->getIncomingValueForBlock(IfTrue); | ||||
2889 | Value *FalseVal = PN->getIncomingValueForBlock(IfFalse); | ||||
2890 | |||||
2891 | Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", DomBI); | ||||
2892 | PN->replaceAllUsesWith(Sel); | ||||
2893 | Sel->takeName(PN); | ||||
2894 | PN->eraseFromParent(); | ||||
2895 | } | ||||
2896 | |||||
2897 | // At this point, all IfBlocks are empty, so our if statement | ||||
2898 | // has been flattened. Change DomBlock to jump directly to our new block to | ||||
2899 | // avoid other simplifycfg's kicking in on the diamond. | ||||
2900 | Builder.CreateBr(BB); | ||||
2901 | |||||
2902 | SmallVector<DominatorTree::UpdateType, 3> Updates; | ||||
2903 | if (DTU) { | ||||
2904 | Updates.push_back({DominatorTree::Insert, DomBlock, BB}); | ||||
2905 | for (auto *Successor : successors(DomBlock)) | ||||
2906 | Updates.push_back({DominatorTree::Delete, DomBlock, Successor}); | ||||
2907 | } | ||||
2908 | |||||
2909 | DomBI->eraseFromParent(); | ||||
2910 | if (DTU) | ||||
2911 | DTU->applyUpdates(Updates); | ||||
2912 | |||||
2913 | return true; | ||||
2914 | } | ||||
2915 | |||||
2916 | static Value *createLogicalOp(IRBuilderBase &Builder, | ||||
2917 | Instruction::BinaryOps Opc, Value *LHS, | ||||
2918 | Value *RHS, const Twine &Name = "") { | ||||
2919 | // Try to relax logical op to binary op. | ||||
2920 | if (impliesPoison(RHS, LHS)) | ||||
2921 | return Builder.CreateBinOp(Opc, LHS, RHS, Name); | ||||
2922 | if (Opc == Instruction::And) | ||||
2923 | return Builder.CreateLogicalAnd(LHS, RHS, Name); | ||||
2924 | if (Opc == Instruction::Or) | ||||
2925 | return Builder.CreateLogicalOr(LHS, RHS, Name); | ||||
2926 | llvm_unreachable("Invalid logical opcode")__builtin_unreachable(); | ||||
2927 | } | ||||
2928 | |||||
2929 | /// Return true if either PBI or BI has branch weight available, and store | ||||
2930 | /// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does | ||||
2931 | /// not have branch weight, use 1:1 as its weight. | ||||
2932 | static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI, | ||||
2933 | uint64_t &PredTrueWeight, | ||||
2934 | uint64_t &PredFalseWeight, | ||||
2935 | uint64_t &SuccTrueWeight, | ||||
2936 | uint64_t &SuccFalseWeight) { | ||||
2937 | bool PredHasWeights = | ||||
2938 | PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight); | ||||
2939 | bool SuccHasWeights = | ||||
2940 | BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight); | ||||
2941 | if (PredHasWeights || SuccHasWeights) { | ||||
2942 | if (!PredHasWeights) | ||||
2943 | PredTrueWeight = PredFalseWeight = 1; | ||||
2944 | if (!SuccHasWeights) | ||||
2945 | SuccTrueWeight = SuccFalseWeight = 1; | ||||
2946 | return true; | ||||
2947 | } else { | ||||
2948 | return false; | ||||
2949 | } | ||||
2950 | } | ||||
2951 | |||||
2952 | /// Determine if the two branches share a common destination and deduce a glue | ||||
2953 | /// that joins the branches' conditions to arrive at the common destination if | ||||
2954 | /// that would be profitable. | ||||
2955 | static Optional<std::pair<Instruction::BinaryOps, bool>> | ||||
2956 | shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI, | ||||
2957 | const TargetTransformInfo *TTI) { | ||||
2958 | assert(BI && PBI && BI->isConditional() && PBI->isConditional() &&((void)0) | ||||
2959 | "Both blocks must end with a conditional branches.")((void)0); | ||||
2960 | assert(is_contained(predecessors(BI->getParent()), PBI->getParent()) &&((void)0) | ||||
2961 | "PredBB must be a predecessor of BB.")((void)0); | ||||
2962 | |||||
2963 | // We have the potential to fold the conditions together, but if the | ||||
2964 | // predecessor branch is predictable, we may not want to merge them. | ||||
2965 | uint64_t PTWeight, PFWeight; | ||||
2966 | BranchProbability PBITrueProb, Likely; | ||||
2967 | if (TTI && !PBI->getMetadata(LLVMContext::MD_unpredictable) && | ||||
2968 | PBI->extractProfMetadata(PTWeight, PFWeight) && | ||||
2969 | (PTWeight + PFWeight) != 0) { | ||||
2970 | PBITrueProb = | ||||
2971 | BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight); | ||||
2972 | Likely = TTI->getPredictableBranchThreshold(); | ||||
2973 | } | ||||
2974 | |||||
2975 | if (PBI->getSuccessor(0) == BI->getSuccessor(0)) { | ||||
2976 | // Speculate the 2nd condition unless the 1st is probably true. | ||||
2977 | if (PBITrueProb.isUnknown() || PBITrueProb < Likely) | ||||
2978 | return {{Instruction::Or, false}}; | ||||
2979 | } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) { | ||||
2980 | // Speculate the 2nd condition unless the 1st is probably false. | ||||
2981 | if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely) | ||||
2982 | return {{Instruction::And, false}}; | ||||
2983 | } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) { | ||||
2984 | // Speculate the 2nd condition unless the 1st is probably true. | ||||
2985 | if (PBITrueProb.isUnknown() || PBITrueProb < Likely) | ||||
2986 | return {{Instruction::And, true}}; | ||||
2987 | } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) { | ||||
2988 | // Speculate the 2nd condition unless the 1st is probably false. | ||||
2989 | if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely) | ||||
2990 | return {{Instruction::Or, true}}; | ||||
2991 | } | ||||
2992 | return None; | ||||
2993 | } | ||||
2994 | |||||
2995 | static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI, | ||||
2996 | DomTreeUpdater *DTU, | ||||
2997 | MemorySSAUpdater *MSSAU, | ||||
2998 | const TargetTransformInfo *TTI) { | ||||
2999 | BasicBlock *BB = BI->getParent(); | ||||
3000 | BasicBlock *PredBlock = PBI->getParent(); | ||||
3001 | |||||
3002 | // Determine if the two branches share a common destination. | ||||
3003 | Instruction::BinaryOps Opc; | ||||
3004 | bool InvertPredCond; | ||||
3005 | std::tie(Opc, InvertPredCond) = | ||||
3006 | *shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI); | ||||
3007 | |||||
3008 | LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB)do { } while (false); | ||||
3009 | |||||
3010 | IRBuilder<> Builder(PBI); | ||||
3011 | // The builder is used to create instructions to eliminate the branch in BB. | ||||
3012 | // If BB's terminator has !annotation metadata, add it to the new | ||||
3013 | // instructions. | ||||
3014 | Builder.CollectMetadataToCopy(BB->getTerminator(), | ||||
3015 | {LLVMContext::MD_annotation}); | ||||
3016 | |||||
3017 | // If we need to invert the condition in the pred block to match, do so now. | ||||
3018 | if (InvertPredCond) { | ||||
3019 | Value *NewCond = PBI->getCondition(); | ||||
3020 | if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) { | ||||
3021 | CmpInst *CI = cast<CmpInst>(NewCond); | ||||
3022 | CI->setPredicate(CI->getInversePredicate()); | ||||
3023 | } else { | ||||
3024 | NewCond = | ||||
3025 | Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not"); | ||||
3026 | } | ||||
3027 | |||||
3028 | PBI->setCondition(NewCond); | ||||
3029 | PBI->swapSuccessors(); | ||||
3030 | } | ||||
3031 | |||||
3032 | BasicBlock *UniqueSucc = | ||||
3033 | PBI->getSuccessor(0) == BB ? BI->getSuccessor(0) : BI->getSuccessor(1); | ||||
3034 | |||||
3035 | // Before cloning instructions, notify the successor basic block that it | ||||
3036 | // is about to have a new predecessor. This will update PHI nodes, | ||||
3037 | // which will allow us to update live-out uses of bonus instructions. | ||||
3038 | AddPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU); | ||||
3039 | |||||
3040 | // Try to update branch weights. | ||||
3041 | uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight; | ||||
3042 | if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight, | ||||
3043 | SuccTrueWeight, SuccFalseWeight)) { | ||||
3044 | SmallVector<uint64_t, 8> NewWeights; | ||||
3045 | |||||
3046 | if (PBI->getSuccessor(0) == BB) { | ||||
3047 | // PBI: br i1 %x, BB, FalseDest | ||||
3048 | // BI: br i1 %y, UniqueSucc, FalseDest | ||||
3049 | // TrueWeight is TrueWeight for PBI * TrueWeight for BI. | ||||
3050 | NewWeights.push_back(PredTrueWeight * SuccTrueWeight); | ||||
3051 | // FalseWeight is FalseWeight for PBI * TotalWeight for BI + | ||||
3052 | // TrueWeight for PBI * FalseWeight for BI. | ||||
3053 | // We assume that total weights of a BranchInst can fit into 32 bits. | ||||
3054 | // Therefore, we will not have overflow using 64-bit arithmetic. | ||||
3055 | NewWeights.push_back(PredFalseWeight * | ||||
3056 | (SuccFalseWeight + SuccTrueWeight) + | ||||
3057 | PredTrueWeight * SuccFalseWeight); | ||||
3058 | } else { | ||||
3059 | // PBI: br i1 %x, TrueDest, BB | ||||
3060 | // BI: br i1 %y, TrueDest, UniqueSucc | ||||
3061 | // TrueWeight is TrueWeight for PBI * TotalWeight for BI + | ||||
3062 | // FalseWeight for PBI * TrueWeight for BI. | ||||
3063 | NewWeights.push_back(PredTrueWeight * (SuccFalseWeight + SuccTrueWeight) + | ||||
3064 | PredFalseWeight * SuccTrueWeight); | ||||
3065 | // FalseWeight is FalseWeight for PBI * FalseWeight for BI. | ||||
3066 | NewWeights.push_back(PredFalseWeight * SuccFalseWeight); | ||||
3067 | } | ||||
3068 | |||||
3069 | // Halve the weights if any of them cannot fit in an uint32_t | ||||
3070 | FitWeights(NewWeights); | ||||
3071 | |||||
3072 | SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end()); | ||||
3073 | setBranchWeights(PBI, MDWeights[0], MDWeights[1]); | ||||
3074 | |||||
3075 | // TODO: If BB is reachable from all paths through PredBlock, then we | ||||
3076 | // could replace PBI's branch probabilities with BI's. | ||||
3077 | } else | ||||
3078 | PBI->setMetadata(LLVMContext::MD_prof, nullptr); | ||||
3079 | |||||
3080 | // Now, update the CFG. | ||||
3081 | PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc); | ||||
3082 | |||||
3083 | if (DTU) | ||||
3084 | DTU->applyUpdates({{DominatorTree::Insert, PredBlock, UniqueSucc}, | ||||
3085 | {DominatorTree::Delete, PredBlock, BB}}); | ||||
3086 | |||||
3087 | // If BI was a loop latch, it may have had associated loop metadata. | ||||
3088 | // We need to copy it to the new latch, that is, PBI. | ||||
3089 | if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop)) | ||||
3090 | PBI->setMetadata(LLVMContext::MD_loop, LoopMD); | ||||
3091 | |||||
3092 | ValueToValueMapTy VMap; // maps original values to cloned values | ||||
3093 | CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(BB, PredBlock, VMap); | ||||
3094 | |||||
3095 | // Now that the Cond was cloned into the predecessor basic block, | ||||
3096 | // or/and the two conditions together. | ||||
3097 | Value *BICond = VMap[BI->getCondition()]; | ||||
3098 | PBI->setCondition( | ||||
3099 | createLogicalOp(Builder, Opc, PBI->getCondition(), BICond, "or.cond")); | ||||
3100 | |||||
3101 | // Copy any debug value intrinsics into the end of PredBlock. | ||||
3102 | for (Instruction &I : *BB) { | ||||
3103 | if (isa<DbgInfoIntrinsic>(I)) { | ||||
3104 | Instruction *NewI = I.clone(); | ||||
3105 | RemapInstruction(NewI, VMap, | ||||
3106 | RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); | ||||
3107 | NewI->insertBefore(PBI); | ||||
3108 | } | ||||
3109 | } | ||||
3110 | |||||
3111 | ++NumFoldBranchToCommonDest; | ||||
3112 | return true; | ||||
3113 | } | ||||
3114 | |||||
3115 | /// If this basic block is simple enough, and if a predecessor branches to us | ||||
3116 | /// and one of our successors, fold the block into the predecessor and use | ||||
3117 | /// logical operations to pick the right destination. | ||||
3118 | bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU, | ||||
3119 | MemorySSAUpdater *MSSAU, | ||||
3120 | const TargetTransformInfo *TTI, | ||||
3121 | unsigned BonusInstThreshold) { | ||||
3122 | // If this block ends with an unconditional branch, | ||||
3123 | // let SpeculativelyExecuteBB() deal with it. | ||||
3124 | if (!BI->isConditional()) | ||||
3125 | return false; | ||||
3126 | |||||
3127 | BasicBlock *BB = BI->getParent(); | ||||
3128 | TargetTransformInfo::TargetCostKind CostKind = | ||||
3129 | BB->getParent()->hasMinSize() ? TargetTransformInfo::TCK_CodeSize | ||||
3130 | : TargetTransformInfo::TCK_SizeAndLatency; | ||||
3131 | |||||
3132 | Instruction *Cond = dyn_cast<Instruction>(BI->getCondition()); | ||||
3133 | |||||
3134 | if (!Cond || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) || | ||||
3135 | Cond->getParent() != BB || !Cond->hasOneUse()) | ||||
3136 | return false; | ||||
3137 | |||||
3138 | // Cond is known to be a compare or binary operator. Check to make sure that | ||||
3139 | // neither operand is a potentially-trapping constant expression. | ||||
3140 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0))) | ||||
3141 | if (CE->canTrap()) | ||||
3142 | return false; | ||||
3143 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1))) | ||||
3144 | if (CE->canTrap()) | ||||
3145 | return false; | ||||
3146 | |||||
3147 | // Finally, don't infinitely unroll conditional loops. | ||||
3148 | if (is_contained(successors(BB), BB)) | ||||
3149 | return false; | ||||
3150 | |||||
3151 | // With which predecessors will we want to deal with? | ||||
3152 | SmallVector<BasicBlock *, 8> Preds; | ||||
3153 | for (BasicBlock *PredBlock : predecessors(BB)) { | ||||
3154 | BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator()); | ||||
3155 | |||||
3156 | // Check that we have two conditional branches. If there is a PHI node in | ||||
3157 | // the common successor, verify that the same value flows in from both | ||||
3158 | // blocks. | ||||
3159 | if (!PBI || PBI->isUnconditional() || !SafeToMergeTerminators(BI, PBI)) | ||||
3160 | continue; | ||||
3161 | |||||
3162 | // Determine if the two branches share a common destination. | ||||
3163 | Instruction::BinaryOps Opc; | ||||
3164 | bool InvertPredCond; | ||||
3165 | if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI)) | ||||
3166 | std::tie(Opc, InvertPredCond) = *Recipe; | ||||
3167 | else | ||||
3168 | continue; | ||||
3169 | |||||
3170 | // Check the cost of inserting the necessary logic before performing the | ||||
3171 | // transformation. | ||||
3172 | if (TTI) { | ||||
3173 | Type *Ty = BI->getCondition()->getType(); | ||||
3174 | InstructionCost Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind); | ||||
3175 | if (InvertPredCond && (!PBI->getCondition()->hasOneUse() || | ||||
3176 | !isa<CmpInst>(PBI->getCondition()))) | ||||
3177 | Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind); | ||||
3178 | |||||
3179 | if (Cost > BranchFoldThreshold) | ||||
3180 | continue; | ||||
3181 | } | ||||
3182 | |||||
3183 | // Ok, we do want to deal with this predecessor. Record it. | ||||
3184 | Preds.emplace_back(PredBlock); | ||||
3185 | } | ||||
3186 | |||||
3187 | // If there aren't any predecessors into which we can fold, | ||||
3188 | // don't bother checking the cost. | ||||
3189 | if (Preds.empty()) | ||||
3190 | return false; | ||||
3191 | |||||
3192 | // Only allow this transformation if computing the condition doesn't involve | ||||
3193 | // too many instructions and these involved instructions can be executed | ||||
3194 | // unconditionally. We denote all involved instructions except the condition | ||||
3195 | // as "bonus instructions", and only allow this transformation when the | ||||
3196 | // number of the bonus instructions we'll need to create when cloning into | ||||
3197 | // each predecessor does not exceed a certain threshold. | ||||
3198 | unsigned NumBonusInsts = 0; | ||||
3199 | const unsigned PredCount = Preds.size(); | ||||
3200 | for (Instruction &I : *BB) { | ||||
3201 | // Don't check the branch condition comparison itself. | ||||
3202 | if (&I == Cond) | ||||
3203 | continue; | ||||
3204 | // Ignore dbg intrinsics, and the terminator. | ||||
3205 | if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I)) | ||||
3206 | continue; | ||||
3207 | // I must be safe to execute unconditionally. | ||||
3208 | if (!isSafeToSpeculativelyExecute(&I)) | ||||
3209 | return false; | ||||
3210 | |||||
3211 | // Account for the cost of duplicating this instruction into each | ||||
3212 | // predecessor. | ||||
3213 | NumBonusInsts += PredCount; | ||||
3214 | // Early exits once we reach the limit. | ||||
3215 | if (NumBonusInsts > BonusInstThreshold) | ||||
3216 | return false; | ||||
3217 | |||||
3218 | auto IsBCSSAUse = [BB, &I](Use &U) { | ||||
3219 | auto *UI = cast<Instruction>(U.getUser()); | ||||
3220 | if (auto *PN = dyn_cast<PHINode>(UI)) | ||||
3221 | return PN->getIncomingBlock(U) == BB; | ||||
3222 | return UI->getParent() == BB && I.comesBefore(UI); | ||||
3223 | }; | ||||
3224 | |||||
3225 | // Does this instruction require rewriting of uses? | ||||
3226 | if (!all_of(I.uses(), IsBCSSAUse)) | ||||
3227 | return false; | ||||
3228 | } | ||||
3229 | |||||
3230 | // Ok, we have the budget. Perform the transformation. | ||||
3231 | for (BasicBlock *PredBlock : Preds) { | ||||
3232 | auto *PBI = cast<BranchInst>(PredBlock->getTerminator()); | ||||
3233 | return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI); | ||||
3234 | } | ||||
3235 | return false; | ||||
3236 | } | ||||
3237 | |||||
3238 | // If there is only one store in BB1 and BB2, return it, otherwise return | ||||
3239 | // nullptr. | ||||
3240 | static StoreInst *findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2) { | ||||
3241 | StoreInst *S = nullptr; | ||||
3242 | for (auto *BB : {BB1, BB2}) { | ||||
3243 | if (!BB) | ||||
3244 | continue; | ||||
3245 | for (auto &I : *BB) | ||||
3246 | if (auto *SI = dyn_cast<StoreInst>(&I)) { | ||||
3247 | if (S) | ||||
3248 | // Multiple stores seen. | ||||
3249 | return nullptr; | ||||
3250 | else | ||||
3251 | S = SI; | ||||
3252 | } | ||||
3253 | } | ||||
3254 | return S; | ||||
3255 | } | ||||
3256 | |||||
3257 | static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB, | ||||
3258 | Value *AlternativeV = nullptr) { | ||||
3259 | // PHI is going to be a PHI node that allows the value V that is defined in | ||||
3260 | // BB to be referenced in BB's only successor. | ||||
3261 | // | ||||
3262 | // If AlternativeV is nullptr, the only value we care about in PHI is V. It | ||||
3263 | // doesn't matter to us what the other operand is (it'll never get used). We | ||||
3264 | // could just create a new PHI with an undef incoming value, but that could | ||||
3265 | // increase register pressure if EarlyCSE/InstCombine can't fold it with some | ||||
3266 | // other PHI. So here we directly look for some PHI in BB's successor with V | ||||
3267 | // as an incoming operand. If we find one, we use it, else we create a new | ||||
3268 | // one. | ||||
3269 | // | ||||
3270 | // If AlternativeV is not nullptr, we care about both incoming values in PHI. | ||||
3271 | // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV] | ||||
3272 | // where OtherBB is the single other predecessor of BB's only successor. | ||||
3273 | PHINode *PHI = nullptr; | ||||
3274 | BasicBlock *Succ = BB->getSingleSuccessor(); | ||||
3275 | |||||
3276 | for (auto I = Succ->begin(); isa<PHINode>(I); ++I) | ||||
3277 | if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) { | ||||
3278 | PHI = cast<PHINode>(I); | ||||
3279 | if (!AlternativeV) | ||||
3280 | break; | ||||
3281 | |||||
3282 | assert(Succ->hasNPredecessors(2))((void)0); | ||||
3283 | auto PredI = pred_begin(Succ); | ||||
3284 | BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI; | ||||
3285 | if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV) | ||||
3286 | break; | ||||
3287 | PHI = nullptr; | ||||
3288 | } | ||||
3289 | if (PHI) | ||||
3290 | return PHI; | ||||
3291 | |||||
3292 | // If V is not an instruction defined in BB, just return it. | ||||
3293 | if (!AlternativeV && | ||||
3294 | (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB)) | ||||
3295 | return V; | ||||
3296 | |||||
3297 | PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front()); | ||||
3298 | PHI->addIncoming(V, BB); | ||||
3299 | for (BasicBlock *PredBB : predecessors(Succ)) | ||||
3300 | if (PredBB != BB) | ||||
3301 | PHI->addIncoming( | ||||
3302 | AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB); | ||||
3303 | return PHI; | ||||
3304 | } | ||||
3305 | |||||
3306 | static bool mergeConditionalStoreToAddress( | ||||
3307 | BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB, | ||||
3308 | BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond, | ||||
3309 | DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) { | ||||
3310 | // For every pointer, there must be exactly two stores, one coming from | ||||
3311 | // PTB or PFB, and the other from QTB or QFB. We don't support more than one | ||||
3312 | // store (to any address) in PTB,PFB or QTB,QFB. | ||||
3313 | // FIXME: We could relax this restriction with a bit more work and performance | ||||
3314 | // testing. | ||||
3315 | StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB); | ||||
3316 | StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB); | ||||
3317 | if (!PStore || !QStore) | ||||
3318 | return false; | ||||
3319 | |||||
3320 | // Now check the stores are compatible. | ||||
3321 | if (!QStore->isUnordered() || !PStore->isUnordered()) | ||||
3322 | return false; | ||||
3323 | |||||
3324 | // Check that sinking the store won't cause program behavior changes. Sinking | ||||
3325 | // the store out of the Q blocks won't change any behavior as we're sinking | ||||
3326 | // from a block to its unconditional successor. But we're moving a store from | ||||
3327 | // the P blocks down through the middle block (QBI) and past both QFB and QTB. | ||||
3328 | // So we need to check that there are no aliasing loads or stores in | ||||
3329 | // QBI, QTB and QFB. We also need to check there are no conflicting memory | ||||
3330 | // operations between PStore and the end of its parent block. | ||||
3331 | // | ||||
3332 | // The ideal way to do this is to query AliasAnalysis, but we don't | ||||
3333 | // preserve AA currently so that is dangerous. Be super safe and just | ||||
3334 | // check there are no other memory operations at all. | ||||
3335 | for (auto &I : *QFB->getSinglePredecessor()) | ||||
3336 | if (I.mayReadOrWriteMemory()) | ||||
3337 | return false; | ||||
3338 | for (auto &I : *QFB) | ||||
3339 | if (&I != QStore && I.mayReadOrWriteMemory()) | ||||
3340 | return false; | ||||
3341 | if (QTB) | ||||
3342 | for (auto &I : *QTB) | ||||
3343 | if (&I != QStore && I.mayReadOrWriteMemory()) | ||||
3344 | return false; | ||||
3345 | for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end(); | ||||
3346 | I != E; ++I) | ||||
3347 | if (&*I != PStore && I->mayReadOrWriteMemory()) | ||||
3348 | return false; | ||||
3349 | |||||
3350 | // If we're not in aggressive mode, we only optimize if we have some | ||||
3351 | // confidence that by optimizing we'll allow P and/or Q to be if-converted. | ||||
3352 | auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) { | ||||
3353 | if (!BB) | ||||
3354 | return true; | ||||
3355 | // Heuristic: if the block can be if-converted/phi-folded and the | ||||
3356 | // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to | ||||
3357 | // thread this store. | ||||
3358 | InstructionCost Cost = 0; | ||||
3359 | InstructionCost Budget = | ||||
3360 | PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; | ||||
3361 | for (auto &I : BB->instructionsWithoutDebug()) { | ||||
3362 | // Consider terminator instruction to be free. | ||||
3363 | if (I.isTerminator()) | ||||
3364 | continue; | ||||
3365 | // If this is one the stores that we want to speculate out of this BB, | ||||
3366 | // then don't count it's cost, consider it to be free. | ||||
3367 | if (auto *S = dyn_cast<StoreInst>(&I)) | ||||
3368 | if (llvm::find(FreeStores, S)) | ||||
3369 | continue; | ||||
3370 | // Else, we have a white-list of instructions that we are ak speculating. | ||||
3371 | if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I)) | ||||
3372 | return false; // Not in white-list - not worthwhile folding. | ||||
3373 | // And finally, if this is a non-free instruction that we are okay | ||||
3374 | // speculating, ensure that we consider the speculation budget. | ||||
3375 | Cost += TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency); | ||||
3376 | if (Cost > Budget) | ||||
3377 | return false; // Eagerly refuse to fold as soon as we're out of budget. | ||||
3378 | } | ||||
3379 | assert(Cost <= Budget &&((void)0) | ||||
3380 | "When we run out of budget we will eagerly return from within the "((void)0) | ||||
3381 | "per-instruction loop.")((void)0); | ||||
3382 | return true; | ||||
3383 | }; | ||||
3384 | |||||
3385 | const std::array<StoreInst *, 2> FreeStores = {PStore, QStore}; | ||||
3386 | if (!MergeCondStoresAggressively && | ||||
3387 | (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) || | ||||
3388 | !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores))) | ||||
3389 | return false; | ||||
3390 | |||||
3391 | // If PostBB has more than two predecessors, we need to split it so we can | ||||
3392 | // sink the store. | ||||
3393 | if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) { | ||||
3394 | // We know that QFB's only successor is PostBB. And QFB has a single | ||||
3395 | // predecessor. If QTB exists, then its only successor is also PostBB. | ||||
3396 | // If QTB does not exist, then QFB's only predecessor has a conditional | ||||
3397 | // branch to QFB and PostBB. | ||||
3398 | BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor(); | ||||
3399 | BasicBlock *NewBB = | ||||
3400 | SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU); | ||||
3401 | if (!NewBB) | ||||
3402 | return false; | ||||
3403 | PostBB = NewBB; | ||||
3404 | } | ||||
3405 | |||||
3406 | // OK, we're going to sink the stores to PostBB. The store has to be | ||||
3407 | // conditional though, so first create the predicate. | ||||
3408 | Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator()) | ||||
3409 | ->getCondition(); | ||||
3410 | Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator()) | ||||
3411 | ->getCondition(); | ||||
3412 | |||||
3413 | Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(), | ||||
3414 | PStore->getParent()); | ||||
3415 | Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(), | ||||
3416 | QStore->getParent(), PPHI); | ||||
3417 | |||||
3418 | IRBuilder<> QB(&*PostBB->getFirstInsertionPt()); | ||||
3419 | |||||
3420 | Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond); | ||||
3421 | Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond); | ||||
3422 | |||||
3423 | if (InvertPCond) | ||||
3424 | PPred = QB.CreateNot(PPred); | ||||
3425 | if (InvertQCond) | ||||
3426 | QPred = QB.CreateNot(QPred); | ||||
3427 | Value *CombinedPred = QB.CreateOr(PPred, QPred); | ||||
3428 | |||||
3429 | auto *T = SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(), | ||||
3430 | /*Unreachable=*/false, | ||||
3431 | /*BranchWeights=*/nullptr, DTU); | ||||
3432 | QB.SetInsertPoint(T); | ||||
3433 | StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address)); | ||||
3434 | AAMDNodes AAMD; | ||||
3435 | PStore->getAAMetadata(AAMD, /*Merge=*/false); | ||||
3436 | PStore->getAAMetadata(AAMD, /*Merge=*/true); | ||||
3437 | SI->setAAMetadata(AAMD); | ||||
3438 | // Choose the minimum alignment. If we could prove both stores execute, we | ||||
3439 | // could use biggest one. In this case, though, we only know that one of the | ||||
3440 | // stores executes. And we don't know it's safe to take the alignment from a | ||||
3441 | // store that doesn't execute. | ||||
3442 | SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign())); | ||||
3443 | |||||
3444 | QStore->eraseFromParent(); | ||||
3445 | PStore->eraseFromParent(); | ||||
3446 | |||||
3447 | return true; | ||||
3448 | } | ||||
3449 | |||||
3450 | static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI, | ||||
3451 | DomTreeUpdater *DTU, const DataLayout &DL, | ||||
3452 | const TargetTransformInfo &TTI) { | ||||
3453 | // The intention here is to find diamonds or triangles (see below) where each | ||||
3454 | // conditional block contains a store to the same address. Both of these | ||||
3455 | // stores are conditional, so they can't be unconditionally sunk. But it may | ||||
3456 | // be profitable to speculatively sink the stores into one merged store at the | ||||
3457 | // end, and predicate the merged store on the union of the two conditions of | ||||
3458 | // PBI and QBI. | ||||
3459 | // | ||||
3460 | // This can reduce the number of stores executed if both of the conditions are | ||||
3461 | // true, and can allow the blocks to become small enough to be if-converted. | ||||
3462 | // This optimization will also chain, so that ladders of test-and-set | ||||
3463 | // sequences can be if-converted away. | ||||
3464 | // | ||||
3465 | // We only deal with simple diamonds or triangles: | ||||
3466 | // | ||||
3467 | // PBI or PBI or a combination of the two | ||||
3468 | // / \ | \ | ||||
3469 | // PTB PFB | PFB | ||||
3470 | // \ / | / | ||||
3471 | // QBI QBI | ||||
3472 | // / \ | \ | ||||
3473 | // QTB QFB | QFB | ||||
3474 | // \ / | / | ||||
3475 | // PostBB PostBB | ||||
3476 | // | ||||
3477 | // We model triangles as a type of diamond with a nullptr "true" block. | ||||
3478 | // Triangles are canonicalized so that the fallthrough edge is represented by | ||||
3479 | // a true condition, as in the diagram above. | ||||
3480 | BasicBlock *PTB = PBI->getSuccessor(0); | ||||
3481 | BasicBlock *PFB = PBI->getSuccessor(1); | ||||
3482 | BasicBlock *QTB = QBI->getSuccessor(0); | ||||
3483 | BasicBlock *QFB = QBI->getSuccessor(1); | ||||
3484 | BasicBlock *PostBB = QFB->getSingleSuccessor(); | ||||
3485 | |||||
3486 | // Make sure we have a good guess for PostBB. If QTB's only successor is | ||||
3487 | // QFB, then QFB is a better PostBB. | ||||
3488 | if (QTB->getSingleSuccessor() == QFB) | ||||
3489 | PostBB = QFB; | ||||
3490 | |||||
3491 | // If we couldn't find a good PostBB, stop. | ||||
3492 | if (!PostBB) | ||||
3493 | return false; | ||||
3494 | |||||
3495 | bool InvertPCond = false, InvertQCond = false; | ||||
3496 | // Canonicalize fallthroughs to the true branches. | ||||
3497 | if (PFB == QBI->getParent()) { | ||||
3498 | std::swap(PFB, PTB); | ||||
3499 | InvertPCond = true; | ||||
3500 | } | ||||
3501 | if (QFB == PostBB) { | ||||
3502 | std::swap(QFB, QTB); | ||||
3503 | InvertQCond = true; | ||||
3504 | } | ||||
3505 | |||||
3506 | // From this point on we can assume PTB or QTB may be fallthroughs but PFB | ||||
3507 | // and QFB may not. Model fallthroughs as a nullptr block. | ||||
3508 | if (PTB == QBI->getParent()) | ||||
3509 | PTB = nullptr; | ||||
3510 | if (QTB == PostBB) | ||||
3511 | QTB = nullptr; | ||||
3512 | |||||
3513 | // Legality bailouts. We must have at least the non-fallthrough blocks and | ||||
3514 | // the post-dominating block, and the non-fallthroughs must only have one | ||||
3515 | // predecessor. | ||||
3516 | auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) { | ||||
3517 | return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S; | ||||
3518 | }; | ||||
3519 | if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) || | ||||
3520 | !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB)) | ||||
3521 | return false; | ||||
3522 | if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) || | ||||
3523 | (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB))) | ||||
3524 | return false; | ||||
3525 | if (!QBI->getParent()->hasNUses(2)) | ||||
3526 | return false; | ||||
3527 | |||||
3528 | // OK, this is a sequence of two diamonds or triangles. | ||||
3529 | // Check if there are stores in PTB or PFB that are repeated in QTB or QFB. | ||||
3530 | SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses; | ||||
3531 | for (auto *BB : {PTB, PFB}) { | ||||
3532 | if (!BB) | ||||
3533 | continue; | ||||
3534 | for (auto &I : *BB) | ||||
3535 | if (StoreInst *SI = dyn_cast<StoreInst>(&I)) | ||||
3536 | PStoreAddresses.insert(SI->getPointerOperand()); | ||||
3537 | } | ||||
3538 | for (auto *BB : {QTB, QFB}) { | ||||
3539 | if (!BB) | ||||
3540 | continue; | ||||
3541 | for (auto &I : *BB) | ||||
3542 | if (StoreInst *SI = dyn_cast<StoreInst>(&I)) | ||||
3543 | QStoreAddresses.insert(SI->getPointerOperand()); | ||||
3544 | } | ||||
3545 | |||||
3546 | set_intersect(PStoreAddresses, QStoreAddresses); | ||||
3547 | // set_intersect mutates PStoreAddresses in place. Rename it here to make it | ||||
3548 | // clear what it contains. | ||||
3549 | auto &CommonAddresses = PStoreAddresses; | ||||
3550 | |||||
3551 | bool Changed = false; | ||||
3552 | for (auto *Address : CommonAddresses) | ||||
3553 | Changed |= | ||||
3554 | mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address, | ||||
3555 | InvertPCond, InvertQCond, DTU, DL, TTI); | ||||
3556 | return Changed; | ||||
3557 | } | ||||
3558 | |||||
3559 | /// If the previous block ended with a widenable branch, determine if reusing | ||||
3560 | /// the target block is profitable and legal. This will have the effect of | ||||
3561 | /// "widening" PBI, but doesn't require us to reason about hosting safety. | ||||
3562 | static bool tryWidenCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, | ||||
3563 | DomTreeUpdater *DTU) { | ||||
3564 | // TODO: This can be generalized in two important ways: | ||||
3565 | // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input | ||||
3566 | // values from the PBI edge. | ||||
3567 | // 2) We can sink side effecting instructions into BI's fallthrough | ||||
3568 | // successor provided they doesn't contribute to computation of | ||||
3569 | // BI's condition. | ||||
3570 | Value *CondWB, *WC; | ||||
3571 | BasicBlock *IfTrueBB, *IfFalseBB; | ||||
3572 | if (!parseWidenableBranch(PBI, CondWB, WC, IfTrueBB, IfFalseBB) || | ||||
3573 | IfTrueBB != BI->getParent() || !BI->getParent()->getSinglePredecessor()) | ||||
3574 | return false; | ||||
3575 | if (!IfFalseBB->phis().empty()) | ||||
3576 | return false; // TODO | ||||
3577 | // Use lambda to lazily compute expensive condition after cheap ones. | ||||
3578 | auto NoSideEffects = [](BasicBlock &BB) { | ||||
3579 | return !llvm::any_of(BB, [](const Instruction &I) { | ||||
3580 | return I.mayWriteToMemory() || I.mayHaveSideEffects(); | ||||
3581 | }); | ||||
3582 | }; | ||||
3583 | if (BI->getSuccessor(1) != IfFalseBB && // no inf looping | ||||
3584 | BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability | ||||
3585 | NoSideEffects(*BI->getParent())) { | ||||
3586 | auto *OldSuccessor = BI->getSuccessor(1); | ||||
3587 | OldSuccessor->removePredecessor(BI->getParent()); | ||||
3588 | BI->setSuccessor(1, IfFalseBB); | ||||
3589 | if (DTU) | ||||
3590 | DTU->applyUpdates( | ||||
3591 | {{DominatorTree::Insert, BI->getParent(), IfFalseBB}, | ||||
3592 | {DominatorTree::Delete, BI->getParent(), OldSuccessor}}); | ||||
3593 | return true; | ||||
3594 | } | ||||
3595 | if (BI->getSuccessor(0) != IfFalseBB && // no inf looping | ||||
3596 | BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability | ||||
3597 | NoSideEffects(*BI->getParent())) { | ||||
3598 | auto *OldSuccessor = BI->getSuccessor(0); | ||||
3599 | OldSuccessor->removePredecessor(BI->getParent()); | ||||
3600 | BI->setSuccessor(0, IfFalseBB); | ||||
3601 | if (DTU) | ||||
3602 | DTU->applyUpdates( | ||||
3603 | {{DominatorTree::Insert, BI->getParent(), IfFalseBB}, | ||||
3604 | {DominatorTree::Delete, BI->getParent(), OldSuccessor}}); | ||||
3605 | return true; | ||||
3606 | } | ||||
3607 | return false; | ||||
3608 | } | ||||
3609 | |||||
3610 | /// If we have a conditional branch as a predecessor of another block, | ||||
3611 | /// this function tries to simplify it. We know | ||||
3612 | /// that PBI and BI are both conditional branches, and BI is in one of the | ||||
3613 | /// successor blocks of PBI - PBI branches to BI. | ||||
3614 | static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, | ||||
3615 | DomTreeUpdater *DTU, | ||||
3616 | const DataLayout &DL, | ||||
3617 | const TargetTransformInfo &TTI) { | ||||
3618 | assert(PBI->isConditional() && BI->isConditional())((void)0); | ||||
3619 | BasicBlock *BB = BI->getParent(); | ||||
3620 | |||||
3621 | // If this block ends with a branch instruction, and if there is a | ||||
3622 | // predecessor that ends on a branch of the same condition, make | ||||
3623 | // this conditional branch redundant. | ||||
3624 | if (PBI->getCondition() == BI->getCondition() && | ||||
3625 | PBI->getSuccessor(0) != PBI->getSuccessor(1)) { | ||||
3626 | // Okay, the outcome of this conditional branch is statically | ||||
3627 | // knowable. If this block had a single pred, handle specially. | ||||
3628 | if (BB->getSinglePredecessor()) { | ||||
3629 | // Turn this into a branch on constant. | ||||
3630 | bool CondIsTrue = PBI->getSuccessor(0) == BB; | ||||
3631 | BI->setCondition( | ||||
3632 | ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue)); | ||||
3633 | return true; // Nuke the branch on constant. | ||||
3634 | } | ||||
3635 | |||||
3636 | // Otherwise, if there are multiple predecessors, insert a PHI that merges | ||||
3637 | // in the constant and simplify the block result. Subsequent passes of | ||||
3638 | // simplifycfg will thread the block. | ||||
3639 | if (BlockIsSimpleEnoughToThreadThrough(BB)) { | ||||
3640 | pred_iterator PB = pred_begin(BB), PE = pred_end(BB); | ||||
3641 | PHINode *NewPN = PHINode::Create( | ||||
3642 | Type::getInt1Ty(BB->getContext()), std::distance(PB, PE), | ||||
3643 | BI->getCondition()->getName() + ".pr", &BB->front()); | ||||
3644 | // Okay, we're going to insert the PHI node. Since PBI is not the only | ||||
3645 | // predecessor, compute the PHI'd conditional value for all of the preds. | ||||
3646 | // Any predecessor where the condition is not computable we keep symbolic. | ||||
3647 | for (pred_iterator PI = PB; PI != PE; ++PI) { | ||||
3648 | BasicBlock *P = *PI; | ||||
3649 | if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI && | ||||
3650 | PBI->isConditional() && PBI->getCondition() == BI->getCondition() && | ||||
3651 | PBI->getSuccessor(0) != PBI->getSuccessor(1)) { | ||||
3652 | bool CondIsTrue = PBI->getSuccessor(0) == BB; | ||||
3653 | NewPN->addIncoming( | ||||
3654 | ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue), | ||||
3655 | P); | ||||
3656 | } else { | ||||
3657 | NewPN->addIncoming(BI->getCondition(), P); | ||||
3658 | } | ||||
3659 | } | ||||
3660 | |||||
3661 | BI->setCondition(NewPN); | ||||
3662 | return true; | ||||
3663 | } | ||||
3664 | } | ||||
3665 | |||||
3666 | // If the previous block ended with a widenable branch, determine if reusing | ||||
3667 | // the target block is profitable and legal. This will have the effect of | ||||
3668 | // "widening" PBI, but doesn't require us to reason about hosting safety. | ||||
3669 | if (tryWidenCondBranchToCondBranch(PBI, BI, DTU)) | ||||
3670 | return true; | ||||
3671 | |||||
3672 | if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition())) | ||||
3673 | if (CE->canTrap()) | ||||
3674 | return false; | ||||
3675 | |||||
3676 | // If both branches are conditional and both contain stores to the same | ||||
3677 | // address, remove the stores from the conditionals and create a conditional | ||||
3678 | // merged store at the end. | ||||
3679 | if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI)) | ||||
3680 | return true; | ||||
3681 | |||||
3682 | // If this is a conditional branch in an empty block, and if any | ||||
3683 | // predecessors are a conditional branch to one of our destinations, | ||||
3684 | // fold the conditions into logical ops and one cond br. | ||||
3685 | |||||
3686 | // Ignore dbg intrinsics. | ||||
3687 | if (&*BB->instructionsWithoutDebug().begin() != BI) | ||||
3688 | return false; | ||||
3689 | |||||
3690 | int PBIOp, BIOp; | ||||
3691 | if (PBI->getSuccessor(0) == BI->getSuccessor(0)) { | ||||
3692 | PBIOp = 0; | ||||
3693 | BIOp = 0; | ||||
3694 | } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) { | ||||
3695 | PBIOp = 0; | ||||
3696 | BIOp = 1; | ||||
3697 | } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) { | ||||
3698 | PBIOp = 1; | ||||
3699 | BIOp = 0; | ||||
3700 | } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) { | ||||
3701 | PBIOp = 1; | ||||
3702 | BIOp = 1; | ||||
3703 | } else { | ||||
3704 | return false; | ||||
3705 | } | ||||
3706 | |||||
3707 | // Check to make sure that the other destination of this branch | ||||
3708 | // isn't BB itself. If so, this is an infinite loop that will | ||||
3709 | // keep getting unwound. | ||||
3710 | if (PBI->getSuccessor(PBIOp) == BB) | ||||
3711 | return false; | ||||
3712 | |||||
3713 | // Do not perform this transformation if it would require | ||||
3714 | // insertion of a large number of select instructions. For targets | ||||
3715 | // without predication/cmovs, this is a big pessimization. | ||||
3716 | |||||
3717 | // Also do not perform this transformation if any phi node in the common | ||||
3718 | // destination block can trap when reached by BB or PBB (PR17073). In that | ||||
3719 | // case, it would be unsafe to hoist the operation into a select instruction. | ||||
3720 | |||||
3721 | BasicBlock *CommonDest = PBI->getSuccessor(PBIOp); | ||||
3722 | BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1); | ||||
3723 | unsigned NumPhis = 0; | ||||
3724 | for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II); | ||||
3725 | ++II, ++NumPhis) { | ||||
3726 | if (NumPhis > 2) // Disable this xform. | ||||
3727 | return false; | ||||
3728 | |||||
3729 | PHINode *PN = cast<PHINode>(II); | ||||
3730 | Value *BIV = PN->getIncomingValueForBlock(BB); | ||||
3731 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV)) | ||||
3732 | if (CE->canTrap()) | ||||
3733 | return false; | ||||
3734 | |||||
3735 | unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent()); | ||||
3736 | Value *PBIV = PN->getIncomingValue(PBBIdx); | ||||
3737 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV)) | ||||
3738 | if (CE->canTrap()) | ||||
3739 | return false; | ||||
3740 | } | ||||
3741 | |||||
3742 | // Finally, if everything is ok, fold the branches to logical ops. | ||||
3743 | BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1); | ||||
3744 | |||||
3745 | LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()do { } while (false) | ||||
3746 | << "AND: " << *BI->getParent())do { } while (false); | ||||
3747 | |||||
3748 | SmallVector<DominatorTree::UpdateType, 5> Updates; | ||||
3749 | |||||
3750 | // If OtherDest *is* BB, then BB is a basic block with a single conditional | ||||
3751 | // branch in it, where one edge (OtherDest) goes back to itself but the other | ||||
3752 | // exits. We don't *know* that the program avoids the infinite loop | ||||
3753 | // (even though that seems likely). If we do this xform naively, we'll end up | ||||
3754 | // recursively unpeeling the loop. Since we know that (after the xform is | ||||
3755 | // done) that the block *is* infinite if reached, we just make it an obviously | ||||
3756 | // infinite loop with no cond branch. | ||||
3757 | if (OtherDest == BB) { | ||||
3758 | // Insert it at the end of the function, because it's either code, | ||||
3759 | // or it won't matter if it's hot. :) | ||||
3760 | BasicBlock *InfLoopBlock = | ||||
3761 | BasicBlock::Create(BB->getContext(), "infloop", BB->getParent()); | ||||
3762 | BranchInst::Create(InfLoopBlock, InfLoopBlock); | ||||
3763 | if (DTU) | ||||
3764 | Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock}); | ||||
3765 | OtherDest = InfLoopBlock; | ||||
3766 | } | ||||
3767 | |||||
3768 | LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { } while (false); | ||||
3769 | |||||
3770 | // BI may have other predecessors. Because of this, we leave | ||||
3771 | // it alone, but modify PBI. | ||||
3772 | |||||
3773 | // Make sure we get to CommonDest on True&True directions. | ||||
3774 | Value *PBICond = PBI->getCondition(); | ||||
3775 | IRBuilder<NoFolder> Builder(PBI); | ||||
3776 | if (PBIOp) | ||||
3777 | PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not"); | ||||
3778 | |||||
3779 | Value *BICond = BI->getCondition(); | ||||
3780 | if (BIOp) | ||||
3781 | BICond = Builder.CreateNot(BICond, BICond->getName() + ".not"); | ||||
3782 | |||||
3783 | // Merge the conditions. | ||||
3784 | Value *Cond = | ||||
3785 | createLogicalOp(Builder, Instruction::Or, PBICond, BICond, "brmerge"); | ||||
3786 | |||||
3787 | // Modify PBI to branch on the new condition to the new dests. | ||||
3788 | PBI->setCondition(Cond); | ||||
3789 | PBI->setSuccessor(0, CommonDest); | ||||
3790 | PBI->setSuccessor(1, OtherDest); | ||||
3791 | |||||
3792 | if (DTU) { | ||||
3793 | Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest}); | ||||
3794 | Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest}); | ||||
3795 | |||||
3796 | DTU->applyUpdates(Updates); | ||||
3797 | } | ||||
3798 | |||||
3799 | // Update branch weight for PBI. | ||||
3800 | uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight; | ||||
3801 | uint64_t PredCommon, PredOther, SuccCommon, SuccOther; | ||||
3802 | bool HasWeights = | ||||
3803 | extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight, | ||||
3804 | SuccTrueWeight, SuccFalseWeight); | ||||
3805 | if (HasWeights) { | ||||
3806 | PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight; | ||||
3807 | PredOther = PBIOp ? PredTrueWeight : PredFalseWeight; | ||||
3808 | SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight; | ||||
3809 | SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight; | ||||
3810 | // The weight to CommonDest should be PredCommon * SuccTotal + | ||||
3811 | // PredOther * SuccCommon. | ||||
3812 | // The weight to OtherDest should be PredOther * SuccOther. | ||||
3813 | uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) + | ||||
3814 | PredOther * SuccCommon, | ||||
3815 | PredOther * SuccOther}; | ||||
3816 | // Halve the weights if any of them cannot fit in an uint32_t | ||||
3817 | FitWeights(NewWeights); | ||||
3818 | |||||
3819 | setBranchWeights(PBI, NewWeights[0], NewWeights[1]); | ||||
3820 | } | ||||
3821 | |||||
3822 | // OtherDest may have phi nodes. If so, add an entry from PBI's | ||||
3823 | // block that are identical to the entries for BI's block. | ||||
3824 | AddPredecessorToBlock(OtherDest, PBI->getParent(), BB); | ||||
3825 | |||||
3826 | // We know that the CommonDest already had an edge from PBI to | ||||
3827 | // it. If it has PHIs though, the PHIs may have different | ||||
3828 | // entries for BB and PBI's BB. If so, insert a select to make | ||||
3829 | // them agree. | ||||
3830 | for (PHINode &PN : CommonDest->phis()) { | ||||
3831 | Value *BIV = PN.getIncomingValueForBlock(BB); | ||||
3832 | unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent()); | ||||
3833 | Value *PBIV = PN.getIncomingValue(PBBIdx); | ||||
3834 | if (BIV != PBIV) { | ||||
3835 | // Insert a select in PBI to pick the right value. | ||||
3836 | SelectInst *NV = cast<SelectInst>( | ||||
3837 | Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux")); | ||||
3838 | PN.setIncomingValue(PBBIdx, NV); | ||||
3839 | // Although the select has the same condition as PBI, the original branch | ||||
3840 | // weights for PBI do not apply to the new select because the select's | ||||
3841 | // 'logical' edges are incoming edges of the phi that is eliminated, not | ||||
3842 | // the outgoing edges of PBI. | ||||
3843 | if (HasWeights) { | ||||
3844 | uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight; | ||||
3845 | uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight; | ||||
3846 | uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight; | ||||
3847 | uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight; | ||||
3848 | // The weight to PredCommonDest should be PredCommon * SuccTotal. | ||||
3849 | // The weight to PredOtherDest should be PredOther * SuccCommon. | ||||
3850 | uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther), | ||||
3851 | PredOther * SuccCommon}; | ||||
3852 | |||||
3853 | FitWeights(NewWeights); | ||||
3854 | |||||
3855 | setBranchWeights(NV, NewWeights[0], NewWeights[1]); | ||||
3856 | } | ||||
3857 | } | ||||
3858 | } | ||||
3859 | |||||
3860 | LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent())do { } while (false); | ||||
3861 | LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { } while (false); | ||||
3862 | |||||
3863 | // This basic block is probably dead. We know it has at least | ||||
3864 | // one fewer predecessor. | ||||
3865 | return true; | ||||
3866 | } | ||||
3867 | |||||
3868 | // Simplifies a terminator by replacing it with a branch to TrueBB if Cond is | ||||
3869 | // true or to FalseBB if Cond is false. | ||||
3870 | // Takes care of updating the successors and removing the old terminator. | ||||
3871 | // Also makes sure not to introduce new successors by assuming that edges to | ||||
3872 | // non-successor TrueBBs and FalseBBs aren't reachable. | ||||
3873 | bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm, | ||||
3874 | Value *Cond, BasicBlock *TrueBB, | ||||
3875 | BasicBlock *FalseBB, | ||||
3876 | uint32_t TrueWeight, | ||||
3877 | uint32_t FalseWeight) { | ||||
3878 | auto *BB = OldTerm->getParent(); | ||||
3879 | // Remove any superfluous successor edges from the CFG. | ||||
3880 | // First, figure out which successors to preserve. | ||||
3881 | // If TrueBB and FalseBB are equal, only try to preserve one copy of that | ||||
3882 | // successor. | ||||
3883 | BasicBlock *KeepEdge1 = TrueBB; | ||||
3884 | BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr; | ||||
3885 | |||||
3886 | SmallPtrSet<BasicBlock *, 2> RemovedSuccessors; | ||||
3887 | |||||
3888 | // Then remove the rest. | ||||
3889 | for (BasicBlock *Succ : successors(OldTerm)) { | ||||
3890 | // Make sure only to keep exactly one copy of each edge. | ||||
3891 | if (Succ == KeepEdge1) | ||||
3892 | KeepEdge1 = nullptr; | ||||
3893 | else if (Succ == KeepEdge2) | ||||
3894 | KeepEdge2 = nullptr; | ||||
3895 | else { | ||||
3896 | Succ->removePredecessor(BB, | ||||
3897 | /*KeepOneInputPHIs=*/true); | ||||
3898 | |||||
3899 | if (Succ != TrueBB && Succ != FalseBB) | ||||
3900 | RemovedSuccessors.insert(Succ); | ||||
3901 | } | ||||
3902 | } | ||||
3903 | |||||
3904 | IRBuilder<> Builder(OldTerm); | ||||
3905 | Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc()); | ||||
3906 | |||||
3907 | // Insert an appropriate new terminator. | ||||
3908 | if (!KeepEdge1 && !KeepEdge2) { | ||||
3909 | if (TrueBB == FalseBB) { | ||||
3910 | // We were only looking for one successor, and it was present. | ||||
3911 | // Create an unconditional branch to it. | ||||
3912 | Builder.CreateBr(TrueBB); | ||||
3913 | } else { | ||||
3914 | // We found both of the successors we were looking for. | ||||
3915 | // Create a conditional branch sharing the condition of the select. | ||||
3916 | BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB); | ||||
3917 | if (TrueWeight != FalseWeight) | ||||
3918 | setBranchWeights(NewBI, TrueWeight, FalseWeight); | ||||
3919 | } | ||||
3920 | } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) { | ||||
3921 | // Neither of the selected blocks were successors, so this | ||||
3922 | // terminator must be unreachable. | ||||
3923 | new UnreachableInst(OldTerm->getContext(), OldTerm); | ||||
3924 | } else { | ||||
3925 | // One of the selected values was a successor, but the other wasn't. | ||||
3926 | // Insert an unconditional branch to the one that was found; | ||||
3927 | // the edge to the one that wasn't must be unreachable. | ||||
3928 | if (!KeepEdge1) { | ||||
3929 | // Only TrueBB was found. | ||||
3930 | Builder.CreateBr(TrueBB); | ||||
3931 | } else { | ||||
3932 | // Only FalseBB was found. | ||||
3933 | Builder.CreateBr(FalseBB); | ||||
3934 | } | ||||
3935 | } | ||||
3936 | |||||
3937 | EraseTerminatorAndDCECond(OldTerm); | ||||
3938 | |||||
3939 | if (DTU) { | ||||
3940 | SmallVector<DominatorTree::UpdateType, 2> Updates; | ||||
3941 | Updates.reserve(RemovedSuccessors.size()); | ||||
3942 | for (auto *RemovedSuccessor : RemovedSuccessors) | ||||
3943 | Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor}); | ||||
3944 | DTU->applyUpdates(Updates); | ||||
3945 | } | ||||
3946 | |||||
3947 | return true; | ||||
3948 | } | ||||
3949 | |||||
3950 | // Replaces | ||||
3951 | // (switch (select cond, X, Y)) on constant X, Y | ||||
3952 | // with a branch - conditional if X and Y lead to distinct BBs, | ||||
3953 | // unconditional otherwise. | ||||
3954 | bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI, | ||||
3955 | SelectInst *Select) { | ||||
3956 | // Check for constant integer values in the select. | ||||
3957 | ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue()); | ||||
3958 | ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue()); | ||||
3959 | if (!TrueVal || !FalseVal) | ||||
3960 | return false; | ||||
3961 | |||||
3962 | // Find the relevant condition and destinations. | ||||
3963 | Value *Condition = Select->getCondition(); | ||||
3964 | BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor(); | ||||
3965 | BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor(); | ||||
3966 | |||||
3967 | // Get weight for TrueBB and FalseBB. | ||||
3968 | uint32_t TrueWeight = 0, FalseWeight = 0; | ||||
3969 | SmallVector<uint64_t, 8> Weights; | ||||
3970 | bool HasWeights = HasBranchWeights(SI); | ||||
3971 | if (HasWeights) { | ||||
3972 | GetBranchWeights(SI, Weights); | ||||
3973 | if (Weights.size() == 1 + SI->getNumCases()) { | ||||
3974 | TrueWeight = | ||||
3975 | (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()]; | ||||
3976 | FalseWeight = | ||||
3977 | (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()]; | ||||
3978 | } | ||||
3979 | } | ||||
3980 | |||||
3981 | // Perform the actual simplification. | ||||
3982 | return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight, | ||||
3983 | FalseWeight); | ||||
3984 | } | ||||
3985 | |||||
3986 | // Replaces | ||||
3987 | // (indirectbr (select cond, blockaddress(@fn, BlockA), | ||||
3988 | // blockaddress(@fn, BlockB))) | ||||
3989 | // with | ||||
3990 | // (br cond, BlockA, BlockB). | ||||
3991 | bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, | ||||
3992 | SelectInst *SI) { | ||||
3993 | // Check that both operands of the select are block addresses. | ||||
3994 | BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue()); | ||||
3995 | BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue()); | ||||
3996 | if (!TBA || !FBA) | ||||
3997 | return false; | ||||
3998 | |||||
3999 | // Extract the actual blocks. | ||||
4000 | BasicBlock *TrueBB = TBA->getBasicBlock(); | ||||
4001 | BasicBlock *FalseBB = FBA->getBasicBlock(); | ||||
4002 | |||||
4003 | // Perform the actual simplification. | ||||
4004 | return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0, | ||||
4005 | 0); | ||||
4006 | } | ||||
4007 | |||||
4008 | /// This is called when we find an icmp instruction | ||||
4009 | /// (a seteq/setne with a constant) as the only instruction in a | ||||
4010 | /// block that ends with an uncond branch. We are looking for a very specific | ||||
4011 | /// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In | ||||
4012 | /// this case, we merge the first two "or's of icmp" into a switch, but then the | ||||
4013 | /// default value goes to an uncond block with a seteq in it, we get something | ||||
4014 | /// like: | ||||
4015 | /// | ||||
4016 | /// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ] | ||||
4017 | /// DEFAULT: | ||||
4018 | /// %tmp = icmp eq i8 %A, 92 | ||||
4019 | /// br label %end | ||||
4020 | /// end: | ||||
4021 | /// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ] | ||||
4022 | /// | ||||
4023 | /// We prefer to split the edge to 'end' so that there is a true/false entry to | ||||
4024 | /// the PHI, merging the third icmp into the switch. | ||||
4025 | bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt( | ||||
4026 | ICmpInst *ICI, IRBuilder<> &Builder) { | ||||
4027 | BasicBlock *BB = ICI->getParent(); | ||||
4028 | |||||
4029 | // If the block has any PHIs in it or the icmp has multiple uses, it is too | ||||
4030 | // complex. | ||||
4031 | if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse()) | ||||
4032 | return false; | ||||
4033 | |||||
4034 | Value *V = ICI->getOperand(0); | ||||
4035 | ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1)); | ||||
4036 | |||||
4037 | // The pattern we're looking for is where our only predecessor is a switch on | ||||
4038 | // 'V' and this block is the default case for the switch. In this case we can | ||||
4039 | // fold the compared value into the switch to simplify things. | ||||
4040 | BasicBlock *Pred = BB->getSinglePredecessor(); | ||||
4041 | if (!Pred || !isa<SwitchInst>(Pred->getTerminator())) | ||||
4042 | return false; | ||||
4043 | |||||
4044 | SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator()); | ||||
4045 | if (SI->getCondition() != V) | ||||
4046 | return false; | ||||
4047 | |||||
4048 | // If BB is reachable on a non-default case, then we simply know the value of | ||||
4049 | // V in this block. Substitute it and constant fold the icmp instruction | ||||
4050 | // away. | ||||
4051 | if (SI->getDefaultDest() != BB) { | ||||
4052 | ConstantInt *VVal = SI->findCaseDest(BB); | ||||
4053 | assert(VVal && "Should have a unique destination value")((void)0); | ||||
4054 | ICI->setOperand(0, VVal); | ||||
4055 | |||||
4056 | if (Value *V = SimplifyInstruction(ICI, {DL, ICI})) { | ||||
4057 | ICI->replaceAllUsesWith(V); | ||||
4058 | ICI->eraseFromParent(); | ||||
4059 | } | ||||
4060 | // BB is now empty, so it is likely to simplify away. | ||||
4061 | return requestResimplify(); | ||||
4062 | } | ||||
4063 | |||||
4064 | // Ok, the block is reachable from the default dest. If the constant we're | ||||
4065 | // comparing exists in one of the other edges, then we can constant fold ICI | ||||
4066 | // and zap it. | ||||
4067 | if (SI->findCaseValue(Cst) != SI->case_default()) { | ||||
4068 | Value *V; | ||||
4069 | if (ICI->getPredicate() == ICmpInst::ICMP_EQ) | ||||
4070 | V = ConstantInt::getFalse(BB->getContext()); | ||||
4071 | else | ||||
4072 | V = ConstantInt::getTrue(BB->getContext()); | ||||
4073 | |||||
4074 | ICI->replaceAllUsesWith(V); | ||||
4075 | ICI->eraseFromParent(); | ||||
4076 | // BB is now empty, so it is likely to simplify away. | ||||
4077 | return requestResimplify(); | ||||
4078 | } | ||||
4079 | |||||
4080 | // The use of the icmp has to be in the 'end' block, by the only PHI node in | ||||
4081 | // the block. | ||||
4082 | BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0); | ||||
4083 | PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back()); | ||||
4084 | if (PHIUse == nullptr || PHIUse != &SuccBlock->front() || | ||||
4085 | isa<PHINode>(++BasicBlock::iterator(PHIUse))) | ||||
4086 | return false; | ||||
4087 | |||||
4088 | // If the icmp is a SETEQ, then the default dest gets false, the new edge gets | ||||
4089 | // true in the PHI. | ||||
4090 | Constant *DefaultCst = ConstantInt::getTrue(BB->getContext()); | ||||
4091 | Constant *NewCst = ConstantInt::getFalse(BB->getContext()); | ||||
4092 | |||||
4093 | if (ICI->getPredicate() == ICmpInst::ICMP_EQ) | ||||
4094 | std::swap(DefaultCst, NewCst); | ||||
4095 | |||||
4096 | // Replace ICI (which is used by the PHI for the default value) with true or | ||||
4097 | // false depending on if it is EQ or NE. | ||||
4098 | ICI->replaceAllUsesWith(DefaultCst); | ||||
4099 | ICI->eraseFromParent(); | ||||
4100 | |||||
4101 | SmallVector<DominatorTree::UpdateType, 2> Updates; | ||||
4102 | |||||
4103 | // Okay, the switch goes to this block on a default value. Add an edge from | ||||
4104 | // the switch to the merge point on the compared value. | ||||
4105 | BasicBlock *NewBB = | ||||
4106 | BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB); | ||||
4107 | { | ||||
4108 | SwitchInstProfUpdateWrapper SIW(*SI); | ||||
4109 | auto W0 = SIW.getSuccessorWeight(0); | ||||
4110 | SwitchInstProfUpdateWrapper::CaseWeightOpt NewW; | ||||
4111 | if (W0) { | ||||
4112 | NewW = ((uint64_t(*W0) + 1) >> 1); | ||||
4113 | SIW.setSuccessorWeight(0, *NewW); | ||||
4114 | } | ||||
4115 | SIW.addCase(Cst, NewBB, NewW); | ||||
4116 | if (DTU) | ||||
4117 | Updates.push_back({DominatorTree::Insert, Pred, NewBB}); | ||||
4118 | } | ||||
4119 | |||||
4120 | // NewBB branches to the phi block, add the uncond branch and the phi entry. | ||||
4121 | Builder.SetInsertPoint(NewBB); | ||||
4122 | Builder.SetCurrentDebugLocation(SI->getDebugLoc()); | ||||
4123 | Builder.CreateBr(SuccBlock); | ||||
4124 | PHIUse->addIncoming(NewCst, NewBB); | ||||
4125 | if (DTU) { | ||||
4126 | Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock}); | ||||
4127 | DTU->applyUpdates(Updates); | ||||
4128 | } | ||||
4129 | return true; | ||||
4130 | } | ||||
4131 | |||||
4132 | /// The specified branch is a conditional branch. | ||||
4133 | /// Check to see if it is branching on an or/and chain of icmp instructions, and | ||||
4134 | /// fold it into a switch instruction if so. | ||||
4135 | bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI, | ||||
4136 | IRBuilder<> &Builder, | ||||
4137 | const DataLayout &DL) { | ||||
4138 | Instruction *Cond = dyn_cast<Instruction>(BI->getCondition()); | ||||
4139 | if (!Cond) | ||||
4140 | return false; | ||||
4141 | |||||
4142 | // Change br (X == 0 | X == 1), T, F into a switch instruction. | ||||
4143 | // If this is a bunch of seteq's or'd together, or if it's a bunch of | ||||
4144 | // 'setne's and'ed together, collect them. | ||||
4145 | |||||
4146 | // Try to gather values from a chain of and/or to be turned into a switch | ||||
4147 | ConstantComparesGatherer ConstantCompare(Cond, DL); | ||||
4148 | // Unpack the result | ||||
4149 | SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals; | ||||
4150 | Value *CompVal = ConstantCompare.CompValue; | ||||
4151 | unsigned UsedICmps = ConstantCompare.UsedICmps; | ||||
4152 | Value *ExtraCase = ConstantCompare.Extra; | ||||
4153 | |||||
4154 | // If we didn't have a multiply compared value, fail. | ||||
4155 | if (!CompVal) | ||||
4156 | return false; | ||||
4157 | |||||
4158 | // Avoid turning single icmps into a switch. | ||||
4159 | if (UsedICmps <= 1) | ||||
4160 | return false; | ||||
4161 | |||||
4162 | bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value())); | ||||
4163 | |||||
4164 | // There might be duplicate constants in the list, which the switch | ||||
4165 | // instruction can't handle, remove them now. | ||||
4166 | array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate); | ||||
4167 | Values.erase(std::unique(Values.begin(), Values.end()), Values.end()); | ||||
4168 | |||||
4169 | // If Extra was used, we require at least two switch values to do the | ||||
4170 | // transformation. A switch with one value is just a conditional branch. | ||||
4171 | if (ExtraCase && Values.size() < 2) | ||||
4172 | return false; | ||||
4173 | |||||
4174 | // TODO: Preserve branch weight metadata, similarly to how | ||||
4175 | // FoldValueComparisonIntoPredecessors preserves it. | ||||
4176 | |||||
4177 | // Figure out which block is which destination. | ||||
4178 | BasicBlock *DefaultBB = BI->getSuccessor(1); | ||||
4179 | BasicBlock *EdgeBB = BI->getSuccessor(0); | ||||
4180 | if (!TrueWhenEqual) | ||||
4181 | std::swap(DefaultBB, EdgeBB); | ||||
4182 | |||||
4183 | BasicBlock *BB = BI->getParent(); | ||||
4184 | |||||
4185 | LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()do { } while (false) | ||||
4186 | << " cases into SWITCH. BB is:\n"do { } while (false) | ||||
4187 | << *BB)do { } while (false); | ||||
4188 | |||||
4189 | SmallVector<DominatorTree::UpdateType, 2> Updates; | ||||
4190 | |||||
4191 | // If there are any extra values that couldn't be folded into the switch | ||||
4192 | // then we evaluate them with an explicit branch first. Split the block | ||||
4193 | // right before the condbr to handle it. | ||||
4194 | if (ExtraCase) { | ||||
4195 | BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr, | ||||
4196 | /*MSSAU=*/nullptr, "switch.early.test"); | ||||
4197 | |||||
4198 | // Remove the uncond branch added to the old block. | ||||
4199 | Instruction *OldTI = BB->getTerminator(); | ||||
4200 | Builder.SetInsertPoint(OldTI); | ||||
4201 | |||||
4202 | // There can be an unintended UB if extra values are Poison. Before the | ||||
4203 | // transformation, extra values may not be evaluated according to the | ||||
4204 | // condition, and it will not raise UB. But after transformation, we are | ||||
4205 | // evaluating extra values before checking the condition, and it will raise | ||||
4206 | // UB. It can be solved by adding freeze instruction to extra values. | ||||
4207 | AssumptionCache *AC = Options.AC; | ||||
4208 | |||||
4209 | if (!isGuaranteedNotToBeUndefOrPoison(ExtraCase, AC, BI, nullptr)) | ||||
4210 | ExtraCase = Builder.CreateFreeze(ExtraCase); | ||||
4211 | |||||
4212 | if (TrueWhenEqual) | ||||
4213 | Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB); | ||||
4214 | else | ||||
4215 | Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB); | ||||
4216 | |||||
4217 | OldTI->eraseFromParent(); | ||||
4218 | |||||
4219 | if (DTU) | ||||
4220 | Updates.push_back({DominatorTree::Insert, BB, EdgeBB}); | ||||
4221 | |||||
4222 | // If there are PHI nodes in EdgeBB, then we need to add a new entry to them | ||||
4223 | // for the edge we just added. | ||||
4224 | AddPredecessorToBlock(EdgeBB, BB, NewBB); | ||||
4225 | |||||
4226 | LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCasedo { } while (false) | ||||
4227 | << "\nEXTRABB = " << *BB)do { } while (false); | ||||
4228 | BB = NewBB; | ||||
4229 | } | ||||
4230 | |||||
4231 | Builder.SetInsertPoint(BI); | ||||
4232 | // Convert pointer to int before we switch. | ||||
4233 | if (CompVal->getType()->isPointerTy()) { | ||||
4234 | CompVal = Builder.CreatePtrToInt( | ||||
4235 | CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr"); | ||||
4236 | } | ||||
4237 | |||||
4238 | // Create the new switch instruction now. | ||||
4239 | SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size()); | ||||
4240 | |||||
4241 | // Add all of the 'cases' to the switch instruction. | ||||
4242 | for (unsigned i = 0, e = Values.size(); i != e; ++i) | ||||
4243 | New->addCase(Values[i], EdgeBB); | ||||
4244 | |||||
4245 | // We added edges from PI to the EdgeBB. As such, if there were any | ||||
4246 | // PHI nodes in EdgeBB, they need entries to be added corresponding to | ||||
4247 | // the number of edges added. | ||||
4248 | for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) { | ||||
4249 | PHINode *PN = cast<PHINode>(BBI); | ||||
4250 | Value *InVal = PN->getIncomingValueForBlock(BB); | ||||
4251 | for (unsigned i = 0, e = Values.size() - 1; i != e; ++i) | ||||
4252 | PN->addIncoming(InVal, BB); | ||||
4253 | } | ||||
4254 | |||||
4255 | // Erase the old branch instruction. | ||||
4256 | EraseTerminatorAndDCECond(BI); | ||||
4257 | if (DTU) | ||||
4258 | DTU->applyUpdates(Updates); | ||||
4259 | |||||
4260 | LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n')do { } while (false); | ||||
4261 | return true; | ||||
4262 | } | ||||
4263 | |||||
4264 | bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) { | ||||
4265 | if (isa<PHINode>(RI->getValue())) | ||||
4266 | return simplifyCommonResume(RI); | ||||
4267 | else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) && | ||||
4268 | RI->getValue() == RI->getParent()->getFirstNonPHI()) | ||||
4269 | // The resume must unwind the exception that caused control to branch here. | ||||
4270 | return simplifySingleResume(RI); | ||||
4271 | |||||
4272 | return false; | ||||
4273 | } | ||||
4274 | |||||
4275 | // Check if cleanup block is empty | ||||
4276 | static bool isCleanupBlockEmpty(iterator_range<BasicBlock::iterator> R) { | ||||
4277 | for (Instruction &I : R) { | ||||
4278 | auto *II = dyn_cast<IntrinsicInst>(&I); | ||||
4279 | if (!II) | ||||
4280 | return false; | ||||
4281 | |||||
4282 | Intrinsic::ID IntrinsicID = II->getIntrinsicID(); | ||||
4283 | switch (IntrinsicID) { | ||||
4284 | case Intrinsic::dbg_declare: | ||||
4285 | case Intrinsic::dbg_value: | ||||
4286 | case Intrinsic::dbg_label: | ||||
4287 | case Intrinsic::lifetime_end: | ||||
4288 | break; | ||||
4289 | default: | ||||
4290 | return false; | ||||
4291 | } | ||||
4292 | } | ||||
4293 | return true; | ||||
4294 | } | ||||
4295 | |||||
4296 | // Simplify resume that is shared by several landing pads (phi of landing pad). | ||||
4297 | bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) { | ||||
4298 | BasicBlock *BB = RI->getParent(); | ||||
4299 | |||||
4300 | // Check that there are no other instructions except for debug and lifetime | ||||
4301 | // intrinsics between the phi's and resume instruction. | ||||
4302 | if (!isCleanupBlockEmpty( | ||||
4303 | make_range(RI->getParent()->getFirstNonPHI(), BB->getTerminator()))) | ||||
4304 | return false; | ||||
4305 | |||||
4306 | SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks; | ||||
4307 | auto *PhiLPInst = cast<PHINode>(RI->getValue()); | ||||
4308 | |||||
4309 | // Check incoming blocks to see if any of them are trivial. | ||||
4310 | for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End; | ||||
4311 | Idx++) { | ||||
4312 | auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx); | ||||
4313 | auto *IncomingValue = PhiLPInst->getIncomingValue(Idx); | ||||
4314 | |||||
4315 | // If the block has other successors, we can not delete it because | ||||
4316 | // it has other dependents. | ||||
4317 | if (IncomingBB->getUniqueSuccessor() != BB) | ||||
4318 | continue; | ||||
4319 | |||||
4320 | auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI()); | ||||
4321 | // Not the landing pad that caused the control to branch here. | ||||
4322 | if (IncomingValue != LandingPad) | ||||
4323 | continue; | ||||
4324 | |||||
4325 | if (isCleanupBlockEmpty( | ||||
4326 | make_range(LandingPad->getNextNode(), IncomingBB->getTerminator()))) | ||||
4327 | TrivialUnwindBlocks.insert(IncomingBB); | ||||
4328 | } | ||||
4329 | |||||
4330 | // If no trivial unwind blocks, don't do any simplifications. | ||||
4331 | if (TrivialUnwindBlocks.empty()) | ||||
4332 | return false; | ||||
4333 | |||||
4334 | // Turn all invokes that unwind here into calls. | ||||
4335 | for (auto *TrivialBB : TrivialUnwindBlocks) { | ||||
4336 | // Blocks that will be simplified should be removed from the phi node. | ||||
4337 | // Note there could be multiple edges to the resume block, and we need | ||||
4338 | // to remove them all. | ||||
4339 | while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1) | ||||
4340 | BB->removePredecessor(TrivialBB, true); | ||||
4341 | |||||
4342 | for (BasicBlock *Pred : | ||||
4343 | llvm::make_early_inc_range(predecessors(TrivialBB))) { | ||||
4344 | removeUnwindEdge(Pred, DTU); | ||||
4345 | ++NumInvokes; | ||||
4346 | } | ||||
4347 | |||||
4348 | // In each SimplifyCFG run, only the current processed block can be erased. | ||||
4349 | // Otherwise, it will break the iteration of SimplifyCFG pass. So instead | ||||
4350 | // of erasing TrivialBB, we only remove the branch to the common resume | ||||
4351 | // block so that we can later erase the resume block since it has no | ||||
4352 | // predecessors. | ||||
4353 | TrivialBB->getTerminator()->eraseFromParent(); | ||||
4354 | new UnreachableInst(RI->getContext(), TrivialBB); | ||||
4355 | if (DTU) | ||||
4356 | DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}}); | ||||
4357 | } | ||||
4358 | |||||
4359 | // Delete the resume block if all its predecessors have been removed. | ||||
4360 | if (pred_empty(BB)) | ||||
4361 | DeleteDeadBlock(BB, DTU); | ||||
4362 | |||||
4363 | return !TrivialUnwindBlocks.empty(); | ||||
4364 | } | ||||
4365 | |||||
4366 | // Simplify resume that is only used by a single (non-phi) landing pad. | ||||
4367 | bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) { | ||||
4368 | BasicBlock *BB = RI->getParent(); | ||||
4369 | auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI()); | ||||
4370 | assert(RI->getValue() == LPInst &&((void)0) | ||||
4371 | "Resume must unwind the exception that caused control to here")((void)0); | ||||
4372 | |||||
4373 | // Check that there are no other instructions except for debug intrinsics. | ||||
4374 | if (!isCleanupBlockEmpty( | ||||
4375 | make_range<Instruction *>(LPInst->getNextNode(), RI))) | ||||
4376 | return false; | ||||
4377 | |||||
4378 | // Turn all invokes that unwind here into calls and delete the basic block. | ||||
4379 | for (BasicBlock *Pred : llvm::make_early_inc_range(predecessors(BB))) { | ||||
4380 | removeUnwindEdge(Pred, DTU); | ||||
4381 | ++NumInvokes; | ||||
4382 | } | ||||
4383 | |||||
4384 | // The landingpad is now unreachable. Zap it. | ||||
4385 | DeleteDeadBlock(BB, DTU); | ||||
4386 | return true; | ||||
4387 | } | ||||
4388 | |||||
4389 | static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) { | ||||
4390 | // If this is a trivial cleanup pad that executes no instructions, it can be | ||||
4391 | // eliminated. If the cleanup pad continues to the caller, any predecessor | ||||
4392 | // that is an EH pad will be updated to continue to the caller and any | ||||
4393 | // predecessor that terminates with an invoke instruction will have its invoke | ||||
4394 | // instruction converted to a call instruction. If the cleanup pad being | ||||
4395 | // simplified does not continue to the caller, each predecessor will be | ||||
4396 | // updated to continue to the unwind destination of the cleanup pad being | ||||
4397 | // simplified. | ||||
4398 | BasicBlock *BB = RI->getParent(); | ||||
4399 | CleanupPadInst *CPInst = RI->getCleanupPad(); | ||||
4400 | if (CPInst->getParent() != BB) | ||||
4401 | // This isn't an empty cleanup. | ||||
4402 | return false; | ||||
4403 | |||||
4404 | // We cannot kill the pad if it has multiple uses. This typically arises | ||||
4405 | // from unreachable basic blocks. | ||||
4406 | if (!CPInst->hasOneUse()) | ||||
4407 | return false; | ||||
4408 | |||||
4409 | // Check that there are no other instructions except for benign intrinsics. | ||||
4410 | if (!isCleanupBlockEmpty( | ||||
4411 | make_range<Instruction *>(CPInst->getNextNode(), RI))) | ||||
4412 | return false; | ||||
4413 | |||||
4414 | // If the cleanup return we are simplifying unwinds to the caller, this will | ||||
4415 | // set UnwindDest to nullptr. | ||||
4416 | BasicBlock *UnwindDest = RI->getUnwindDest(); | ||||
4417 | Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr; | ||||
4418 | |||||
4419 | // We're about to remove BB from the control flow. Before we do, sink any | ||||
4420 | // PHINodes into the unwind destination. Doing this before changing the | ||||
4421 | // control flow avoids some potentially slow checks, since we can currently | ||||
4422 | // be certain that UnwindDest and BB have no common predecessors (since they | ||||
4423 | // are both EH pads). | ||||
4424 | if (UnwindDest) { | ||||
4425 | // First, go through the PHI nodes in UnwindDest and update any nodes that | ||||
4426 | // reference the block we are removing | ||||
4427 | for (PHINode &DestPN : UnwindDest->phis()) { | ||||
4428 | int Idx = DestPN.getBasicBlockIndex(BB); | ||||
4429 | // Since BB unwinds to UnwindDest, it has to be in the PHI node. | ||||
4430 | assert(Idx != -1)((void)0); | ||||
4431 | // This PHI node has an incoming value that corresponds to a control | ||||
4432 | // path through the cleanup pad we are removing. If the incoming | ||||
4433 | // value is in the cleanup pad, it must be a PHINode (because we | ||||
4434 | // verified above that the block is otherwise empty). Otherwise, the | ||||
4435 | // value is either a constant or a value that dominates the cleanup | ||||
4436 | // pad being removed. | ||||
4437 | // | ||||
4438 | // Because BB and UnwindDest are both EH pads, all of their | ||||
4439 | // predecessors must unwind to these blocks, and since no instruction | ||||
4440 | // can have multiple unwind destinations, there will be no overlap in | ||||
4441 | // incoming blocks between SrcPN and DestPN. | ||||
4442 | Value *SrcVal = DestPN.getIncomingValue(Idx); | ||||
4443 | PHINode *SrcPN = dyn_cast<PHINode>(SrcVal); | ||||
4444 | |||||
4445 | bool NeedPHITranslation = SrcPN && SrcPN->getParent() == BB; | ||||
4446 | for (auto *Pred : predecessors(BB)) { | ||||
4447 | Value *Incoming = | ||||
4448 | NeedPHITranslation ? SrcPN->getIncomingValueForBlock(Pred) : SrcVal; | ||||
4449 | DestPN.addIncoming(Incoming, Pred); | ||||
4450 | } | ||||
4451 | } | ||||
4452 | |||||
4453 | // Sink any remaining PHI nodes directly into UnwindDest. | ||||
4454 | Instruction *InsertPt = DestEHPad; | ||||
4455 | for (PHINode &PN : make_early_inc_range(BB->phis())) { | ||||
4456 | if (PN.use_empty() || !PN.isUsedOutsideOfBlock(BB)) | ||||
4457 | // If the PHI node has no uses or all of its uses are in this basic | ||||
4458 | // block (meaning they are debug or lifetime intrinsics), just leave | ||||
4459 | // it. It will be erased when we erase BB below. | ||||
4460 | continue; | ||||
4461 | |||||
4462 | // Otherwise, sink this PHI node into UnwindDest. | ||||
4463 | // Any predecessors to UnwindDest which are not already represented | ||||
4464 | // must be back edges which inherit the value from the path through | ||||
4465 | // BB. In this case, the PHI value must reference itself. | ||||
4466 | for (auto *pred : predecessors(UnwindDest)) | ||||
4467 | if (pred != BB) | ||||
4468 | PN.addIncoming(&PN, pred); | ||||
4469 | PN.moveBefore(InsertPt); | ||||
4470 | // Also, add a dummy incoming value for the original BB itself, | ||||
4471 | // so that the PHI is well-formed until we drop said predecessor. | ||||
4472 | PN.addIncoming(UndefValue::get(PN.getType()), BB); | ||||
4473 | } | ||||
4474 | } | ||||
4475 | |||||
4476 | std::vector<DominatorTree::UpdateType> Updates; | ||||
4477 | |||||
4478 | // We use make_early_inc_range here because we will remove all predecessors. | ||||
4479 | for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(BB))) { | ||||
4480 | if (UnwindDest == nullptr) { | ||||
4481 | if (DTU) { | ||||
4482 | DTU->applyUpdates(Updates); | ||||
4483 | Updates.clear(); | ||||
4484 | } | ||||
4485 | removeUnwindEdge(PredBB, DTU); | ||||
4486 | ++NumInvokes; | ||||
4487 | } else { | ||||
4488 | BB->removePredecessor(PredBB); | ||||
4489 | Instruction *TI = PredBB->getTerminator(); | ||||
4490 | TI->replaceUsesOfWith(BB, UnwindDest); | ||||
4491 | if (DTU) { | ||||
4492 | Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest}); | ||||
4493 | Updates.push_back({DominatorTree::Delete, PredBB, BB}); | ||||
4494 | } | ||||
4495 | } | ||||
4496 | } | ||||
4497 | |||||
4498 | if (DTU) | ||||
4499 | DTU->applyUpdates(Updates); | ||||
4500 | |||||
4501 | DeleteDeadBlock(BB, DTU); | ||||
4502 | |||||
4503 | return true; | ||||
4504 | } | ||||
4505 | |||||
4506 | // Try to merge two cleanuppads together. | ||||
4507 | static bool mergeCleanupPad(CleanupReturnInst *RI) { | ||||
4508 | // Skip any cleanuprets which unwind to caller, there is nothing to merge | ||||
4509 | // with. | ||||
4510 | BasicBlock *UnwindDest = RI->getUnwindDest(); | ||||
4511 | if (!UnwindDest) | ||||
4512 | return false; | ||||
4513 | |||||
4514 | // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't | ||||
4515 | // be safe to merge without code duplication. | ||||
4516 | if (UnwindDest->getSinglePredecessor() != RI->getParent()) | ||||
4517 | return false; | ||||
4518 | |||||
4519 | // Verify that our cleanuppad's unwind destination is another cleanuppad. | ||||
4520 | auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front()); | ||||
4521 | if (!SuccessorCleanupPad) | ||||
4522 | return false; | ||||
4523 | |||||
4524 | CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad(); | ||||
4525 | // Replace any uses of the successor cleanupad with the predecessor pad | ||||
4526 | // The only cleanuppad uses should be this cleanupret, it's cleanupret and | ||||
4527 | // funclet bundle operands. | ||||
4528 | SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad); | ||||
4529 | // Remove the old cleanuppad. | ||||
4530 | SuccessorCleanupPad->eraseFromParent(); | ||||
4531 | // Now, we simply replace the cleanupret with a branch to the unwind | ||||
4532 | // destination. | ||||
4533 | BranchInst::Create(UnwindDest, RI->getParent()); | ||||
4534 | RI->eraseFromParent(); | ||||
4535 | |||||
4536 | return true; | ||||
4537 | } | ||||
4538 | |||||
4539 | bool SimplifyCFGOpt::simplifyCleanupReturn(CleanupReturnInst *RI) { | ||||
4540 | // It is possible to transiantly have an undef cleanuppad operand because we | ||||
4541 | // have deleted some, but not all, dead blocks. | ||||
4542 | // Eventually, this block will be deleted. | ||||
4543 | if (isa<UndefValue>(RI->getOperand(0))) | ||||
4544 | return false; | ||||
4545 | |||||
4546 | if (mergeCleanupPad(RI)) | ||||
4547 | return true; | ||||
4548 | |||||
4549 | if (removeEmptyCleanup(RI, DTU)) | ||||
4550 | return true; | ||||
4551 | |||||
4552 | return false; | ||||
4553 | } | ||||
4554 | |||||
4555 | // WARNING: keep in sync with InstCombinerImpl::visitUnreachableInst()! | ||||
4556 | bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) { | ||||
4557 | BasicBlock *BB = UI->getParent(); | ||||
4558 | |||||
4559 | bool Changed = false; | ||||
4560 | |||||
4561 | // If there are any instructions immediately before the unreachable that can | ||||
4562 | // be removed, do so. | ||||
4563 | while (UI->getIterator() != BB->begin()) { | ||||
4564 | BasicBlock::iterator BBI = UI->getIterator(); | ||||
4565 | --BBI; | ||||
4566 | |||||
4567 | if (!isGuaranteedToTransferExecutionToSuccessor(&*BBI)) | ||||
4568 | break; // Can not drop any more instructions. We're done here. | ||||
4569 | // Otherwise, this instruction can be freely erased, | ||||
4570 | // even if it is not side-effect free. | ||||
4571 | |||||
4572 | // Note that deleting EH's here is in fact okay, although it involves a bit | ||||
4573 | // of subtle reasoning. If this inst is an EH, all the predecessors of this | ||||
4574 | // block will be the unwind edges of Invoke/CatchSwitch/CleanupReturn, | ||||
4575 | // and we can therefore guarantee this block will be erased. | ||||
4576 | |||||
4577 | // Delete this instruction (any uses are guaranteed to be dead) | ||||
4578 | BBI->replaceAllUsesWith(PoisonValue::get(BBI->getType())); | ||||
4579 | BBI->eraseFromParent(); | ||||
4580 | Changed = true; | ||||
4581 | } | ||||
4582 | |||||
4583 | // If the unreachable instruction is the first in the block, take a gander | ||||
4584 | // at all of the predecessors of this instruction, and simplify them. | ||||
4585 | if (&BB->front() != UI) | ||||
4586 | return Changed; | ||||
4587 | |||||
4588 | std::vector<DominatorTree::UpdateType> Updates; | ||||
4589 | |||||
4590 | SmallSetVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB)); | ||||
4591 | for (unsigned i = 0, e = Preds.size(); i != e; ++i) { | ||||
4592 | auto *Predecessor = Preds[i]; | ||||
4593 | Instruction *TI = Predecessor->getTerminator(); | ||||
4594 | IRBuilder<> Builder(TI); | ||||
4595 | if (auto *BI = dyn_cast<BranchInst>(TI)) { | ||||
4596 | // We could either have a proper unconditional branch, | ||||
4597 | // or a degenerate conditional branch with matching destinations. | ||||
4598 | if (all_of(BI->successors(), | ||||
4599 | [BB](auto *Successor) { return Successor == BB; })) { | ||||
4600 | new UnreachableInst(TI->getContext(), TI); | ||||
4601 | TI->eraseFromParent(); | ||||
4602 | Changed = true; | ||||
4603 | } else { | ||||
4604 | assert(BI->isConditional() && "Can't get here with an uncond branch.")((void)0); | ||||
4605 | Value* Cond = BI->getCondition(); | ||||
4606 | assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&((void)0) | ||||
4607 | "The destinations are guaranteed to be different here.")((void)0); | ||||
4608 | if (BI->getSuccessor(0) == BB) { | ||||
4609 | Builder.CreateAssumption(Builder.CreateNot(Cond)); | ||||
4610 | Builder.CreateBr(BI->getSuccessor(1)); | ||||
4611 | } else { | ||||
4612 | assert(BI->getSuccessor(1) == BB && "Incorrect CFG")((void)0); | ||||
4613 | Builder.CreateAssumption(Cond); | ||||
4614 | Builder.CreateBr(BI->getSuccessor(0)); | ||||
4615 | } | ||||
4616 | EraseTerminatorAndDCECond(BI); | ||||
4617 | Changed = true; | ||||
4618 | } | ||||
4619 | if (DTU) | ||||
4620 | Updates.push_back({DominatorTree::Delete, Predecessor, BB}); | ||||
4621 | } else if (auto *SI = dyn_cast<SwitchInst>(TI)) { | ||||
4622 | SwitchInstProfUpdateWrapper SU(*SI); | ||||
4623 | for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) { | ||||
4624 | if (i->getCaseSuccessor() != BB) { | ||||
4625 | ++i; | ||||
4626 | continue; | ||||
4627 | } | ||||
4628 | BB->removePredecessor(SU->getParent()); | ||||
4629 | i = SU.removeCase(i); | ||||
4630 | e = SU->case_end(); | ||||
4631 | Changed = true; | ||||
4632 | } | ||||
4633 | // Note that the default destination can't be removed! | ||||
4634 | if (DTU && SI->getDefaultDest() != BB) | ||||
4635 | Updates.push_back({DominatorTree::Delete, Predecessor, BB}); | ||||
4636 | } else if (auto *II = dyn_cast<InvokeInst>(TI)) { | ||||
4637 | if (II->getUnwindDest() == BB) { | ||||
4638 | if (DTU) { | ||||
4639 | DTU->applyUpdates(Updates); | ||||
4640 | Updates.clear(); | ||||
4641 | } | ||||
4642 | removeUnwindEdge(TI->getParent(), DTU); | ||||
4643 | Changed = true; | ||||
4644 | } | ||||
4645 | } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) { | ||||
4646 | if (CSI->getUnwindDest() == BB) { | ||||
4647 | if (DTU) { | ||||
4648 | DTU->applyUpdates(Updates); | ||||
4649 | Updates.clear(); | ||||
4650 | } | ||||
4651 | removeUnwindEdge(TI->getParent(), DTU); | ||||
4652 | Changed = true; | ||||
4653 | continue; | ||||
4654 | } | ||||
4655 | |||||
4656 | for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(), | ||||
4657 | E = CSI->handler_end(); | ||||
4658 | I != E; ++I) { | ||||
4659 | if (*I == BB) { | ||||
4660 | CSI->removeHandler(I); | ||||
4661 | --I; | ||||
4662 | --E; | ||||
4663 | Changed = true; | ||||
4664 | } | ||||
4665 | } | ||||
4666 | if (DTU) | ||||
4667 | Updates.push_back({DominatorTree::Delete, Predecessor, BB}); | ||||
4668 | if (CSI->getNumHandlers() == 0) { | ||||
4669 | if (CSI->hasUnwindDest()) { | ||||
4670 | // Redirect all predecessors of the block containing CatchSwitchInst | ||||
4671 | // to instead branch to the CatchSwitchInst's unwind destination. | ||||
4672 | if (DTU) { | ||||
4673 | for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) { | ||||
4674 | Updates.push_back({DominatorTree::Insert, | ||||
4675 | PredecessorOfPredecessor, | ||||
4676 | CSI->getUnwindDest()}); | ||||
4677 | Updates.push_back({DominatorTree::Delete, | ||||
4678 | PredecessorOfPredecessor, Predecessor}); | ||||
4679 | } | ||||
4680 | } | ||||
4681 | Predecessor->replaceAllUsesWith(CSI->getUnwindDest()); | ||||
4682 | } else { | ||||
4683 | // Rewrite all preds to unwind to caller (or from invoke to call). | ||||
4684 | if (DTU) { | ||||
4685 | DTU->applyUpdates(Updates); | ||||
4686 | Updates.clear(); | ||||
4687 | } | ||||
4688 | SmallVector<BasicBlock *, 8> EHPreds(predecessors(Predecessor)); | ||||
4689 | for (BasicBlock *EHPred : EHPreds) | ||||
4690 | removeUnwindEdge(EHPred, DTU); | ||||
4691 | } | ||||
4692 | // The catchswitch is no longer reachable. | ||||
4693 | new UnreachableInst(CSI->getContext(), CSI); | ||||
4694 | CSI->eraseFromParent(); | ||||
4695 | Changed = true; | ||||
4696 | } | ||||
4697 | } else if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) { | ||||
4698 | (void)CRI; | ||||
4699 | assert(CRI->hasUnwindDest() && CRI->getUnwindDest() == BB &&((void)0) | ||||
4700 | "Expected to always have an unwind to BB.")((void)0); | ||||
4701 | if (DTU) | ||||
4702 | Updates.push_back({DominatorTree::Delete, Predecessor, BB}); | ||||
4703 | new UnreachableInst(TI->getContext(), TI); | ||||
4704 | TI->eraseFromParent(); | ||||
4705 | Changed = true; | ||||
4706 | } | ||||
4707 | } | ||||
4708 | |||||
4709 | if (DTU) | ||||
4710 | DTU->applyUpdates(Updates); | ||||
4711 | |||||
4712 | // If this block is now dead, remove it. | ||||
4713 | if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) { | ||||
4714 | DeleteDeadBlock(BB, DTU); | ||||
4715 | return true; | ||||
4716 | } | ||||
4717 | |||||
4718 | return Changed; | ||||
4719 | } | ||||
4720 | |||||
4721 | static bool CasesAreContiguous(SmallVectorImpl<ConstantInt *> &Cases) { | ||||
4722 | assert(Cases.size() >= 1)((void)0); | ||||
4723 | |||||
4724 | array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate); | ||||
4725 | for (size_t I = 1, E = Cases.size(); I != E; ++I) { | ||||
4726 | if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1) | ||||
4727 | return false; | ||||
4728 | } | ||||
4729 | return true; | ||||
4730 | } | ||||
4731 | |||||
4732 | static void createUnreachableSwitchDefault(SwitchInst *Switch, | ||||
4733 | DomTreeUpdater *DTU) { | ||||
4734 | LLVM_DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n")do { } while (false); | ||||
4735 | auto *BB = Switch->getParent(); | ||||
4736 | BasicBlock *NewDefaultBlock = SplitBlockPredecessors( | ||||
4737 | Switch->getDefaultDest(), Switch->getParent(), "", DTU); | ||||
4738 | auto *OrigDefaultBlock = Switch->getDefaultDest(); | ||||
4739 | Switch->setDefaultDest(&*NewDefaultBlock); | ||||
4740 | if (DTU) | ||||
4741 | DTU->applyUpdates({{DominatorTree::Insert, BB, &*NewDefaultBlock}, | ||||
4742 | {DominatorTree::Delete, BB, OrigDefaultBlock}}); | ||||
4743 | SplitBlock(&*NewDefaultBlock, &NewDefaultBlock->front(), DTU); | ||||
4744 | SmallVector<DominatorTree::UpdateType, 2> Updates; | ||||
4745 | if (DTU) | ||||
4746 | for (auto *Successor : successors(NewDefaultBlock)) | ||||
4747 | Updates.push_back({DominatorTree::Delete, NewDefaultBlock, Successor}); | ||||
4748 | auto *NewTerminator = NewDefaultBlock->getTerminator(); | ||||
4749 | new UnreachableInst(Switch->getContext(), NewTerminator); | ||||
4750 | EraseTerminatorAndDCECond(NewTerminator); | ||||
4751 | if (DTU) | ||||
4752 | DTU->applyUpdates(Updates); | ||||
4753 | } | ||||
4754 | |||||
4755 | /// Turn a switch with two reachable destinations into an integer range | ||||
4756 | /// comparison and branch. | ||||
4757 | bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI, | ||||
4758 | IRBuilder<> &Builder) { | ||||
4759 | assert(SI->getNumCases() > 1 && "Degenerate switch?")((void)0); | ||||
4760 | |||||
4761 | bool HasDefault = | ||||
4762 | !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); | ||||
4763 | |||||
4764 | auto *BB = SI->getParent(); | ||||
4765 | |||||
4766 | // Partition the cases into two sets with different destinations. | ||||
4767 | BasicBlock *DestA = HasDefault
| ||||
4768 | BasicBlock *DestB = nullptr; | ||||
4769 | SmallVector<ConstantInt *, 16> CasesA; | ||||
4770 | SmallVector<ConstantInt *, 16> CasesB; | ||||
4771 | |||||
4772 | for (auto Case : SI->cases()) { | ||||
4773 | BasicBlock *Dest = Case.getCaseSuccessor(); | ||||
4774 | if (!DestA) | ||||
4775 | DestA = Dest; | ||||
4776 | if (Dest == DestA) { | ||||
4777 | CasesA.push_back(Case.getCaseValue()); | ||||
4778 | continue; | ||||
4779 | } | ||||
4780 | if (!DestB) | ||||
4781 | DestB = Dest; | ||||
4782 | if (Dest == DestB) { | ||||
4783 | CasesB.push_back(Case.getCaseValue()); | ||||
4784 | continue; | ||||
4785 | } | ||||
4786 | return false; // More than two destinations. | ||||
4787 | } | ||||
4788 | |||||
4789 | assert(DestA && DestB &&((void)0) | ||||
4790 | "Single-destination switch should have been folded.")((void)0); | ||||
4791 | assert(DestA != DestB)((void)0); | ||||
4792 | assert(DestB != SI->getDefaultDest())((void)0); | ||||
4793 | assert(!CasesB.empty() && "There must be non-default cases.")((void)0); | ||||
4794 | assert(!CasesA.empty() || HasDefault)((void)0); | ||||
4795 | |||||
4796 | // Figure out if one of the sets of cases form a contiguous range. | ||||
4797 | SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr; | ||||
4798 | BasicBlock *ContiguousDest = nullptr; | ||||
4799 | BasicBlock *OtherDest = nullptr; | ||||
4800 | if (!CasesA.empty() && CasesAreContiguous(CasesA)) { | ||||
4801 | ContiguousCases = &CasesA; | ||||
4802 | ContiguousDest = DestA; | ||||
4803 | OtherDest = DestB; | ||||
4804 | } else if (CasesAreContiguous(CasesB)) { | ||||
4805 | ContiguousCases = &CasesB; | ||||
4806 | ContiguousDest = DestB; | ||||
4807 | OtherDest = DestA; | ||||
4808 | } else | ||||
4809 | return false; | ||||
4810 | |||||
4811 | // Start building the compare and branch. | ||||
4812 | |||||
4813 | Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back()); | ||||
4814 | Constant *NumCases = | ||||
4815 | ConstantInt::get(Offset->getType(), ContiguousCases->size()); | ||||
4816 | |||||
4817 | Value *Sub = SI->getCondition(); | ||||
4818 | if (!Offset->isNullValue()) | ||||
4819 | Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off"); | ||||
4820 | |||||
4821 | Value *Cmp; | ||||
4822 | // If NumCases overflowed, then all possible values jump to the successor. | ||||
4823 | if (NumCases->isNullValue() && !ContiguousCases->empty()) | ||||
4824 | Cmp = ConstantInt::getTrue(SI->getContext()); | ||||
4825 | else | ||||
4826 | Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch"); | ||||
4827 | BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest); | ||||
4828 | |||||
4829 | // Update weight for the newly-created conditional branch. | ||||
4830 | if (HasBranchWeights(SI)) { | ||||
4831 | SmallVector<uint64_t, 8> Weights; | ||||
4832 | GetBranchWeights(SI, Weights); | ||||
4833 | if (Weights.size() == 1 + SI->getNumCases()) { | ||||
4834 | uint64_t TrueWeight = 0; | ||||
4835 | uint64_t FalseWeight = 0; | ||||
4836 | for (size_t I = 0, E = Weights.size(); I != E; ++I) { | ||||
4837 | if (SI->getSuccessor(I) == ContiguousDest) | ||||
4838 | TrueWeight += Weights[I]; | ||||
4839 | else | ||||
4840 | FalseWeight += Weights[I]; | ||||
4841 | } | ||||
4842 | while (TrueWeight > UINT32_MAX0xffffffffU || FalseWeight > UINT32_MAX0xffffffffU) { | ||||
4843 | TrueWeight /= 2; | ||||
4844 | FalseWeight /= 2; | ||||
4845 | } | ||||
4846 | setBranchWeights(NewBI, TrueWeight, FalseWeight); | ||||
4847 | } | ||||
4848 | } | ||||
4849 | |||||
4850 | // Prune obsolete incoming values off the successors' PHI nodes. | ||||
4851 | for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) { | ||||
4852 | unsigned PreviousEdges = ContiguousCases->size(); | ||||
4853 | if (ContiguousDest == SI->getDefaultDest()) | ||||
4854 | ++PreviousEdges; | ||||
4855 | for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I) | ||||
4856 | cast<PHINode>(BBI)->removeIncomingValue(SI->getParent()); | ||||
4857 | } | ||||
4858 | for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) { | ||||
| |||||
4859 | unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size(); | ||||
4860 | if (OtherDest == SI->getDefaultDest()) | ||||
4861 | ++PreviousEdges; | ||||
4862 | for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I) | ||||
4863 | cast<PHINode>(BBI)->removeIncomingValue(SI->getParent()); | ||||
4864 | } | ||||
4865 | |||||
4866 | // Clean up the default block - it may have phis or other instructions before | ||||
4867 | // the unreachable terminator. | ||||
4868 | if (!HasDefault) | ||||
4869 | createUnreachableSwitchDefault(SI, DTU); | ||||
4870 | |||||
4871 | auto *UnreachableDefault = SI->getDefaultDest(); | ||||
4872 | |||||
4873 | // Drop the switch. | ||||
4874 | SI->eraseFromParent(); | ||||
4875 | |||||
4876 | if (!HasDefault && DTU) | ||||
4877 | DTU->applyUpdates({{DominatorTree::Delete, BB, UnreachableDefault}}); | ||||
4878 | |||||
4879 | return true; | ||||
4880 | } | ||||
4881 | |||||
4882 | /// Compute masked bits for the condition of a switch | ||||
4883 | /// and use it to remove dead cases. | ||||
4884 | static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU, | ||||
4885 | AssumptionCache *AC, | ||||
4886 | const DataLayout &DL) { | ||||
4887 | Value *Cond = SI->getCondition(); | ||||
4888 | unsigned Bits = Cond->getType()->getIntegerBitWidth(); | ||||
4889 | KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI); | ||||
4890 | |||||
4891 | // We can also eliminate cases by determining that their values are outside of | ||||
4892 | // the limited range of the condition based on how many significant (non-sign) | ||||
4893 | // bits are in the condition value. | ||||
4894 | unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1; | ||||
4895 | unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits; | ||||
4896 | |||||
4897 | // Gather dead cases. | ||||
4898 | SmallVector<ConstantInt *, 8> DeadCases; | ||||
4899 | SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases; | ||||
4900 | for (auto &Case : SI->cases()) { | ||||
4901 | auto *Successor = Case.getCaseSuccessor(); | ||||
4902 | if (DTU) | ||||
4903 | ++NumPerSuccessorCases[Successor]; | ||||
4904 | const APInt &CaseVal = Case.getCaseValue()->getValue(); | ||||
4905 | if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) || | ||||
4906 | (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) { | ||||
4907 | DeadCases.push_back(Case.getCaseValue()); | ||||
4908 | if (DTU) | ||||
4909 | --NumPerSuccessorCases[Successor]; | ||||
4910 | LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseValdo { } while (false) | ||||
4911 | << " is dead.\n")do { } while (false); | ||||
4912 | } | ||||
4913 | } | ||||
4914 | |||||
4915 | // If we can prove that the cases must cover all possible values, the | ||||
4916 | // default destination becomes dead and we can remove it. If we know some | ||||
4917 | // of the bits in the value, we can use that to more precisely compute the | ||||
4918 | // number of possible unique case values. | ||||
4919 | bool HasDefault = | ||||
4920 | !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); | ||||
4921 | const unsigned NumUnknownBits = | ||||
4922 | Bits - (Known.Zero | Known.One).countPopulation(); | ||||
4923 | assert(NumUnknownBits <= Bits)((void)0); | ||||
4924 | if (HasDefault && DeadCases.empty() && | ||||
4925 | NumUnknownBits < 64 /* avoid overflow */ && | ||||
4926 | SI->getNumCases() == (1ULL << NumUnknownBits)) { | ||||
4927 | createUnreachableSwitchDefault(SI, DTU); | ||||
4928 | return true; | ||||
4929 | } | ||||
4930 | |||||
4931 | if (DeadCases.empty()) | ||||
4932 | return false; | ||||
4933 | |||||
4934 | SwitchInstProfUpdateWrapper SIW(*SI); | ||||
4935 | for (ConstantInt *DeadCase : DeadCases) { | ||||
4936 | SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase); | ||||
4937 | assert(CaseI != SI->case_default() &&((void)0) | ||||
4938 | "Case was not found. Probably mistake in DeadCases forming.")((void)0); | ||||
4939 | // Prune unused values from PHI nodes. | ||||
4940 | CaseI->getCaseSuccessor()->removePredecessor(SI->getParent()); | ||||
4941 | SIW.removeCase(CaseI); | ||||
4942 | } | ||||
4943 | |||||
4944 | if (DTU) { | ||||
4945 | std::vector<DominatorTree::UpdateType> Updates; | ||||
4946 | for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases) | ||||
4947 | if (I.second == 0) | ||||
4948 | Updates.push_back({DominatorTree::Delete, SI->getParent(), I.first}); | ||||
4949 | DTU->applyUpdates(Updates); | ||||
4950 | } | ||||
4951 | |||||
4952 | return true; | ||||
4953 | } | ||||
4954 | |||||
4955 | /// If BB would be eligible for simplification by | ||||
4956 | /// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated | ||||
4957 | /// by an unconditional branch), look at the phi node for BB in the successor | ||||
4958 | /// block and see if the incoming value is equal to CaseValue. If so, return | ||||
4959 | /// the phi node, and set PhiIndex to BB's index in the phi node. | ||||
4960 | static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue, | ||||
4961 | BasicBlock *BB, int *PhiIndex) { | ||||
4962 | if (BB->getFirstNonPHIOrDbg() != BB->getTerminator()) | ||||
4963 | return nullptr; // BB must be empty to be a candidate for simplification. | ||||
4964 | if (!BB->getSinglePredecessor()) | ||||
4965 | return nullptr; // BB must be dominated by the switch. | ||||
4966 | |||||
4967 | BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator()); | ||||
4968 | if (!Branch || !Branch->isUnconditional()) | ||||
4969 | return nullptr; // Terminator must be unconditional branch. | ||||
4970 | |||||
4971 | BasicBlock *Succ = Branch->getSuccessor(0); | ||||
4972 | |||||
4973 | for (PHINode &PHI : Succ->phis()) { | ||||
4974 | int Idx = PHI.getBasicBlockIndex(BB); | ||||
4975 | assert(Idx >= 0 && "PHI has no entry for predecessor?")((void)0); | ||||
4976 | |||||
4977 | Value *InValue = PHI.getIncomingValue(Idx); | ||||
4978 | if (InValue != CaseValue) | ||||
4979 | continue; | ||||
4980 | |||||
4981 | *PhiIndex = Idx; | ||||
4982 | return &PHI; | ||||
4983 | } | ||||
4984 | |||||
4985 | return nullptr; | ||||
4986 | } | ||||
4987 | |||||
4988 | /// Try to forward the condition of a switch instruction to a phi node | ||||
4989 | /// dominated by the switch, if that would mean that some of the destination | ||||
4990 | /// blocks of the switch can be folded away. Return true if a change is made. | ||||
4991 | static bool ForwardSwitchConditionToPHI(SwitchInst *SI) { | ||||
4992 | using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>; | ||||
4993 | |||||
4994 | ForwardingNodesMap ForwardingNodes; | ||||
4995 | BasicBlock *SwitchBlock = SI->getParent(); | ||||
4996 | bool Changed = false; | ||||
4997 | for (auto &Case : SI->cases()) { | ||||
4998 | ConstantInt *CaseValue = Case.getCaseValue(); | ||||
4999 | BasicBlock *CaseDest = Case.getCaseSuccessor(); | ||||
5000 | |||||
5001 | // Replace phi operands in successor blocks that are using the constant case | ||||
5002 | // value rather than the switch condition variable: | ||||
5003 | // switchbb: | ||||
5004 | // switch i32 %x, label %default [ | ||||
5005 | // i32 17, label %succ | ||||
5006 | // ... | ||||
5007 | // succ: | ||||
5008 | // %r = phi i32 ... [ 17, %switchbb ] ... | ||||
5009 | // --> | ||||
5010 | // %r = phi i32 ... [ %x, %switchbb ] ... | ||||
5011 | |||||
5012 | for (PHINode &Phi : CaseDest->phis()) { | ||||
5013 | // This only works if there is exactly 1 incoming edge from the switch to | ||||
5014 | // a phi. If there is >1, that means multiple cases of the switch map to 1 | ||||
5015 | // value in the phi, and that phi value is not the switch condition. Thus, | ||||
5016 | // this transform would not make sense (the phi would be invalid because | ||||
5017 | // a phi can't have different incoming values from the same block). | ||||
5018 | int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock); | ||||
5019 | if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue && | ||||
5020 | count(Phi.blocks(), SwitchBlock) == 1) { | ||||
5021 | Phi.setIncomingValue(SwitchBBIdx, SI->getCondition()); | ||||
5022 | Changed = true; | ||||
5023 | } | ||||
5024 | } | ||||
5025 | |||||
5026 | // Collect phi nodes that are indirectly using this switch's case constants. | ||||
5027 | int PhiIdx; | ||||
5028 | if (auto *Phi = FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx)) | ||||
5029 | ForwardingNodes[Phi].push_back(PhiIdx); | ||||
5030 | } | ||||
5031 | |||||
5032 | for (auto &ForwardingNode : ForwardingNodes) { | ||||
5033 | PHINode *Phi = ForwardingNode.first; | ||||
5034 | SmallVectorImpl<int> &Indexes = ForwardingNode.second; | ||||
5035 | if (Indexes.size() < 2) | ||||
5036 | continue; | ||||
5037 | |||||
5038 | for (int Index : Indexes) | ||||
5039 | Phi->setIncomingValue(Index, SI->getCondition()); | ||||
5040 | Changed = true; | ||||
5041 | } | ||||
5042 | |||||
5043 | return Changed; | ||||
5044 | } | ||||
5045 | |||||
5046 | /// Return true if the backend will be able to handle | ||||
5047 | /// initializing an array of constants like C. | ||||
5048 | static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI) { | ||||
5049 | if (C->isThreadDependent()) | ||||
5050 | return false; | ||||
5051 | if (C->isDLLImportDependent()) | ||||
5052 | return false; | ||||
5053 | |||||
5054 | if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) && | ||||
5055 | !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) && | ||||
5056 | !isa<UndefValue>(C) && !isa<ConstantExpr>(C)) | ||||
5057 | return false; | ||||
5058 | |||||
5059 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { | ||||
5060 | if (!CE->isGEPWithNoNotionalOverIndexing()) | ||||
5061 | return false; | ||||
5062 | if (!ValidLookupTableConstant(CE->getOperand(0), TTI)) | ||||
5063 | return false; | ||||
5064 | } | ||||
5065 | |||||
5066 | if (!TTI.shouldBuildLookupTablesForConstant(C)) | ||||
5067 | return false; | ||||
5068 | |||||
5069 | return true; | ||||
5070 | } | ||||
5071 | |||||
5072 | /// If V is a Constant, return it. Otherwise, try to look up | ||||
5073 | /// its constant value in ConstantPool, returning 0 if it's not there. | ||||
5074 | static Constant * | ||||
5075 | LookupConstant(Value *V, | ||||
5076 | const SmallDenseMap<Value *, Constant *> &ConstantPool) { | ||||
5077 | if (Constant *C = dyn_cast<Constant>(V)) | ||||
5078 | return C; | ||||
5079 | return ConstantPool.lookup(V); | ||||
5080 | } | ||||
5081 | |||||
5082 | /// Try to fold instruction I into a constant. This works for | ||||
5083 | /// simple instructions such as binary operations where both operands are | ||||
5084 | /// constant or can be replaced by constants from the ConstantPool. Returns the | ||||
5085 | /// resulting constant on success, 0 otherwise. | ||||
5086 | static Constant * | ||||
5087 | ConstantFold(Instruction *I, const DataLayout &DL, | ||||
5088 | const SmallDenseMap<Value *, Constant *> &ConstantPool) { | ||||
5089 | if (SelectInst *Select = dyn_cast<SelectInst>(I)) { | ||||
5090 | Constant *A = LookupConstant(Select->getCondition(), ConstantPool); | ||||
5091 | if (!A) | ||||
5092 | return nullptr; | ||||
5093 | if (A->isAllOnesValue()) | ||||
5094 | return LookupConstant(Select->getTrueValue(), ConstantPool); | ||||
5095 | if (A->isNullValue()) | ||||
5096 | return LookupConstant(Select->getFalseValue(), ConstantPool); | ||||
5097 | return nullptr; | ||||
5098 | } | ||||
5099 | |||||
5100 | SmallVector<Constant *, 4> COps; | ||||
5101 | for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) { | ||||
5102 | if (Constant *A = LookupConstant(I->getOperand(N), ConstantPool)) | ||||
5103 | COps.push_back(A); | ||||
5104 | else | ||||
5105 | return nullptr; | ||||
5106 | } | ||||
5107 | |||||
5108 | if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) { | ||||
5109 | return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0], | ||||
5110 | COps[1], DL); | ||||
5111 | } | ||||
5112 | |||||
5113 | return ConstantFoldInstOperands(I, COps, DL); | ||||
5114 | } | ||||
5115 | |||||
5116 | /// Try to determine the resulting constant values in phi nodes | ||||
5117 | /// at the common destination basic block, *CommonDest, for one of the case | ||||
5118 | /// destionations CaseDest corresponding to value CaseVal (0 for the default | ||||
5119 | /// case), of a switch instruction SI. | ||||
5120 | static bool | ||||
5121 | GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest, | ||||
5122 | BasicBlock **CommonDest, | ||||
5123 | SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res, | ||||
5124 | const DataLayout &DL, const TargetTransformInfo &TTI) { | ||||
5125 | // The block from which we enter the common destination. | ||||
5126 | BasicBlock *Pred = SI->getParent(); | ||||
5127 | |||||
5128 | // If CaseDest is empty except for some side-effect free instructions through | ||||
5129 | // which we can constant-propagate the CaseVal, continue to its successor. | ||||
5130 | SmallDenseMap<Value *, Constant *> ConstantPool; | ||||
5131 | ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal)); | ||||
5132 | for (Instruction &I :CaseDest->instructionsWithoutDebug()) { | ||||
5133 | if (I.isTerminator()) { | ||||
5134 | // If the terminator is a simple branch, continue to the next block. | ||||
5135 | if (I.getNumSuccessors() != 1 || I.isExceptionalTerminator()) | ||||
5136 | return false; | ||||
5137 | Pred = CaseDest; | ||||
5138 | CaseDest = I.getSuccessor(0); | ||||
5139 | } else if (Constant *C = ConstantFold(&I, DL, ConstantPool)) { | ||||
5140 | // Instruction is side-effect free and constant. | ||||
5141 | |||||
5142 | // If the instruction has uses outside this block or a phi node slot for | ||||
5143 | // the block, it is not safe to bypass the instruction since it would then | ||||
5144 | // no longer dominate all its uses. | ||||
5145 | for (auto &Use : I.uses()) { | ||||
5146 | User *User = Use.getUser(); | ||||
5147 | if (Instruction *I = dyn_cast<Instruction>(User)) | ||||
5148 | if (I->getParent() == CaseDest) | ||||
5149 | continue; | ||||
5150 | if (PHINode *Phi = dyn_cast<PHINode>(User)) | ||||
5151 | if (Phi->getIncomingBlock(Use) == CaseDest) | ||||
5152 | continue; | ||||
5153 | return false; | ||||
5154 | } | ||||
5155 | |||||
5156 | ConstantPool.insert(std::make_pair(&I, C)); | ||||
5157 | } else { | ||||
5158 | break; | ||||
5159 | } | ||||
5160 | } | ||||
5161 | |||||
5162 | // If we did not have a CommonDest before, use the current one. | ||||
5163 | if (!*CommonDest) | ||||
5164 | *CommonDest = CaseDest; | ||||
5165 | // If the destination isn't the common one, abort. | ||||
5166 | if (CaseDest != *CommonDest) | ||||
5167 | return false; | ||||
5168 | |||||
5169 | // Get the values for this case from phi nodes in the destination block. | ||||
5170 | for (PHINode &PHI : (*CommonDest)->phis()) { | ||||
5171 | int Idx = PHI.getBasicBlockIndex(Pred); | ||||
5172 | if (Idx == -1) | ||||
5173 | continue; | ||||
5174 | |||||
5175 | Constant *ConstVal = | ||||
5176 | LookupConstant(PHI.getIncomingValue(Idx), ConstantPool); | ||||
5177 | if (!ConstVal) | ||||
5178 | return false; | ||||
5179 | |||||
5180 | // Be conservative about which kinds of constants we support. | ||||
5181 | if (!ValidLookupTableConstant(ConstVal, TTI)) | ||||
5182 | return false; | ||||
5183 | |||||
5184 | Res.push_back(std::make_pair(&PHI, ConstVal)); | ||||
5185 | } | ||||
5186 | |||||
5187 | return Res.size() > 0; | ||||
5188 | } | ||||
5189 | |||||
5190 | // Helper function used to add CaseVal to the list of cases that generate | ||||
5191 | // Result. Returns the updated number of cases that generate this result. | ||||
5192 | static uintptr_t MapCaseToResult(ConstantInt *CaseVal, | ||||
5193 | SwitchCaseResultVectorTy &UniqueResults, | ||||
5194 | Constant *Result) { | ||||
5195 | for (auto &I : UniqueResults) { | ||||
5196 | if (I.first == Result) { | ||||
5197 | I.second.push_back(CaseVal); | ||||
5198 | return I.second.size(); | ||||
5199 | } | ||||
5200 | } | ||||
5201 | UniqueResults.push_back( | ||||
5202 | std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal))); | ||||
5203 | return 1; | ||||
5204 | } | ||||
5205 | |||||
5206 | // Helper function that initializes a map containing | ||||
5207 | // results for the PHI node of the common destination block for a switch | ||||
5208 | // instruction. Returns false if multiple PHI nodes have been found or if | ||||
5209 | // there is not a common destination block for the switch. | ||||
5210 | static bool | ||||
5211 | InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest, | ||||
5212 | SwitchCaseResultVectorTy &UniqueResults, | ||||
5213 | Constant *&DefaultResult, const DataLayout &DL, | ||||
5214 | const TargetTransformInfo &TTI, | ||||
5215 | uintptr_t MaxUniqueResults, uintptr_t MaxCasesPerResult) { | ||||
5216 | for (auto &I : SI->cases()) { | ||||
5217 | ConstantInt *CaseVal = I.getCaseValue(); | ||||
5218 | |||||
5219 | // Resulting value at phi nodes for this case value. | ||||
5220 | SwitchCaseResultsTy Results; | ||||
5221 | if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results, | ||||
5222 | DL, TTI)) | ||||
5223 | return false; | ||||
5224 | |||||
5225 | // Only one value per case is permitted. | ||||
5226 | if (Results.size() > 1) | ||||
5227 | return false; | ||||
5228 | |||||
5229 | // Add the case->result mapping to UniqueResults. | ||||
5230 | const uintptr_t NumCasesForResult = | ||||
5231 | MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second); | ||||
5232 | |||||
5233 | // Early out if there are too many cases for this result. | ||||
5234 | if (NumCasesForResult > MaxCasesPerResult) | ||||
5235 | return false; | ||||
5236 | |||||
5237 | // Early out if there are too many unique results. | ||||
5238 | if (UniqueResults.size() > MaxUniqueResults) | ||||
5239 | return false; | ||||
5240 | |||||
5241 | // Check the PHI consistency. | ||||
5242 | if (!PHI) | ||||
5243 | PHI = Results[0].first; | ||||
5244 | else if (PHI != Results[0].first) | ||||
5245 | return false; | ||||
5246 | } | ||||
5247 | // Find the default result value. | ||||
5248 | SmallVector<std::pair<PHINode *, Constant *>, 1> DefaultResults; | ||||
5249 | BasicBlock *DefaultDest = SI->getDefaultDest(); | ||||
5250 | GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults, | ||||
5251 | DL, TTI); | ||||
5252 | // If the default value is not found abort unless the default destination | ||||
5253 | // is unreachable. | ||||
5254 | DefaultResult = | ||||
5255 | DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr; | ||||
5256 | if ((!DefaultResult && | ||||
5257 | !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg()))) | ||||
5258 | return false; | ||||
5259 | |||||
5260 | return true; | ||||
5261 | } | ||||
5262 | |||||
5263 | // Helper function that checks if it is possible to transform a switch with only | ||||
5264 | // two cases (or two cases + default) that produces a result into a select. | ||||
5265 | // Example: | ||||
5266 | // switch (a) { | ||||
5267 | // case 10: %0 = icmp eq i32 %a, 10 | ||||
5268 | // return 10; %1 = select i1 %0, i32 10, i32 4 | ||||
5269 | // case 20: ----> %2 = icmp eq i32 %a, 20 | ||||
5270 | // return 2; %3 = select i1 %2, i32 2, i32 %1 | ||||
5271 | // default: | ||||
5272 | // return 4; | ||||
5273 | // } | ||||
5274 | static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector, | ||||
5275 | Constant *DefaultResult, Value *Condition, | ||||
5276 | IRBuilder<> &Builder) { | ||||
5277 | // If we are selecting between only two cases transform into a simple | ||||
5278 | // select or a two-way select if default is possible. | ||||
5279 | if (ResultVector.size() == 2 && ResultVector[0].second.size() == 1 && | ||||
5280 | ResultVector[1].second.size() == 1) { | ||||
5281 | ConstantInt *const FirstCase = ResultVector[0].second[0]; | ||||
5282 | ConstantInt *const SecondCase = ResultVector[1].second[0]; | ||||
5283 | |||||
5284 | bool DefaultCanTrigger = DefaultResult; | ||||
5285 | Value *SelectValue = ResultVector[1].first; | ||||
5286 | if (DefaultCanTrigger) { | ||||
5287 | Value *const ValueCompare = | ||||
5288 | Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp"); | ||||
5289 | SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first, | ||||
5290 | DefaultResult, "switch.select"); | ||||
5291 | } | ||||
5292 | Value *const ValueCompare = | ||||
5293 | Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp"); | ||||
5294 | return Builder.CreateSelect(ValueCompare, ResultVector[0].first, | ||||
5295 | SelectValue, "switch.select"); | ||||
5296 | } | ||||
5297 | |||||
5298 | // Handle the degenerate case where two cases have the same value. | ||||
5299 | if (ResultVector.size() == 1 && ResultVector[0].second.size() == 2 && | ||||
5300 | DefaultResult) { | ||||
5301 | Value *Cmp1 = Builder.CreateICmpEQ( | ||||
5302 | Condition, ResultVector[0].second[0], "switch.selectcmp.case1"); | ||||
5303 | Value *Cmp2 = Builder.CreateICmpEQ( | ||||
5304 | Condition, ResultVector[0].second[1], "switch.selectcmp.case2"); | ||||
5305 | Value *Cmp = Builder.CreateOr(Cmp1, Cmp2, "switch.selectcmp"); | ||||
5306 | return Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult); | ||||
5307 | } | ||||
5308 | |||||
5309 | return nullptr; | ||||
5310 | } | ||||
5311 | |||||
5312 | // Helper function to cleanup a switch instruction that has been converted into | ||||
5313 | // a select, fixing up PHI nodes and basic blocks. | ||||
5314 | static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI, | ||||
5315 | Value *SelectValue, | ||||
5316 | IRBuilder<> &Builder, | ||||
5317 | DomTreeUpdater *DTU) { | ||||
5318 | std::vector<DominatorTree::UpdateType> Updates; | ||||
5319 | |||||
5320 | BasicBlock *SelectBB = SI->getParent(); | ||||
5321 | BasicBlock *DestBB = PHI->getParent(); | ||||
5322 | |||||
5323 | if (DTU && !is_contained(predecessors(DestBB), SelectBB)) | ||||
5324 | Updates.push_back({DominatorTree::Insert, SelectBB, DestBB}); | ||||
5325 | Builder.CreateBr(DestBB); | ||||
5326 | |||||
5327 | // Remove the switch. | ||||
5328 | |||||
5329 | while (PHI->getBasicBlockIndex(SelectBB) >= 0) | ||||
5330 | PHI->removeIncomingValue(SelectBB); | ||||
5331 | PHI->addIncoming(SelectValue, SelectBB); | ||||
5332 | |||||
5333 | SmallPtrSet<BasicBlock *, 4> RemovedSuccessors; | ||||
5334 | for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) { | ||||
5335 | BasicBlock *Succ = SI->getSuccessor(i); | ||||
5336 | |||||
5337 | if (Succ == DestBB) | ||||
5338 | continue; | ||||
5339 | Succ->removePredecessor(SelectBB); | ||||
5340 | if (DTU && RemovedSuccessors.insert(Succ).second) | ||||
5341 | Updates.push_back({DominatorTree::Delete, SelectBB, Succ}); | ||||
5342 | } | ||||
5343 | SI->eraseFromParent(); | ||||
5344 | if (DTU) | ||||
5345 | DTU->applyUpdates(Updates); | ||||
5346 | } | ||||
5347 | |||||
5348 | /// If the switch is only used to initialize one or more | ||||
5349 | /// phi nodes in a common successor block with only two different | ||||
5350 | /// constant values, replace the switch with select. | ||||
5351 | static bool switchToSelect(SwitchInst *SI, IRBuilder<> &Builder, | ||||
5352 | DomTreeUpdater *DTU, const DataLayout &DL, | ||||
5353 | const TargetTransformInfo &TTI) { | ||||
5354 | Value *const Cond = SI->getCondition(); | ||||
5355 | PHINode *PHI = nullptr; | ||||
5356 | BasicBlock *CommonDest = nullptr; | ||||
5357 | Constant *DefaultResult; | ||||
5358 | SwitchCaseResultVectorTy UniqueResults; | ||||
5359 | // Collect all the cases that will deliver the same value from the switch. | ||||
5360 | if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult, | ||||
5361 | DL, TTI, /*MaxUniqueResults*/2, | ||||
5362 | /*MaxCasesPerResult*/2)) | ||||
5363 | return false; | ||||
5364 | assert(PHI != nullptr && "PHI for value select not found")((void)0); | ||||
5365 | |||||
5366 | Builder.SetInsertPoint(SI); | ||||
5367 | Value *SelectValue = | ||||
5368 | ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder); | ||||
5369 | if (SelectValue) { | ||||
5370 | RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder, DTU); | ||||
5371 | return true; | ||||
5372 | } | ||||
5373 | // The switch couldn't be converted into a select. | ||||
5374 | return false; | ||||
5375 | } | ||||
5376 | |||||
5377 | namespace { | ||||
5378 | |||||
5379 | /// This class represents a lookup table that can be used to replace a switch. | ||||
5380 | class SwitchLookupTable { | ||||
5381 | public: | ||||
5382 | /// Create a lookup table to use as a switch replacement with the contents | ||||
5383 | /// of Values, using DefaultValue to fill any holes in the table. | ||||
5384 | SwitchLookupTable( | ||||
5385 | Module &M, uint64_t TableSize, ConstantInt *Offset, | ||||
5386 | const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values, | ||||
5387 | Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName); | ||||
5388 | |||||
5389 | /// Build instructions with Builder to retrieve the value at | ||||
5390 | /// the position given by Index in the lookup table. | ||||
5391 | Value *BuildLookup(Value *Index, IRBuilder<> &Builder); | ||||
5392 | |||||
5393 | /// Return true if a table with TableSize elements of | ||||
5394 | /// type ElementType would fit in a target-legal register. | ||||
5395 | static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize, | ||||
5396 | Type *ElementType); | ||||
5397 | |||||
5398 | private: | ||||
5399 | // Depending on the contents of the table, it can be represented in | ||||
5400 | // different ways. | ||||
5401 | enum { | ||||
5402 | // For tables where each element contains the same value, we just have to | ||||
5403 | // store that single value and return it for each lookup. | ||||
5404 | SingleValueKind, | ||||
5405 | |||||
5406 | // For tables where there is a linear relationship between table index | ||||
5407 | // and values. We calculate the result with a simple multiplication | ||||
5408 | // and addition instead of a table lookup. | ||||
5409 | LinearMapKind, | ||||
5410 | |||||
5411 | // For small tables with integer elements, we can pack them into a bitmap | ||||
5412 | // that fits into a target-legal register. Values are retrieved by | ||||
5413 | // shift and mask operations. | ||||
5414 | BitMapKind, | ||||
5415 | |||||
5416 | // The table is stored as an array of values. Values are retrieved by load | ||||
5417 | // instructions from the table. | ||||
5418 | ArrayKind | ||||
5419 | } Kind; | ||||
5420 | |||||
5421 | // For SingleValueKind, this is the single value. | ||||
5422 | Constant *SingleValue = nullptr; | ||||
5423 | |||||
5424 | // For BitMapKind, this is the bitmap. | ||||
5425 | ConstantInt *BitMap = nullptr; | ||||
5426 | IntegerType *BitMapElementTy = nullptr; | ||||
5427 | |||||
5428 | // For LinearMapKind, these are the constants used to derive the value. | ||||
5429 | ConstantInt *LinearOffset = nullptr; | ||||
5430 | ConstantInt *LinearMultiplier = nullptr; | ||||
5431 | |||||
5432 | // For ArrayKind, this is the array. | ||||
5433 | GlobalVariable *Array = nullptr; | ||||
5434 | }; | ||||
5435 | |||||
5436 | } // end anonymous namespace | ||||
5437 | |||||
5438 | SwitchLookupTable::SwitchLookupTable( | ||||
5439 | Module &M, uint64_t TableSize, ConstantInt *Offset, | ||||
5440 | const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values, | ||||
5441 | Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName) { | ||||
5442 | assert(Values.size() && "Can't build lookup table without values!")((void)0); | ||||
5443 | assert(TableSize >= Values.size() && "Can't fit values in table!")((void)0); | ||||
5444 | |||||
5445 | // If all values in the table are equal, this is that value. | ||||
5446 | SingleValue = Values.begin()->second; | ||||
5447 | |||||
5448 | Type *ValueType = Values.begin()->second->getType(); | ||||
5449 | |||||
5450 | // Build up the table contents. | ||||
5451 | SmallVector<Constant *, 64> TableContents(TableSize); | ||||
5452 | for (size_t I = 0, E = Values.size(); I != E; ++I) { | ||||
5453 | ConstantInt *CaseVal = Values[I].first; | ||||
5454 | Constant *CaseRes = Values[I].second; | ||||
5455 | assert(CaseRes->getType() == ValueType)((void)0); | ||||
5456 | |||||
5457 | uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue(); | ||||
5458 | TableContents[Idx] = CaseRes; | ||||
5459 | |||||
5460 | if (CaseRes != SingleValue) | ||||
5461 | SingleValue = nullptr; | ||||
5462 | } | ||||
5463 | |||||
5464 | // Fill in any holes in the table with the default result. | ||||
5465 | if (Values.size() < TableSize) { | ||||
5466 | assert(DefaultValue &&((void)0) | ||||
5467 | "Need a default value to fill the lookup table holes.")((void)0); | ||||
5468 | assert(DefaultValue->getType() == ValueType)((void)0); | ||||
5469 | for (uint64_t I = 0; I < TableSize; ++I) { | ||||
5470 | if (!TableContents[I]) | ||||
5471 | TableContents[I] = DefaultValue; | ||||
5472 | } | ||||
5473 | |||||
5474 | if (DefaultValue != SingleValue) | ||||
5475 | SingleValue = nullptr; | ||||
5476 | } | ||||
5477 | |||||
5478 | // If each element in the table contains the same value, we only need to store | ||||
5479 | // that single value. | ||||
5480 | if (SingleValue) { | ||||
5481 | Kind = SingleValueKind; | ||||
5482 | return; | ||||
5483 | } | ||||
5484 | |||||
5485 | // Check if we can derive the value with a linear transformation from the | ||||
5486 | // table index. | ||||
5487 | if (isa<IntegerType>(ValueType)) { | ||||
5488 | bool LinearMappingPossible = true; | ||||
5489 | APInt PrevVal; | ||||
5490 | APInt DistToPrev; | ||||
5491 | assert(TableSize >= 2 && "Should be a SingleValue table.")((void)0); | ||||
5492 | // Check if there is the same distance between two consecutive values. | ||||
5493 | for (uint64_t I = 0; I < TableSize; ++I) { | ||||
5494 | ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]); | ||||
5495 | if (!ConstVal) { | ||||
5496 | // This is an undef. We could deal with it, but undefs in lookup tables | ||||
5497 | // are very seldom. It's probably not worth the additional complexity. | ||||
5498 | LinearMappingPossible = false; | ||||
5499 | break; | ||||
5500 | } | ||||
5501 | const APInt &Val = ConstVal->getValue(); | ||||
5502 | if (I != 0) { | ||||
5503 | APInt Dist = Val - PrevVal; | ||||
5504 | if (I == 1) { | ||||
5505 | DistToPrev = Dist; | ||||
5506 | } else if (Dist != DistToPrev) { | ||||
5507 | LinearMappingPossible = false; | ||||
5508 | break; | ||||
5509 | } | ||||
5510 | } | ||||
5511 | PrevVal = Val; | ||||
5512 | } | ||||
5513 | if (LinearMappingPossible) { | ||||
5514 | LinearOffset = cast<ConstantInt>(TableContents[0]); | ||||
5515 | LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev); | ||||
5516 | Kind = LinearMapKind; | ||||
5517 | ++NumLinearMaps; | ||||
5518 | return; | ||||
5519 | } | ||||
5520 | } | ||||
5521 | |||||
5522 | // If the type is integer and the table fits in a register, build a bitmap. | ||||
5523 | if (WouldFitInRegister(DL, TableSize, ValueType)) { | ||||
5524 | IntegerType *IT = cast<IntegerType>(ValueType); | ||||
5525 | APInt TableInt(TableSize * IT->getBitWidth(), 0); | ||||
5526 | for (uint64_t I = TableSize; I > 0; --I) { | ||||
5527 | TableInt <<= IT->getBitWidth(); | ||||
5528 | // Insert values into the bitmap. Undef values are set to zero. | ||||
5529 | if (!isa<UndefValue>(TableContents[I - 1])) { | ||||
5530 | ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]); | ||||
5531 | TableInt |= Val->getValue().zext(TableInt.getBitWidth()); | ||||
5532 | } | ||||
5533 | } | ||||
5534 | BitMap = ConstantInt::get(M.getContext(), TableInt); | ||||
5535 | BitMapElementTy = IT; | ||||
5536 | Kind = BitMapKind; | ||||
5537 | ++NumBitMaps; | ||||
5538 | return; | ||||
5539 | } | ||||
5540 | |||||
5541 | // Store the table in an array. | ||||
5542 | ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize); | ||||
5543 | Constant *Initializer = ConstantArray::get(ArrayTy, TableContents); | ||||
5544 | |||||
5545 | Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true, | ||||
5546 | GlobalVariable::PrivateLinkage, Initializer, | ||||
5547 | "switch.table." + FuncName); | ||||
5548 | Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); | ||||
5549 | // Set the alignment to that of an array items. We will be only loading one | ||||
5550 | // value out of it. | ||||
5551 | Array->setAlignment(Align(DL.getPrefTypeAlignment(ValueType))); | ||||
5552 | Kind = ArrayKind; | ||||
5553 | } | ||||
5554 | |||||
5555 | Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) { | ||||
5556 | switch (Kind) { | ||||
5557 | case SingleValueKind: | ||||
5558 | return SingleValue; | ||||
5559 | case LinearMapKind: { | ||||
5560 | // Derive the result value from the input value. | ||||
5561 | Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(), | ||||
5562 | false, "switch.idx.cast"); | ||||
5563 | if (!LinearMultiplier->isOne()) | ||||
5564 | Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult"); | ||||
5565 | if (!LinearOffset->isZero()) | ||||
5566 | Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset"); | ||||
5567 | return Result; | ||||
5568 | } | ||||
5569 | case BitMapKind: { | ||||
5570 | // Type of the bitmap (e.g. i59). | ||||
5571 | IntegerType *MapTy = BitMap->getType(); | ||||
5572 | |||||
5573 | // Cast Index to the same type as the bitmap. | ||||
5574 | // Note: The Index is <= the number of elements in the table, so | ||||
5575 | // truncating it to the width of the bitmask is safe. | ||||
5576 | Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast"); | ||||
5577 | |||||
5578 | // Multiply the shift amount by the element width. | ||||
5579 | ShiftAmt = Builder.CreateMul( | ||||
5580 | ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()), | ||||
5581 | "switch.shiftamt"); | ||||
5582 | |||||
5583 | // Shift down. | ||||
5584 | Value *DownShifted = | ||||
5585 | Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift"); | ||||
5586 | // Mask off. | ||||
5587 | return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked"); | ||||
5588 | } | ||||
5589 | case ArrayKind: { | ||||
5590 | // Make sure the table index will not overflow when treated as signed. | ||||
5591 | IntegerType *IT = cast<IntegerType>(Index->getType()); | ||||
5592 | uint64_t TableSize = | ||||
5593 | Array->getInitializer()->getType()->getArrayNumElements(); | ||||
5594 | if (TableSize > (1ULL << (IT->getBitWidth() - 1))) | ||||
5595 | Index = Builder.CreateZExt( | ||||
5596 | Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1), | ||||
5597 | "switch.tableidx.zext"); | ||||
5598 | |||||
5599 | Value *GEPIndices[] = {Builder.getInt32(0), Index}; | ||||
5600 | Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array, | ||||
5601 | GEPIndices, "switch.gep"); | ||||
5602 | return Builder.CreateLoad( | ||||
5603 | cast<ArrayType>(Array->getValueType())->getElementType(), GEP, | ||||
5604 | "switch.load"); | ||||
5605 | } | ||||
5606 | } | ||||
5607 | llvm_unreachable("Unknown lookup table kind!")__builtin_unreachable(); | ||||
5608 | } | ||||
5609 | |||||
5610 | bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL, | ||||
5611 | uint64_t TableSize, | ||||
5612 | Type *ElementType) { | ||||
5613 | auto *IT = dyn_cast<IntegerType>(ElementType); | ||||
5614 | if (!IT) | ||||
5615 | return false; | ||||
5616 | // FIXME: If the type is wider than it needs to be, e.g. i8 but all values | ||||
5617 | // are <= 15, we could try to narrow the type. | ||||
5618 | |||||
5619 | // Avoid overflow, fitsInLegalInteger uses unsigned int for the width. | ||||
5620 | if (TableSize >= UINT_MAX(2147483647 *2U +1U) / IT->getBitWidth()) | ||||
5621 | return false; | ||||
5622 | return DL.fitsInLegalInteger(TableSize * IT->getBitWidth()); | ||||
5623 | } | ||||
5624 | |||||
5625 | /// Determine whether a lookup table should be built for this switch, based on | ||||
5626 | /// the number of cases, size of the table, and the types of the results. | ||||
5627 | static bool | ||||
5628 | ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize, | ||||
5629 | const TargetTransformInfo &TTI, const DataLayout &DL, | ||||
5630 | const SmallDenseMap<PHINode *, Type *> &ResultTypes) { | ||||
5631 | if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX0xffffffffffffffffULL / 10) | ||||
5632 | return false; // TableSize overflowed, or mul below might overflow. | ||||
5633 | |||||
5634 | bool AllTablesFitInRegister = true; | ||||
5635 | bool HasIllegalType = false; | ||||
5636 | for (const auto &I : ResultTypes) { | ||||
5637 | Type *Ty = I.second; | ||||
5638 | |||||
5639 | // Saturate this flag to true. | ||||
5640 | HasIllegalType = HasIllegalType || !TTI.isTypeLegal(Ty); | ||||
5641 | |||||
5642 | // Saturate this flag to false. | ||||
5643 | AllTablesFitInRegister = | ||||
5644 | AllTablesFitInRegister && | ||||
5645 | SwitchLookupTable::WouldFitInRegister(DL, TableSize, Ty); | ||||
5646 | |||||
5647 | // If both flags saturate, we're done. NOTE: This *only* works with | ||||
5648 | // saturating flags, and all flags have to saturate first due to the | ||||
5649 | // non-deterministic behavior of iterating over a dense map. | ||||
5650 | if (HasIllegalType && !AllTablesFitInRegister) | ||||
5651 | break; | ||||
5652 | } | ||||
5653 | |||||
5654 | // If each table would fit in a register, we should build it anyway. | ||||
5655 | if (AllTablesFitInRegister) | ||||
5656 | return true; | ||||
5657 | |||||
5658 | // Don't build a table that doesn't fit in-register if it has illegal types. | ||||
5659 | if (HasIllegalType) | ||||
5660 | return false; | ||||
5661 | |||||
5662 | // The table density should be at least 40%. This is the same criterion as for | ||||
5663 | // jump tables, see SelectionDAGBuilder::handleJTSwitchCase. | ||||
5664 | // FIXME: Find the best cut-off. | ||||
5665 | return SI->getNumCases() * 10 >= TableSize * 4; | ||||
5666 | } | ||||
5667 | |||||
5668 | /// Try to reuse the switch table index compare. Following pattern: | ||||
5669 | /// \code | ||||
5670 | /// if (idx < tablesize) | ||||
5671 | /// r = table[idx]; // table does not contain default_value | ||||
5672 | /// else | ||||
5673 | /// r = default_value; | ||||
5674 | /// if (r != default_value) | ||||
5675 | /// ... | ||||
5676 | /// \endcode | ||||
5677 | /// Is optimized to: | ||||
5678 | /// \code | ||||
5679 | /// cond = idx < tablesize; | ||||
5680 | /// if (cond) | ||||
5681 | /// r = table[idx]; | ||||
5682 | /// else | ||||
5683 | /// r = default_value; | ||||
5684 | /// if (cond) | ||||
5685 | /// ... | ||||
5686 | /// \endcode | ||||
5687 | /// Jump threading will then eliminate the second if(cond). | ||||
5688 | static void reuseTableCompare( | ||||
5689 | User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch, | ||||
5690 | Constant *DefaultValue, | ||||
5691 | const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) { | ||||
5692 | ICmpInst *CmpInst = dyn_cast<ICmpInst>(PhiUser); | ||||
5693 | if (!CmpInst) | ||||
5694 | return; | ||||
5695 | |||||
5696 | // We require that the compare is in the same block as the phi so that jump | ||||
5697 | // threading can do its work afterwards. | ||||
5698 | if (CmpInst->getParent() != PhiBlock) | ||||
5699 | return; | ||||
5700 | |||||
5701 | Constant *CmpOp1 = dyn_cast<Constant>(CmpInst->getOperand(1)); | ||||
5702 | if (!CmpOp1) | ||||
5703 | return; | ||||
5704 | |||||
5705 | Value *RangeCmp = RangeCheckBranch->getCondition(); | ||||
5706 | Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType()); | ||||
5707 | Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType()); | ||||
5708 | |||||
5709 | // Check if the compare with the default value is constant true or false. | ||||
5710 | Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(), | ||||
5711 | DefaultValue, CmpOp1, true); | ||||
5712 | if (DefaultConst != TrueConst && DefaultConst != FalseConst) | ||||
5713 | return; | ||||
5714 | |||||
5715 | // Check if the compare with the case values is distinct from the default | ||||
5716 | // compare result. | ||||
5717 | for (auto ValuePair : Values) { | ||||
5718 | Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(), | ||||
5719 | ValuePair.second, CmpOp1, true); | ||||
5720 | if (!CaseConst || CaseConst == DefaultConst || isa<UndefValue>(CaseConst)) | ||||
5721 | return; | ||||
5722 | assert((CaseConst == TrueConst || CaseConst == FalseConst) &&((void)0) | ||||
5723 | "Expect true or false as compare result.")((void)0); | ||||
5724 | } | ||||
5725 | |||||
5726 | // Check if the branch instruction dominates the phi node. It's a simple | ||||
5727 | // dominance check, but sufficient for our needs. | ||||
5728 | // Although this check is invariant in the calling loops, it's better to do it | ||||
5729 | // at this late stage. Practically we do it at most once for a switch. | ||||
5730 | BasicBlock *BranchBlock = RangeCheckBranch->getParent(); | ||||
5731 | for (BasicBlock *Pred : predecessors(PhiBlock)) { | ||||
5732 | if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock) | ||||
5733 | return; | ||||
5734 | } | ||||
5735 | |||||
5736 | if (DefaultConst == FalseConst) { | ||||
5737 | // The compare yields the same result. We can replace it. | ||||
5738 | CmpInst->replaceAllUsesWith(RangeCmp); | ||||
5739 | ++NumTableCmpReuses; | ||||
5740 | } else { | ||||
5741 | // The compare yields the same result, just inverted. We can replace it. | ||||
5742 | Value *InvertedTableCmp = BinaryOperator::CreateXor( | ||||
5743 | RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp", | ||||
5744 | RangeCheckBranch); | ||||
5745 | CmpInst->replaceAllUsesWith(InvertedTableCmp); | ||||
5746 | ++NumTableCmpReuses; | ||||
5747 | } | ||||
5748 | } | ||||
5749 | |||||
5750 | /// If the switch is only used to initialize one or more phi nodes in a common | ||||
5751 | /// successor block with different constant values, replace the switch with | ||||
5752 | /// lookup tables. | ||||
5753 | static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, | ||||
5754 | DomTreeUpdater *DTU, const DataLayout &DL, | ||||
5755 | const TargetTransformInfo &TTI) { | ||||
5756 | assert(SI->getNumCases() > 1 && "Degenerate switch?")((void)0); | ||||
5757 | |||||
5758 | BasicBlock *BB = SI->getParent(); | ||||
5759 | Function *Fn = BB->getParent(); | ||||
5760 | // Only build lookup table when we have a target that supports it or the | ||||
5761 | // attribute is not set. | ||||
5762 | if (!TTI.shouldBuildLookupTables() || | ||||
5763 | (Fn->getFnAttribute("no-jump-tables").getValueAsBool())) | ||||
5764 | return false; | ||||
5765 | |||||
5766 | // FIXME: If the switch is too sparse for a lookup table, perhaps we could | ||||
5767 | // split off a dense part and build a lookup table for that. | ||||
5768 | |||||
5769 | // FIXME: This creates arrays of GEPs to constant strings, which means each | ||||
5770 | // GEP needs a runtime relocation in PIC code. We should just build one big | ||||
5771 | // string and lookup indices into that. | ||||
5772 | |||||
5773 | // Ignore switches with less than three cases. Lookup tables will not make | ||||
5774 | // them faster, so we don't analyze them. | ||||
5775 | if (SI->getNumCases() < 3) | ||||
5776 | return false; | ||||
5777 | |||||
5778 | // Figure out the corresponding result for each case value and phi node in the | ||||
5779 | // common destination, as well as the min and max case values. | ||||
5780 | assert(!SI->cases().empty())((void)0); | ||||
5781 | SwitchInst::CaseIt CI = SI->case_begin(); | ||||
5782 | ConstantInt *MinCaseVal = CI->getCaseValue(); | ||||
5783 | ConstantInt *MaxCaseVal = CI->getCaseValue(); | ||||
5784 | |||||
5785 | BasicBlock *CommonDest = nullptr; | ||||
5786 | |||||
5787 | using ResultListTy = SmallVector<std::pair<ConstantInt *, Constant *>, 4>; | ||||
5788 | SmallDenseMap<PHINode *, ResultListTy> ResultLists; | ||||
5789 | |||||
5790 | SmallDenseMap<PHINode *, Constant *> DefaultResults; | ||||
5791 | SmallDenseMap<PHINode *, Type *> ResultTypes; | ||||
5792 | SmallVector<PHINode *, 4> PHIs; | ||||
5793 | |||||
5794 | for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) { | ||||
5795 | ConstantInt *CaseVal = CI->getCaseValue(); | ||||
5796 | if (CaseVal->getValue().slt(MinCaseVal->getValue())) | ||||
5797 | MinCaseVal = CaseVal; | ||||
5798 | if (CaseVal->getValue().sgt(MaxCaseVal->getValue())) | ||||
5799 | MaxCaseVal = CaseVal; | ||||
5800 | |||||
5801 | // Resulting value at phi nodes for this case value. | ||||
5802 | using ResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>; | ||||
5803 | ResultsTy Results; | ||||
5804 | if (!GetCaseResults(SI, CaseVal, CI->getCaseSuccessor(), &CommonDest, | ||||
5805 | Results, DL, TTI)) | ||||
5806 | return false; | ||||
5807 | |||||
5808 | // Append the result from this case to the list for each phi. | ||||
5809 | for (const auto &I : Results) { | ||||
5810 | PHINode *PHI = I.first; | ||||
5811 | Constant *Value = I.second; | ||||
5812 | if (!ResultLists.count(PHI)) | ||||
5813 | PHIs.push_back(PHI); | ||||
5814 | ResultLists[PHI].push_back(std::make_pair(CaseVal, Value)); | ||||
5815 | } | ||||
5816 | } | ||||
5817 | |||||
5818 | // Keep track of the result types. | ||||
5819 | for (PHINode *PHI : PHIs) { | ||||
5820 | ResultTypes[PHI] = ResultLists[PHI][0].second->getType(); | ||||
5821 | } | ||||
5822 | |||||
5823 | uint64_t NumResults = ResultLists[PHIs[0]].size(); | ||||
5824 | APInt RangeSpread = MaxCaseVal->getValue() - MinCaseVal->getValue(); | ||||
5825 | uint64_t TableSize = RangeSpread.getLimitedValue() + 1; | ||||
5826 | bool TableHasHoles = (NumResults < TableSize); | ||||
5827 | |||||
5828 | // If the table has holes, we need a constant result for the default case | ||||
5829 | // or a bitmask that fits in a register. | ||||
5830 | SmallVector<std::pair<PHINode *, Constant *>, 4> DefaultResultsList; | ||||
5831 | bool HasDefaultResults = | ||||
5832 | GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, | ||||
5833 | DefaultResultsList, DL, TTI); | ||||
5834 | |||||
5835 | bool NeedMask = (TableHasHoles && !HasDefaultResults); | ||||
5836 | if (NeedMask) { | ||||
5837 | // As an extra penalty for the validity test we require more cases. | ||||
5838 | if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark). | ||||
5839 | return false; | ||||
5840 | if (!DL.fitsInLegalInteger(TableSize)) | ||||
5841 | return false; | ||||
5842 | } | ||||
5843 | |||||
5844 | for (const auto &I : DefaultResultsList) { | ||||
5845 | PHINode *PHI = I.first; | ||||
5846 | Constant *Result = I.second; | ||||
5847 | DefaultResults[PHI] = Result; | ||||
5848 | } | ||||
5849 | |||||
5850 | if (!ShouldBuildLookupTable(SI, TableSize, TTI, DL, ResultTypes)) | ||||
5851 | return false; | ||||
5852 | |||||
5853 | std::vector<DominatorTree::UpdateType> Updates; | ||||
5854 | |||||
5855 | // Create the BB that does the lookups. | ||||
5856 | Module &Mod = *CommonDest->getParent()->getParent(); | ||||
5857 | BasicBlock *LookupBB = BasicBlock::Create( | ||||
5858 | Mod.getContext(), "switch.lookup", CommonDest->getParent(), CommonDest); | ||||
5859 | |||||
5860 | // Compute the table index value. | ||||
5861 | Builder.SetInsertPoint(SI); | ||||
5862 | Value *TableIndex; | ||||
5863 | if (MinCaseVal->isNullValue()) | ||||
5864 | TableIndex = SI->getCondition(); | ||||
5865 | else | ||||
5866 | TableIndex = Builder.CreateSub(SI->getCondition(), MinCaseVal, | ||||
5867 | "switch.tableidx"); | ||||
5868 | |||||
5869 | // Compute the maximum table size representable by the integer type we are | ||||
5870 | // switching upon. | ||||
5871 | unsigned CaseSize = MinCaseVal->getType()->getPrimitiveSizeInBits(); | ||||
5872 | uint64_t MaxTableSize = CaseSize > 63 ? UINT64_MAX0xffffffffffffffffULL : 1ULL << CaseSize; | ||||
5873 | assert(MaxTableSize >= TableSize &&((void)0) | ||||
5874 | "It is impossible for a switch to have more entries than the max "((void)0) | ||||
5875 | "representable value of its input integer type's size.")((void)0); | ||||
5876 | |||||
5877 | // If the default destination is unreachable, or if the lookup table covers | ||||
5878 | // all values of the conditional variable, branch directly to the lookup table | ||||
5879 | // BB. Otherwise, check that the condition is within the case range. | ||||
5880 | const bool DefaultIsReachable = | ||||
5881 | !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); | ||||
5882 | const bool GeneratingCoveredLookupTable = (MaxTableSize == TableSize); | ||||
5883 | BranchInst *RangeCheckBranch = nullptr; | ||||
5884 | |||||
5885 | if (!DefaultIsReachable || GeneratingCoveredLookupTable) { | ||||
5886 | Builder.CreateBr(LookupBB); | ||||
5887 | if (DTU) | ||||
5888 | Updates.push_back({DominatorTree::Insert, BB, LookupBB}); | ||||
5889 | // Note: We call removeProdecessor later since we need to be able to get the | ||||
5890 | // PHI value for the default case in case we're using a bit mask. | ||||
5891 | } else { | ||||
5892 | Value *Cmp = Builder.CreateICmpULT( | ||||
5893 | TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize)); | ||||
5894 | RangeCheckBranch = | ||||
5895 | Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest()); | ||||
5896 | if (DTU) | ||||
5897 | Updates.push_back({DominatorTree::Insert, BB, LookupBB}); | ||||
5898 | } | ||||
5899 | |||||
5900 | // Populate the BB that does the lookups. | ||||
5901 | Builder.SetInsertPoint(LookupBB); | ||||
5902 | |||||
5903 | if (NeedMask) { | ||||
5904 | // Before doing the lookup, we do the hole check. The LookupBB is therefore | ||||
5905 | // re-purposed to do the hole check, and we create a new LookupBB. | ||||
5906 | BasicBlock *MaskBB = LookupBB; | ||||
5907 | MaskBB->setName("switch.hole_check"); | ||||
5908 | LookupBB = BasicBlock::Create(Mod.getContext(), "switch.lookup", | ||||
5909 | CommonDest->getParent(), CommonDest); | ||||
5910 | |||||
5911 | // Make the mask's bitwidth at least 8-bit and a power-of-2 to avoid | ||||
5912 | // unnecessary illegal types. | ||||
5913 | uint64_t TableSizePowOf2 = NextPowerOf2(std::max(7ULL, TableSize - 1ULL)); | ||||
5914 | APInt MaskInt(TableSizePowOf2, 0); | ||||
5915 | APInt One(TableSizePowOf2, 1); | ||||
5916 | // Build bitmask; fill in a 1 bit for every case. | ||||
5917 | const ResultListTy &ResultList = ResultLists[PHIs[0]]; | ||||
5918 | for (size_t I = 0, E = ResultList.size(); I != E; ++I) { | ||||
5919 | uint64_t Idx = (ResultList[I].first->getValue() - MinCaseVal->getValue()) | ||||
5920 | .getLimitedValue(); | ||||
5921 | MaskInt |= One << Idx; | ||||
5922 | } | ||||
5923 | ConstantInt *TableMask = ConstantInt::get(Mod.getContext(), MaskInt); | ||||
5924 | |||||
5925 | // Get the TableIndex'th bit of the bitmask. | ||||
5926 | // If this bit is 0 (meaning hole) jump to the default destination, | ||||
5927 | // else continue with table lookup. | ||||
5928 | IntegerType *MapTy = TableMask->getType(); | ||||
5929 | Value *MaskIndex = | ||||
5930 | Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex"); | ||||
5931 | Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted"); | ||||
5932 | Value *LoBit = Builder.CreateTrunc( | ||||
5933 | Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit"); | ||||
5934 | Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest()); | ||||
5935 | if (DTU) { | ||||
5936 | Updates.push_back({DominatorTree::Insert, MaskBB, LookupBB}); | ||||
5937 | Updates.push_back({DominatorTree::Insert, MaskBB, SI->getDefaultDest()}); | ||||
5938 | } | ||||
5939 | Builder.SetInsertPoint(LookupBB); | ||||
5940 | AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, BB); | ||||
5941 | } | ||||
5942 | |||||
5943 | if (!DefaultIsReachable || GeneratingCoveredLookupTable) { | ||||
5944 | // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later, | ||||
5945 | // do not delete PHINodes here. | ||||
5946 | SI->getDefaultDest()->removePredecessor(BB, | ||||
5947 | /*KeepOneInputPHIs=*/true); | ||||
5948 | if (DTU) | ||||
5949 | Updates.push_back({DominatorTree::Delete, BB, SI->getDefaultDest()}); | ||||
5950 | } | ||||
5951 | |||||
5952 | for (PHINode *PHI : PHIs) { | ||||
5953 | const ResultListTy &ResultList = ResultLists[PHI]; | ||||
5954 | |||||
5955 | // If using a bitmask, use any value to fill the lookup table holes. | ||||
5956 | Constant *DV = NeedMask ? ResultLists[PHI][0].second : DefaultResults[PHI]; | ||||
5957 | StringRef FuncName = Fn->getName(); | ||||
5958 | SwitchLookupTable Table(Mod, TableSize, MinCaseVal, ResultList, DV, DL, | ||||
5959 | FuncName); | ||||
5960 | |||||
5961 | Value *Result = Table.BuildLookup(TableIndex, Builder); | ||||
5962 | |||||
5963 | // Do a small peephole optimization: re-use the switch table compare if | ||||
5964 | // possible. | ||||
5965 | if (!TableHasHoles && HasDefaultResults && RangeCheckBranch) { | ||||
5966 | BasicBlock *PhiBlock = PHI->getParent(); | ||||
5967 | // Search for compare instructions which use the phi. | ||||
5968 | for (auto *User : PHI->users()) { | ||||
5969 | reuseTableCompare(User, PhiBlock, RangeCheckBranch, DV, ResultList); | ||||
5970 | } | ||||
5971 | } | ||||
5972 | |||||
5973 | PHI->addIncoming(Result, LookupBB); | ||||
5974 | } | ||||
5975 | |||||
5976 | Builder.CreateBr(CommonDest); | ||||
5977 | if (DTU) | ||||
5978 | Updates.push_back({DominatorTree::Insert, LookupBB, CommonDest}); | ||||
5979 | |||||
5980 | // Remove the switch. | ||||
5981 | SmallPtrSet<BasicBlock *, 8> RemovedSuccessors; | ||||
5982 | for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) { | ||||
5983 | BasicBlock *Succ = SI->getSuccessor(i); | ||||
5984 | |||||
5985 | if (Succ == SI->getDefaultDest()) | ||||
5986 | continue; | ||||
5987 | Succ->removePredecessor(BB); | ||||
5988 | RemovedSuccessors.insert(Succ); | ||||
5989 | } | ||||
5990 | SI->eraseFromParent(); | ||||
5991 | |||||
5992 | if (DTU) { | ||||
5993 | for (BasicBlock *RemovedSuccessor : RemovedSuccessors) | ||||
5994 | Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor}); | ||||
5995 | DTU->applyUpdates(Updates); | ||||
5996 | } | ||||
5997 | |||||
5998 | ++NumLookupTables; | ||||
5999 | if (NeedMask) | ||||
6000 | ++NumLookupTablesHoles; | ||||
6001 | return true; | ||||
6002 | } | ||||
6003 | |||||
6004 | static bool isSwitchDense(ArrayRef<int64_t> Values) { | ||||
6005 | // See also SelectionDAGBuilder::isDense(), which this function was based on. | ||||
6006 | uint64_t Diff = (uint64_t)Values.back() - (uint64_t)Values.front(); | ||||
6007 | uint64_t Range = Diff + 1; | ||||
6008 | uint64_t NumCases = Values.size(); | ||||
6009 | // 40% is the default density for building a jump table in optsize/minsize mode. | ||||
6010 | uint64_t MinDensity = 40; | ||||
6011 | |||||
6012 | return NumCases * 100 >= Range * MinDensity; | ||||
6013 | } | ||||
6014 | |||||
6015 | /// Try to transform a switch that has "holes" in it to a contiguous sequence | ||||
6016 | /// of cases. | ||||
6017 | /// | ||||
6018 | /// A switch such as: switch(i) {case 5: case 9: case 13: case 17:} can be | ||||
6019 | /// range-reduced to: switch ((i-5) / 4) {case 0: case 1: case 2: case 3:}. | ||||
6020 | /// | ||||
6021 | /// This converts a sparse switch into a dense switch which allows better | ||||
6022 | /// lowering and could also allow transforming into a lookup table. | ||||
6023 | static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder, | ||||
6024 | const DataLayout &DL, | ||||
6025 | const TargetTransformInfo &TTI) { | ||||
6026 | auto *CondTy = cast<IntegerType>(SI->getCondition()->getType()); | ||||
6027 | if (CondTy->getIntegerBitWidth() > 64 || | ||||
6028 | !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth())) | ||||
6029 | return false; | ||||
6030 | // Only bother with this optimization if there are more than 3 switch cases; | ||||
6031 | // SDAG will only bother creating jump tables for 4 or more cases. | ||||
6032 | if (SI->getNumCases() < 4) | ||||
6033 | return false; | ||||
6034 | |||||
6035 | // This transform is agnostic to the signedness of the input or case values. We | ||||
6036 | // can treat the case values as signed or unsigned. We can optimize more common | ||||
6037 | // cases such as a sequence crossing zero {-4,0,4,8} if we interpret case values | ||||
6038 | // as signed. | ||||
6039 | SmallVector<int64_t,4> Values; | ||||
6040 | for (auto &C : SI->cases()) | ||||
6041 | Values.push_back(C.getCaseValue()->getValue().getSExtValue()); | ||||
6042 | llvm::sort(Values); | ||||
6043 | |||||
6044 | // If the switch is already dense, there's nothing useful to do here. | ||||
6045 | if (isSwitchDense(Values)) | ||||
6046 | return false; | ||||
6047 | |||||
6048 | // First, transform the values such that they start at zero and ascend. | ||||
6049 | int64_t Base = Values[0]; | ||||
6050 | for (auto &V : Values) | ||||
6051 | V -= (uint64_t)(Base); | ||||
6052 | |||||
6053 | // Now we have signed numbers that have been shifted so that, given enough | ||||
6054 | // precision, there are no negative values. Since the rest of the transform | ||||
6055 | // is bitwise only, we switch now to an unsigned representation. | ||||
6056 | |||||
6057 | // This transform can be done speculatively because it is so cheap - it | ||||
6058 | // results in a single rotate operation being inserted. | ||||
6059 | // FIXME: It's possible that optimizing a switch on powers of two might also | ||||
6060 | // be beneficial - flag values are often powers of two and we could use a CLZ | ||||
6061 | // as the key function. | ||||
6062 | |||||
6063 | // countTrailingZeros(0) returns 64. As Values is guaranteed to have more than | ||||
6064 | // one element and LLVM disallows duplicate cases, Shift is guaranteed to be | ||||
6065 | // less than 64. | ||||
6066 | unsigned Shift = 64; | ||||
6067 | for (auto &V : Values) | ||||
6068 | Shift = std::min(Shift, countTrailingZeros((uint64_t)V)); | ||||
6069 | assert(Shift < 64)((void)0); | ||||
6070 | if (Shift > 0) | ||||
6071 | for (auto &V : Values) | ||||
6072 | V = (int64_t)((uint64_t)V >> Shift); | ||||
6073 | |||||
6074 | if (!isSwitchDense(Values)) | ||||
6075 | // Transform didn't create a dense switch. | ||||
6076 | return false; | ||||
6077 | |||||
6078 | // The obvious transform is to shift the switch condition right and emit a | ||||
6079 | // check that the condition actually cleanly divided by GCD, i.e. | ||||
6080 | // C & (1 << Shift - 1) == 0 | ||||
6081 | // inserting a new CFG edge to handle the case where it didn't divide cleanly. | ||||
6082 | // | ||||
6083 | // A cheaper way of doing this is a simple ROTR(C, Shift). This performs the | ||||
6084 | // shift and puts the shifted-off bits in the uppermost bits. If any of these | ||||
6085 | // are nonzero then the switch condition will be very large and will hit the | ||||
6086 | // default case. | ||||
6087 | |||||
6088 | auto *Ty = cast<IntegerType>(SI->getCondition()->getType()); | ||||
6089 | Builder.SetInsertPoint(SI); | ||||
6090 | auto *ShiftC = ConstantInt::get(Ty, Shift); | ||||
6091 | auto *Sub = Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base)); | ||||
6092 | auto *LShr = Builder.CreateLShr(Sub, ShiftC); | ||||
6093 | auto *Shl = Builder.CreateShl(Sub, Ty->getBitWidth() - Shift); | ||||
6094 | auto *Rot = Builder.CreateOr(LShr, Shl); | ||||
6095 | SI->replaceUsesOfWith(SI->getCondition(), Rot); | ||||
6096 | |||||
6097 | for (auto Case : SI->cases()) { | ||||
6098 | auto *Orig = Case.getCaseValue(); | ||||
6099 | auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base); | ||||
6100 | Case.setValue( | ||||
6101 | cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(ShiftC->getValue())))); | ||||
6102 | } | ||||
6103 | return true; | ||||
6104 | } | ||||
6105 | |||||
6106 | bool SimplifyCFGOpt::simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) { | ||||
6107 | BasicBlock *BB = SI->getParent(); | ||||
6108 | |||||
6109 | if (isValueEqualityComparison(SI)) { | ||||
6110 | // If we only have one predecessor, and if it is a branch on this value, | ||||
6111 | // see if that predecessor totally determines the outcome of this switch. | ||||
6112 | if (BasicBlock *OnlyPred = BB->getSinglePredecessor()) | ||||
6113 | if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder)) | ||||
6114 | return requestResimplify(); | ||||
6115 | |||||
6116 | Value *Cond = SI->getCondition(); | ||||
6117 | if (SelectInst *Select = dyn_cast<SelectInst>(Cond)) | ||||
6118 | if (SimplifySwitchOnSelect(SI, Select)) | ||||
6119 | return requestResimplify(); | ||||
6120 | |||||
6121 | // If the block only contains the switch, see if we can fold the block | ||||
6122 | // away into any preds. | ||||
6123 | if (SI == &*BB->instructionsWithoutDebug().begin()) | ||||
6124 | if (FoldValueComparisonIntoPredecessors(SI, Builder)) | ||||
6125 | return requestResimplify(); | ||||
6126 | } | ||||
6127 | |||||
6128 | // Try to transform the switch into an icmp and a branch. | ||||
6129 | if (TurnSwitchRangeIntoICmp(SI, Builder)) | ||||
6130 | return requestResimplify(); | ||||
6131 | |||||
6132 | // Remove unreachable cases. | ||||
6133 | if (eliminateDeadSwitchCases(SI, DTU, Options.AC, DL)) | ||||
6134 | return requestResimplify(); | ||||
6135 | |||||
6136 | if (switchToSelect(SI, Builder, DTU, DL, TTI)) | ||||
6137 | return requestResimplify(); | ||||
6138 | |||||
6139 | if (Options.ForwardSwitchCondToPhi && ForwardSwitchConditionToPHI(SI)) | ||||
6140 | return requestResimplify(); | ||||
6141 | |||||
6142 | // The conversion from switch to lookup tables results in difficult-to-analyze | ||||
6143 | // code and makes pruning branches much harder. This is a problem if the | ||||
6144 | // switch expression itself can still be restricted as a result of inlining or | ||||
6145 | // CVP. Therefore, only apply this transformation during late stages of the | ||||
6146 | // optimisation pipeline. | ||||
6147 | if (Options.ConvertSwitchToLookupTable && | ||||
6148 | SwitchToLookupTable(SI, Builder, DTU, DL, TTI)) | ||||
6149 | return requestResimplify(); | ||||
6150 | |||||
6151 | if (ReduceSwitchRange(SI, Builder, DL, TTI)) | ||||
6152 | return requestResimplify(); | ||||
6153 | |||||
6154 | return false; | ||||
6155 | } | ||||
6156 | |||||
6157 | bool SimplifyCFGOpt::simplifyIndirectBr(IndirectBrInst *IBI) { | ||||
6158 | BasicBlock *BB = IBI->getParent(); | ||||
6159 | bool Changed = false; | ||||
6160 | |||||
6161 | // Eliminate redundant destinations. | ||||
6162 | SmallPtrSet<Value *, 8> Succs; | ||||
6163 | SmallPtrSet<BasicBlock *, 8> RemovedSuccs; | ||||
6164 | for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) { | ||||
6165 | BasicBlock *Dest = IBI->getDestination(i); | ||||
6166 | if (!Dest->hasAddressTaken() || !Succs.insert(Dest).second) { | ||||
6167 | if (!Dest->hasAddressTaken()) | ||||
6168 | RemovedSuccs.insert(Dest); | ||||
6169 | Dest->removePredecessor(BB); | ||||
6170 | IBI->removeDestination(i); | ||||
6171 | --i; | ||||
6172 | --e; | ||||
6173 | Changed = true; | ||||
6174 | } | ||||
6175 | } | ||||
6176 | |||||
6177 | if (DTU) { | ||||
6178 | std::vector<DominatorTree::UpdateType> Updates; | ||||
6179 | Updates.reserve(RemovedSuccs.size()); | ||||
6180 | for (auto *RemovedSucc : RemovedSuccs) | ||||
6181 | Updates.push_back({DominatorTree::Delete, BB, RemovedSucc}); | ||||
6182 | DTU->applyUpdates(Updates); | ||||
6183 | } | ||||
6184 | |||||
6185 | if (IBI->getNumDestinations() == 0) { | ||||
6186 | // If the indirectbr has no successors, change it to unreachable. | ||||
6187 | new UnreachableInst(IBI->getContext(), IBI); | ||||
6188 | EraseTerminatorAndDCECond(IBI); | ||||
6189 | return true; | ||||
6190 | } | ||||
6191 | |||||
6192 | if (IBI->getNumDestinations() == 1) { | ||||
6193 | // If the indirectbr has one successor, change it to a direct branch. | ||||
6194 | BranchInst::Create(IBI->getDestination(0), IBI); | ||||
6195 | EraseTerminatorAndDCECond(IBI); | ||||
6196 | return true; | ||||
6197 | } | ||||
6198 | |||||
6199 | if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) { | ||||
6200 | if (SimplifyIndirectBrOnSelect(IBI, SI)) | ||||
6201 | return requestResimplify(); | ||||
6202 | } | ||||
6203 | return Changed; | ||||
6204 | } | ||||
6205 | |||||
6206 | /// Given an block with only a single landing pad and a unconditional branch | ||||
6207 | /// try to find another basic block which this one can be merged with. This | ||||
6208 | /// handles cases where we have multiple invokes with unique landing pads, but | ||||
6209 | /// a shared handler. | ||||
6210 | /// | ||||
6211 | /// We specifically choose to not worry about merging non-empty blocks | ||||
6212 | /// here. That is a PRE/scheduling problem and is best solved elsewhere. In | ||||
6213 | /// practice, the optimizer produces empty landing pad blocks quite frequently | ||||
6214 | /// when dealing with exception dense code. (see: instcombine, gvn, if-else | ||||
6215 | /// sinking in this file) | ||||
6216 | /// | ||||
6217 | /// This is primarily a code size optimization. We need to avoid performing | ||||
6218 | /// any transform which might inhibit optimization (such as our ability to | ||||
6219 | /// specialize a particular handler via tail commoning). We do this by not | ||||
6220 | /// merging any blocks which require us to introduce a phi. Since the same | ||||
6221 | /// values are flowing through both blocks, we don't lose any ability to | ||||
6222 | /// specialize. If anything, we make such specialization more likely. | ||||
6223 | /// | ||||
6224 | /// TODO - This transformation could remove entries from a phi in the target | ||||
6225 | /// block when the inputs in the phi are the same for the two blocks being | ||||
6226 | /// merged. In some cases, this could result in removal of the PHI entirely. | ||||
6227 | static bool TryToMergeLandingPad(LandingPadInst *LPad, BranchInst *BI, | ||||
6228 | BasicBlock *BB, DomTreeUpdater *DTU) { | ||||
6229 | auto Succ = BB->getUniqueSuccessor(); | ||||
6230 | assert(Succ)((void)0); | ||||
6231 | // If there's a phi in the successor block, we'd likely have to introduce | ||||
6232 | // a phi into the merged landing pad block. | ||||
6233 | if (isa<PHINode>(*Succ->begin())) | ||||
6234 | return false; | ||||
6235 | |||||
6236 | for (BasicBlock *OtherPred : predecessors(Succ)) { | ||||
6237 | if (BB == OtherPred) | ||||
6238 | continue; | ||||
6239 | BasicBlock::iterator I = OtherPred->begin(); | ||||
6240 | LandingPadInst *LPad2 = dyn_cast<LandingPadInst>(I); | ||||
6241 | if (!LPad2 || !LPad2->isIdenticalTo(LPad)) | ||||
6242 | continue; | ||||
6243 | for (++I; isa<DbgInfoIntrinsic>(I); ++I) | ||||
6244 | ; | ||||
6245 | BranchInst *BI2 = dyn_cast<BranchInst>(I); | ||||
6246 | if (!BI2 || !BI2->isIdenticalTo(BI)) | ||||
6247 | continue; | ||||
6248 | |||||
6249 | std::vector<DominatorTree::UpdateType> Updates; | ||||
6250 | |||||
6251 | // We've found an identical block. Update our predecessors to take that | ||||
6252 | // path instead and make ourselves dead. | ||||
6253 | SmallPtrSet<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB)); | ||||
6254 | for (BasicBlock *Pred : Preds) { | ||||
6255 | InvokeInst *II = cast<InvokeInst>(Pred->getTerminator()); | ||||
6256 | assert(II->getNormalDest() != BB && II->getUnwindDest() == BB &&((void)0) | ||||
6257 | "unexpected successor")((void)0); | ||||
6258 | II->setUnwindDest(OtherPred); | ||||
6259 | if (DTU) { | ||||
6260 | Updates.push_back({DominatorTree::Insert, Pred, OtherPred}); | ||||
6261 | Updates.push_back({DominatorTree::Delete, Pred, BB}); | ||||
6262 | } | ||||
6263 | } | ||||
6264 | |||||
6265 | // The debug info in OtherPred doesn't cover the merged control flow that | ||||
6266 | // used to go through BB. We need to delete it or update it. | ||||
6267 | for (auto I = OtherPred->begin(), E = OtherPred->end(); I != E;) { | ||||
6268 | Instruction &Inst = *I; | ||||
6269 | I++; | ||||
6270 | if (isa<DbgInfoIntrinsic>(Inst)) | ||||
6271 | Inst.eraseFromParent(); | ||||
6272 | } | ||||
6273 | |||||
6274 | SmallPtrSet<BasicBlock *, 16> Succs(succ_begin(BB), succ_end(BB)); | ||||
6275 | for (BasicBlock *Succ : Succs) { | ||||
6276 | Succ->removePredecessor(BB); | ||||
6277 | if (DTU) | ||||
6278 | Updates.push_back({DominatorTree::Delete, BB, Succ}); | ||||
6279 | } | ||||
6280 | |||||
6281 | IRBuilder<> Builder(BI); | ||||
6282 | Builder.CreateUnreachable(); | ||||
6283 | BI->eraseFromParent(); | ||||
6284 | if (DTU) | ||||
6285 | DTU->applyUpdates(Updates); | ||||
6286 | return true; | ||||
6287 | } | ||||
6288 | return false; | ||||
6289 | } | ||||
6290 | |||||
6291 | bool SimplifyCFGOpt::simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder) { | ||||
6292 | return Branch->isUnconditional() ? simplifyUncondBranch(Branch, Builder) | ||||
6293 | : simplifyCondBranch(Branch, Builder); | ||||
6294 | } | ||||
6295 | |||||
6296 | bool SimplifyCFGOpt::simplifyUncondBranch(BranchInst *BI, | ||||
6297 | IRBuilder<> &Builder) { | ||||
6298 | BasicBlock *BB = BI->getParent(); | ||||
6299 | BasicBlock *Succ = BI->getSuccessor(0); | ||||
6300 | |||||
6301 | // If the Terminator is the only non-phi instruction, simplify the block. | ||||
6302 | // If LoopHeader is provided, check if the block or its successor is a loop | ||||
6303 | // header. (This is for early invocations before loop simplify and | ||||
6304 | // vectorization to keep canonical loop forms for nested loops. These blocks | ||||
6305 | // can be eliminated when the pass is invoked later in the back-end.) | ||||
6306 | // Note that if BB has only one predecessor then we do not introduce new | ||||
6307 | // backedge, so we can eliminate BB. | ||||
6308 | bool NeedCanonicalLoop = | ||||
6309 | Options.NeedCanonicalLoop && | ||||
6310 | (!LoopHeaders.empty() && BB->hasNPredecessorsOrMore(2) && | ||||
6311 | (is_contained(LoopHeaders, BB) || is_contained(LoopHeaders, Succ))); | ||||
6312 | BasicBlock::iterator I = BB->getFirstNonPHIOrDbg(true)->getIterator(); | ||||
6313 | if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() && | ||||
6314 | !NeedCanonicalLoop && TryToSimplifyUncondBranchFromEmptyBlock(BB, DTU)) | ||||
6315 | return true; | ||||
6316 | |||||
6317 | // If the only instruction in the block is a seteq/setne comparison against a | ||||
6318 | // constant, try to simplify the block. | ||||
6319 | if (ICmpInst *ICI = dyn_cast<ICmpInst>(I)) | ||||
6320 | if (ICI->isEquality() && isa<ConstantInt>(ICI->getOperand(1))) { | ||||
6321 | for (++I; isa<DbgInfoIntrinsic>(I); ++I) | ||||
6322 | ; | ||||
6323 | if (I->isTerminator() && | ||||
6324 | tryToSimplifyUncondBranchWithICmpInIt(ICI, Builder)) | ||||
6325 | return true; | ||||
6326 | } | ||||
6327 | |||||
6328 | // See if we can merge an empty landing pad block with another which is | ||||
6329 | // equivalent. | ||||
6330 | if (LandingPadInst *LPad = dyn_cast<LandingPadInst>(I)) { | ||||
6331 | for (++I; isa<DbgInfoIntrinsic>(I); ++I) | ||||
6332 | ; | ||||
6333 | if (I->isTerminator() && TryToMergeLandingPad(LPad, BI, BB, DTU)) | ||||
6334 | return true; | ||||
6335 | } | ||||
6336 | |||||
6337 | // If this basic block is ONLY a compare and a branch, and if a predecessor | ||||
6338 | // branches to us and our successor, fold the comparison into the | ||||
6339 | // predecessor and use logical operations to update the incoming value | ||||
6340 | // for PHI nodes in common successor. | ||||
6341 | if (FoldBranchToCommonDest(BI, DTU, /*MSSAU=*/nullptr, &TTI, | ||||
6342 | Options.BonusInstThreshold)) | ||||
6343 | return requestResimplify(); | ||||
6344 | return false; | ||||
6345 | } | ||||
6346 | |||||
6347 | static BasicBlock *allPredecessorsComeFromSameSource(BasicBlock *BB) { | ||||
6348 | BasicBlock *PredPred = nullptr; | ||||
6349 | for (auto *P : predecessors(BB)) { | ||||
6350 | BasicBlock *PPred = P->getSinglePredecessor(); | ||||
6351 | if (!PPred || (PredPred && PredPred != PPred)) | ||||
6352 | return nullptr; | ||||
6353 | PredPred = PPred; | ||||
6354 | } | ||||
6355 | return PredPred; | ||||
6356 | } | ||||
6357 | |||||
6358 | bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) { | ||||
6359 | BasicBlock *BB = BI->getParent(); | ||||
6360 | if (!Options.SimplifyCondBranch) | ||||
6361 | return false; | ||||
6362 | |||||
6363 | // Conditional branch | ||||
6364 | if (isValueEqualityComparison(BI)) { | ||||
6365 | // If we only have one predecessor, and if it is a branch on this value, | ||||
6366 | // see if that predecessor totally determines the outcome of this | ||||
6367 | // switch. | ||||
6368 | if (BasicBlock *OnlyPred = BB->getSinglePredecessor()) | ||||
6369 | if (SimplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder)) | ||||
6370 | return requestResimplify(); | ||||
6371 | |||||
6372 | // This block must be empty, except for the setcond inst, if it exists. | ||||
6373 | // Ignore dbg and pseudo intrinsics. | ||||
6374 | auto I = BB->instructionsWithoutDebug(true).begin(); | ||||
6375 | if (&*I == BI) { | ||||
6376 | if (FoldValueComparisonIntoPredecessors(BI, Builder)) | ||||
6377 | return requestResimplify(); | ||||
6378 | } else if (&*I == cast<Instruction>(BI->getCondition())) { | ||||
6379 | ++I; | ||||
6380 | if (&*I == BI && FoldValueComparisonIntoPredecessors(BI, Builder)) | ||||
6381 | return requestResimplify(); | ||||
6382 | } | ||||
6383 | } | ||||
6384 | |||||
6385 | // Try to turn "br (X == 0 | X == 1), T, F" into a switch instruction. | ||||
6386 | if (SimplifyBranchOnICmpChain(BI, Builder, DL)) | ||||
6387 | return true; | ||||
6388 | |||||
6389 | // If this basic block has dominating predecessor blocks and the dominating | ||||
6390 | // blocks' conditions imply BI's condition, we know the direction of BI. | ||||
6391 | Optional<bool> Imp = isImpliedByDomCondition(BI->getCondition(), BI, DL); | ||||
6392 | if (Imp) { | ||||
6393 | // Turn this into a branch on constant. | ||||
6394 | auto *OldCond = BI->getCondition(); | ||||
6395 | ConstantInt *TorF = *Imp ? ConstantInt::getTrue(BB->getContext()) | ||||
6396 | : ConstantInt::getFalse(BB->getContext()); | ||||
6397 | BI->setCondition(TorF); | ||||
6398 | RecursivelyDeleteTriviallyDeadInstructions(OldCond); | ||||
6399 | return requestResimplify(); | ||||
6400 | } | ||||
6401 | |||||
6402 | // If this basic block is ONLY a compare and a branch, and if a predecessor | ||||
6403 | // branches to us and one of our successors, fold the comparison into the | ||||
6404 | // predecessor and use logical operations to pick the right destination. | ||||
6405 | if (FoldBranchToCommonDest(BI, DTU, /*MSSAU=*/nullptr, &TTI, | ||||
6406 | Options.BonusInstThreshold)) | ||||
6407 | return requestResimplify(); | ||||
6408 | |||||
6409 | // We have a conditional branch to two blocks that are only reachable | ||||
6410 | // from BI. We know that the condbr dominates the two blocks, so see if | ||||
6411 | // there is any identical code in the "then" and "else" blocks. If so, we | ||||
6412 | // can hoist it up to the branching block. | ||||
6413 | if (BI->getSuccessor(0)->getSinglePredecessor()) { | ||||
6414 | if (BI->getSuccessor(1)->getSinglePredecessor()) { | ||||
6415 | if (HoistCommon && | ||||
6416 | HoistThenElseCodeToIf(BI, TTI, !Options.HoistCommonInsts)) | ||||
6417 | return requestResimplify(); | ||||
6418 | } else { | ||||
6419 | // If Successor #1 has multiple preds, we may be able to conditionally | ||||
6420 | // execute Successor #0 if it branches to Successor #1. | ||||
6421 | Instruction *Succ0TI = BI->getSuccessor(0)->getTerminator(); | ||||
6422 | if (Succ0TI->getNumSuccessors() == 1 && | ||||
6423 | Succ0TI->getSuccessor(0) == BI->getSuccessor(1)) | ||||
6424 | if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0), TTI)) | ||||
6425 | return requestResimplify(); | ||||
6426 | } | ||||
6427 | } else if (BI->getSuccessor(1)->getSinglePredecessor()) { | ||||
6428 | // If Successor #0 has multiple preds, we may be able to conditionally | ||||
6429 | // execute Successor #1 if it branches to Successor #0. | ||||
6430 | Instruction *Succ1TI = BI->getSuccessor(1)->getTerminator(); | ||||
6431 | if (Succ1TI->getNumSuccessors() == 1 && | ||||
6432 | Succ1TI->getSuccessor(0) == BI->getSuccessor(0)) | ||||
6433 | if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1), TTI)) | ||||
6434 | return requestResimplify(); | ||||
6435 | } | ||||
6436 | |||||
6437 | // If this is a branch on a phi node in the current block, thread control | ||||
6438 | // through this block if any PHI node entries are constants. | ||||
6439 | if (PHINode *PN = dyn_cast<PHINode>(BI->getCondition())) | ||||
6440 | if (PN->getParent() == BI->getParent()) | ||||
6441 | if (FoldCondBranchOnPHI(BI, DTU, DL, Options.AC)) | ||||
6442 | return requestResimplify(); | ||||
6443 | |||||
6444 | // Scan predecessor blocks for conditional branches. | ||||
6445 | for (BasicBlock *Pred : predecessors(BB)) | ||||
6446 | if (BranchInst *PBI = dyn_cast<BranchInst>(Pred->getTerminator())) | ||||
6447 | if (PBI != BI && PBI->isConditional()) | ||||
6448 | if (SimplifyCondBranchToCondBranch(PBI, BI, DTU, DL, TTI)) | ||||
6449 | return requestResimplify(); | ||||
6450 | |||||
6451 | // Look for diamond patterns. | ||||
6452 | if (MergeCondStores) | ||||
6453 | if (BasicBlock *PrevBB = allPredecessorsComeFromSameSource(BB)) | ||||
6454 | if (BranchInst *PBI = dyn_cast<BranchInst>(PrevBB->getTerminator())) | ||||
6455 | if (PBI != BI && PBI->isConditional()) | ||||
6456 | if (mergeConditionalStores(PBI, BI, DTU, DL, TTI)) | ||||
6457 | return requestResimplify(); | ||||
6458 | |||||
6459 | return false; | ||||
6460 | } | ||||
6461 | |||||
6462 | /// Check if passing a value to an instruction will cause undefined behavior. | ||||
6463 | static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified) { | ||||
6464 | Constant *C = dyn_cast<Constant>(V); | ||||
6465 | if (!C) | ||||
6466 | return false; | ||||
6467 | |||||
6468 | if (I->use_empty()) | ||||
6469 | return false; | ||||
6470 | |||||
6471 | if (C->isNullValue() || isa<UndefValue>(C)) { | ||||
6472 | // Only look at the first use, avoid hurting compile time with long uselists | ||||
6473 | User *Use = *I->user_begin(); | ||||
6474 | |||||
6475 | // Now make sure that there are no instructions in between that can alter | ||||
6476 | // control flow (eg. calls) | ||||
6477 | for (BasicBlock::iterator | ||||
6478 | i = ++BasicBlock::iterator(I), | ||||
6479 | UI = BasicBlock::iterator(dyn_cast<Instruction>(Use)); | ||||
6480 | i != UI; ++i) { | ||||
6481 | if (i == I->getParent()->end()) | ||||
6482 | return false; | ||||
6483 | if (!isGuaranteedToTransferExecutionToSuccessor(&*i)) | ||||
6484 | return false; | ||||
6485 | } | ||||
6486 | |||||
6487 | // Look through GEPs. A load from a GEP derived from NULL is still undefined | ||||
6488 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Use)) | ||||
6489 | if (GEP->getPointerOperand() == I) { | ||||
6490 | if (!GEP->isInBounds() || !GEP->hasAllZeroIndices()) | ||||
6491 | PtrValueMayBeModified = true; | ||||
6492 | return passingValueIsAlwaysUndefined(V, GEP, PtrValueMayBeModified); | ||||
6493 | } | ||||
6494 | |||||
6495 | // Look through bitcasts. | ||||
6496 | if (BitCastInst *BC = dyn_cast<BitCastInst>(Use)) | ||||
6497 | return passingValueIsAlwaysUndefined(V, BC, PtrValueMayBeModified); | ||||
6498 | |||||
6499 | // Load from null is undefined. | ||||
6500 | if (LoadInst *LI = dyn_cast<LoadInst>(Use)) | ||||
6501 | if (!LI->isVolatile()) | ||||
6502 | return !NullPointerIsDefined(LI->getFunction(), | ||||
6503 | LI->getPointerAddressSpace()); | ||||
6504 | |||||
6505 | // Store to null is undefined. | ||||
6506 | if (StoreInst *SI = dyn_cast<StoreInst>(Use)) | ||||
6507 | if (!SI->isVolatile()) | ||||
6508 | return (!NullPointerIsDefined(SI->getFunction(), | ||||
6509 | SI->getPointerAddressSpace())) && | ||||
6510 | SI->getPointerOperand() == I; | ||||
6511 | |||||
6512 | if (auto *CB = dyn_cast<CallBase>(Use)) { | ||||
6513 | if (C->isNullValue() && NullPointerIsDefined(CB->getFunction())) | ||||
6514 | return false; | ||||
6515 | // A call to null is undefined. | ||||
6516 | if (CB->getCalledOperand() == I) | ||||
6517 | return true; | ||||
6518 | |||||
6519 | if (C->isNullValue()) { | ||||
6520 | for (const llvm::Use &Arg : CB->args()) | ||||
6521 | if (Arg == I) { | ||||
6522 | unsigned ArgIdx = CB->getArgOperandNo(&Arg); | ||||
6523 | if (CB->isPassingUndefUB(ArgIdx) && | ||||
6524 | CB->paramHasAttr(ArgIdx, Attribute::NonNull)) { | ||||
6525 | // Passing null to a nonnnull+noundef argument is undefined. | ||||
6526 | return !PtrValueMayBeModified; | ||||
6527 | } | ||||
6528 | } | ||||
6529 | } else if (isa<UndefValue>(C)) { | ||||
6530 | // Passing undef to a noundef argument is undefined. | ||||
6531 | for (const llvm::Use &Arg : CB->args()) | ||||
6532 | if (Arg == I) { | ||||
6533 | unsigned ArgIdx = CB->getArgOperandNo(&Arg); | ||||
6534 | if (CB->isPassingUndefUB(ArgIdx)) { | ||||
6535 | // Passing undef to a noundef argument is undefined. | ||||
6536 | return true; | ||||
6537 | } | ||||
6538 | } | ||||
6539 | } | ||||
6540 | } | ||||
6541 | } | ||||
6542 | return false; | ||||
6543 | } | ||||
6544 | |||||
6545 | /// If BB has an incoming value that will always trigger undefined behavior | ||||
6546 | /// (eg. null pointer dereference), remove the branch leading here. | ||||
6547 | static bool removeUndefIntroducingPredecessor(BasicBlock *BB, | ||||
6548 | DomTreeUpdater *DTU) { | ||||
6549 | for (PHINode &PHI : BB->phis()) | ||||
6550 | for (unsigned i = 0, e = PHI.getNumIncomingValues(); i != e; ++i) | ||||
6551 | if (passingValueIsAlwaysUndefined(PHI.getIncomingValue(i), &PHI)) { | ||||
6552 | BasicBlock *Predecessor = PHI.getIncomingBlock(i); | ||||
6553 | Instruction *T = Predecessor->getTerminator(); | ||||
6554 | IRBuilder<> Builder(T); | ||||
6555 | if (BranchInst *BI = dyn_cast<BranchInst>(T)) { | ||||
6556 | BB->removePredecessor(Predecessor); | ||||
6557 | // Turn uncoditional branches into unreachables and remove the dead | ||||
6558 | // destination from conditional branches. | ||||
6559 | if (BI->isUnconditional()) | ||||
6560 | Builder.CreateUnreachable(); | ||||
6561 | else | ||||
6562 | Builder.CreateBr(BI->getSuccessor(0) == BB ? BI->getSuccessor(1) | ||||
6563 | : BI->getSuccessor(0)); | ||||
6564 | BI->eraseFromParent(); | ||||
6565 | if (DTU) | ||||
6566 | DTU->applyUpdates({{DominatorTree::Delete, Predecessor, BB}}); | ||||
6567 | return true; | ||||
6568 | } | ||||
6569 | // TODO: SwitchInst. | ||||
6570 | } | ||||
6571 | |||||
6572 | return false; | ||||
6573 | } | ||||
6574 | |||||
6575 | bool SimplifyCFGOpt::simplifyOnceImpl(BasicBlock *BB) { | ||||
6576 | bool Changed = false; | ||||
6577 | |||||
6578 | assert(BB && BB->getParent() && "Block not embedded in function!")((void)0); | ||||
6579 | assert(BB->getTerminator() && "Degenerate basic block encountered!")((void)0); | ||||
6580 | |||||
6581 | // Remove basic blocks that have no predecessors (except the entry block)... | ||||
6582 | // or that just have themself as a predecessor. These are unreachable. | ||||
6583 | if ((pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) || | ||||
6584 | BB->getSinglePredecessor() == BB) { | ||||
6585 | LLVM_DEBUG(dbgs() << "Removing BB: \n" << *BB)do { } while (false); | ||||
6586 | DeleteDeadBlock(BB, DTU); | ||||
6587 | return true; | ||||
6588 | } | ||||
6589 | |||||
6590 | // Check to see if we can constant propagate this terminator instruction | ||||
6591 | // away... | ||||
6592 | Changed |= ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true, | ||||
6593 | /*TLI=*/nullptr, DTU); | ||||
6594 | |||||
6595 | // Check for and eliminate duplicate PHI nodes in this block. | ||||
6596 | Changed |= EliminateDuplicatePHINodes(BB); | ||||
6597 | |||||
6598 | // Check for and remove branches that will always cause undefined behavior. | ||||
6599 | Changed |= removeUndefIntroducingPredecessor(BB, DTU); | ||||
6600 | |||||
6601 | // Merge basic blocks into their predecessor if there is only one distinct | ||||
6602 | // pred, and if there is only one distinct successor of the predecessor, and | ||||
6603 | // if there are no PHI nodes. | ||||
6604 | if (MergeBlockIntoPredecessor(BB, DTU)) | ||||
6605 | return true; | ||||
6606 | |||||
6607 | if (SinkCommon && Options.SinkCommonInsts) | ||||
6608 | if (SinkCommonCodeFromPredecessors(BB, DTU)) { | ||||
6609 | // SinkCommonCodeFromPredecessors() does not automatically CSE PHI's, | ||||
6610 | // so we may now how duplicate PHI's. | ||||
6611 | // Let's rerun EliminateDuplicatePHINodes() first, | ||||
6612 | // before FoldTwoEntryPHINode() potentially converts them into select's, | ||||
6613 | // after which we'd need a whole EarlyCSE pass run to cleanup them. | ||||
6614 | return true; | ||||
6615 | } | ||||
6616 | |||||
6617 | IRBuilder<> Builder(BB); | ||||
6618 | |||||
6619 | if (Options.FoldTwoEntryPHINode) { | ||||
6620 | // If there is a trivial two-entry PHI node in this basic block, and we can | ||||
6621 | // eliminate it, do so now. | ||||
6622 | if (auto *PN = dyn_cast<PHINode>(BB->begin())) | ||||
6623 | if (PN->getNumIncomingValues() == 2) | ||||
6624 | Changed |= FoldTwoEntryPHINode(PN, TTI, DTU, DL); | ||||
6625 | } | ||||
6626 | |||||
6627 | Instruction *Terminator = BB->getTerminator(); | ||||
6628 | Builder.SetInsertPoint(Terminator); | ||||
6629 | switch (Terminator->getOpcode()) { | ||||
6630 | case Instruction::Br: | ||||
6631 | Changed |= simplifyBranch(cast<BranchInst>(Terminator), Builder); | ||||
6632 | break; | ||||
6633 | case Instruction::Resume: | ||||
6634 | Changed |= simplifyResume(cast<ResumeInst>(Terminator), Builder); | ||||
6635 | break; | ||||
6636 | case Instruction::CleanupRet: | ||||
6637 | Changed |= simplifyCleanupReturn(cast<CleanupReturnInst>(Terminator)); | ||||
6638 | break; | ||||
6639 | case Instruction::Switch: | ||||
6640 | Changed |= simplifySwitch(cast<SwitchInst>(Terminator), Builder); | ||||
6641 | break; | ||||
6642 | case Instruction::Unreachable: | ||||
6643 | Changed |= simplifyUnreachable(cast<UnreachableInst>(Terminator)); | ||||
6644 | break; | ||||
6645 | case Instruction::IndirectBr: | ||||
6646 | Changed |= simplifyIndirectBr(cast<IndirectBrInst>(Terminator)); | ||||
6647 | break; | ||||
6648 | } | ||||
6649 | |||||
6650 | return Changed; | ||||
6651 | } | ||||
6652 | |||||
6653 | bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) { | ||||
6654 | bool Changed = simplifyOnceImpl(BB); | ||||
6655 | |||||
6656 | return Changed; | ||||
6657 | } | ||||
6658 | |||||
6659 | bool SimplifyCFGOpt::run(BasicBlock *BB) { | ||||
6660 | bool Changed = false; | ||||
6661 | |||||
6662 | // Repeated simplify BB as long as resimplification is requested. | ||||
6663 | do { | ||||
6664 | Resimplify = false; | ||||
6665 | |||||
6666 | // Perform one round of simplifcation. Resimplify flag will be set if | ||||
6667 | // another iteration is requested. | ||||
6668 | Changed |= simplifyOnce(BB); | ||||
6669 | } while (Resimplify); | ||||
6670 | |||||
6671 | return Changed; | ||||
6672 | } | ||||
6673 | |||||
6674 | bool llvm::simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI, | ||||
6675 | DomTreeUpdater *DTU, const SimplifyCFGOptions &Options, | ||||
6676 | ArrayRef<WeakVH> LoopHeaders) { | ||||
6677 | return SimplifyCFGOpt(TTI, DTU, BB->getModule()->getDataLayout(), LoopHeaders, | ||||
| |||||
6678 | Options) | ||||
6679 | .run(BB); | ||||
6680 | } |
1 | //===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file defines the SmallVector class. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #ifndef LLVM_ADT_SMALLVECTOR_H |
14 | #define LLVM_ADT_SMALLVECTOR_H |
15 | |
16 | #include "llvm/ADT/iterator_range.h" |
17 | #include "llvm/Support/Compiler.h" |
18 | #include "llvm/Support/ErrorHandling.h" |
19 | #include "llvm/Support/MemAlloc.h" |
20 | #include "llvm/Support/type_traits.h" |
21 | #include <algorithm> |
22 | #include <cassert> |
23 | #include <cstddef> |
24 | #include <cstdlib> |
25 | #include <cstring> |
26 | #include <functional> |
27 | #include <initializer_list> |
28 | #include <iterator> |
29 | #include <limits> |
30 | #include <memory> |
31 | #include <new> |
32 | #include <type_traits> |
33 | #include <utility> |
34 | |
35 | namespace llvm { |
36 | |
37 | /// This is all the stuff common to all SmallVectors. |
38 | /// |
39 | /// The template parameter specifies the type which should be used to hold the |
40 | /// Size and Capacity of the SmallVector, so it can be adjusted. |
41 | /// Using 32 bit size is desirable to shrink the size of the SmallVector. |
42 | /// Using 64 bit size is desirable for cases like SmallVector<char>, where a |
43 | /// 32 bit size would limit the vector to ~4GB. SmallVectors are used for |
44 | /// buffering bitcode output - which can exceed 4GB. |
45 | template <class Size_T> class SmallVectorBase { |
46 | protected: |
47 | void *BeginX; |
48 | Size_T Size = 0, Capacity; |
49 | |
50 | /// The maximum value of the Size_T used. |
51 | static constexpr size_t SizeTypeMax() { |
52 | return std::numeric_limits<Size_T>::max(); |
53 | } |
54 | |
55 | SmallVectorBase() = delete; |
56 | SmallVectorBase(void *FirstEl, size_t TotalCapacity) |
57 | : BeginX(FirstEl), Capacity(TotalCapacity) {} |
58 | |
59 | /// This is a helper for \a grow() that's out of line to reduce code |
60 | /// duplication. This function will report a fatal error if it can't grow at |
61 | /// least to \p MinSize. |
62 | void *mallocForGrow(size_t MinSize, size_t TSize, size_t &NewCapacity); |
63 | |
64 | /// This is an implementation of the grow() method which only works |
65 | /// on POD-like data types and is out of line to reduce code duplication. |
66 | /// This function will report a fatal error if it cannot increase capacity. |
67 | void grow_pod(void *FirstEl, size_t MinSize, size_t TSize); |
68 | |
69 | public: |
70 | size_t size() const { return Size; } |
71 | size_t capacity() const { return Capacity; } |
72 | |
73 | LLVM_NODISCARD[[clang::warn_unused_result]] bool empty() const { return !Size; } |
74 | |
75 | /// Set the array size to \p N, which the current array must have enough |
76 | /// capacity for. |
77 | /// |
78 | /// This does not construct or destroy any elements in the vector. |
79 | /// |
80 | /// Clients can use this in conjunction with capacity() to write past the end |
81 | /// of the buffer when they know that more elements are available, and only |
82 | /// update the size later. This avoids the cost of value initializing elements |
83 | /// which will only be overwritten. |
84 | void set_size(size_t N) { |
85 | assert(N <= capacity())((void)0); |
86 | Size = N; |
87 | } |
88 | }; |
89 | |
90 | template <class T> |
91 | using SmallVectorSizeType = |
92 | typename std::conditional<sizeof(T) < 4 && sizeof(void *) >= 8, uint64_t, |
93 | uint32_t>::type; |
94 | |
95 | /// Figure out the offset of the first element. |
96 | template <class T, typename = void> struct SmallVectorAlignmentAndSize { |
97 | alignas(SmallVectorBase<SmallVectorSizeType<T>>) char Base[sizeof( |
98 | SmallVectorBase<SmallVectorSizeType<T>>)]; |
99 | alignas(T) char FirstEl[sizeof(T)]; |
100 | }; |
101 | |
102 | /// This is the part of SmallVectorTemplateBase which does not depend on whether |
103 | /// the type T is a POD. The extra dummy template argument is used by ArrayRef |
104 | /// to avoid unnecessarily requiring T to be complete. |
105 | template <typename T, typename = void> |
106 | class SmallVectorTemplateCommon |
107 | : public SmallVectorBase<SmallVectorSizeType<T>> { |
108 | using Base = SmallVectorBase<SmallVectorSizeType<T>>; |
109 | |
110 | /// Find the address of the first element. For this pointer math to be valid |
111 | /// with small-size of 0 for T with lots of alignment, it's important that |
112 | /// SmallVectorStorage is properly-aligned even for small-size of 0. |
113 | void *getFirstEl() const { |
114 | return const_cast<void *>(reinterpret_cast<const void *>( |
115 | reinterpret_cast<const char *>(this) + |
116 | offsetof(SmallVectorAlignmentAndSize<T>, FirstEl)__builtin_offsetof(SmallVectorAlignmentAndSize<T>, FirstEl ))); |
117 | } |
118 | // Space after 'FirstEl' is clobbered, do not add any instance vars after it. |
119 | |
120 | protected: |
121 | SmallVectorTemplateCommon(size_t Size) : Base(getFirstEl(), Size) {} |
122 | |
123 | void grow_pod(size_t MinSize, size_t TSize) { |
124 | Base::grow_pod(getFirstEl(), MinSize, TSize); |
125 | } |
126 | |
127 | /// Return true if this is a smallvector which has not had dynamic |
128 | /// memory allocated for it. |
129 | bool isSmall() const { return this->BeginX == getFirstEl(); } |
130 | |
131 | /// Put this vector in a state of being small. |
132 | void resetToSmall() { |
133 | this->BeginX = getFirstEl(); |
134 | this->Size = this->Capacity = 0; // FIXME: Setting Capacity to 0 is suspect. |
135 | } |
136 | |
137 | /// Return true if V is an internal reference to the given range. |
138 | bool isReferenceToRange(const void *V, const void *First, const void *Last) const { |
139 | // Use std::less to avoid UB. |
140 | std::less<> LessThan; |
141 | return !LessThan(V, First) && LessThan(V, Last); |
142 | } |
143 | |
144 | /// Return true if V is an internal reference to this vector. |
145 | bool isReferenceToStorage(const void *V) const { |
146 | return isReferenceToRange(V, this->begin(), this->end()); |
147 | } |
148 | |
149 | /// Return true if First and Last form a valid (possibly empty) range in this |
150 | /// vector's storage. |
151 | bool isRangeInStorage(const void *First, const void *Last) const { |
152 | // Use std::less to avoid UB. |
153 | std::less<> LessThan; |
154 | return !LessThan(First, this->begin()) && !LessThan(Last, First) && |
155 | !LessThan(this->end(), Last); |
156 | } |
157 | |
158 | /// Return true unless Elt will be invalidated by resizing the vector to |
159 | /// NewSize. |
160 | bool isSafeToReferenceAfterResize(const void *Elt, size_t NewSize) { |
161 | // Past the end. |
162 | if (LLVM_LIKELY(!isReferenceToStorage(Elt))__builtin_expect((bool)(!isReferenceToStorage(Elt)), true)) |
163 | return true; |
164 | |
165 | // Return false if Elt will be destroyed by shrinking. |
166 | if (NewSize <= this->size()) |
167 | return Elt < this->begin() + NewSize; |
168 | |
169 | // Return false if we need to grow. |
170 | return NewSize <= this->capacity(); |
171 | } |
172 | |
173 | /// Check whether Elt will be invalidated by resizing the vector to NewSize. |
174 | void assertSafeToReferenceAfterResize(const void *Elt, size_t NewSize) { |
175 | assert(isSafeToReferenceAfterResize(Elt, NewSize) &&((void)0) |
176 | "Attempting to reference an element of the vector in an operation "((void)0) |
177 | "that invalidates it")((void)0); |
178 | } |
179 | |
180 | /// Check whether Elt will be invalidated by increasing the size of the |
181 | /// vector by N. |
182 | void assertSafeToAdd(const void *Elt, size_t N = 1) { |
183 | this->assertSafeToReferenceAfterResize(Elt, this->size() + N); |
184 | } |
185 | |
186 | /// Check whether any part of the range will be invalidated by clearing. |
187 | void assertSafeToReferenceAfterClear(const T *From, const T *To) { |
188 | if (From == To) |
189 | return; |
190 | this->assertSafeToReferenceAfterResize(From, 0); |
191 | this->assertSafeToReferenceAfterResize(To - 1, 0); |
192 | } |
193 | template < |
194 | class ItTy, |
195 | std::enable_if_t<!std::is_same<std::remove_const_t<ItTy>, T *>::value, |
196 | bool> = false> |
197 | void assertSafeToReferenceAfterClear(ItTy, ItTy) {} |
198 | |
199 | /// Check whether any part of the range will be invalidated by growing. |
200 | void assertSafeToAddRange(const T *From, const T *To) { |
201 | if (From == To) |
202 | return; |
203 | this->assertSafeToAdd(From, To - From); |
204 | this->assertSafeToAdd(To - 1, To - From); |
205 | } |
206 | template < |
207 | class ItTy, |
208 | std::enable_if_t<!std::is_same<std::remove_const_t<ItTy>, T *>::value, |
209 | bool> = false> |
210 | void assertSafeToAddRange(ItTy, ItTy) {} |
211 | |
212 | /// Reserve enough space to add one element, and return the updated element |
213 | /// pointer in case it was a reference to the storage. |
214 | template <class U> |
215 | static const T *reserveForParamAndGetAddressImpl(U *This, const T &Elt, |
216 | size_t N) { |
217 | size_t NewSize = This->size() + N; |
218 | if (LLVM_LIKELY(NewSize <= This->capacity())__builtin_expect((bool)(NewSize <= This->capacity()), true )) |
219 | return &Elt; |
220 | |
221 | bool ReferencesStorage = false; |
222 | int64_t Index = -1; |
223 | if (!U::TakesParamByValue) { |
224 | if (LLVM_UNLIKELY(This->isReferenceToStorage(&Elt))__builtin_expect((bool)(This->isReferenceToStorage(&Elt )), false)) { |
225 | ReferencesStorage = true; |
226 | Index = &Elt - This->begin(); |
227 | } |
228 | } |
229 | This->grow(NewSize); |
230 | return ReferencesStorage ? This->begin() + Index : &Elt; |
231 | } |
232 | |
233 | public: |
234 | using size_type = size_t; |
235 | using difference_type = ptrdiff_t; |
236 | using value_type = T; |
237 | using iterator = T *; |
238 | using const_iterator = const T *; |
239 | |
240 | using const_reverse_iterator = std::reverse_iterator<const_iterator>; |
241 | using reverse_iterator = std::reverse_iterator<iterator>; |
242 | |
243 | using reference = T &; |
244 | using const_reference = const T &; |
245 | using pointer = T *; |
246 | using const_pointer = const T *; |
247 | |
248 | using Base::capacity; |
249 | using Base::empty; |
250 | using Base::size; |
251 | |
252 | // forward iterator creation methods. |
253 | iterator begin() { return (iterator)this->BeginX; } |
254 | const_iterator begin() const { return (const_iterator)this->BeginX; } |
255 | iterator end() { return begin() + size(); } |
256 | const_iterator end() const { return begin() + size(); } |
257 | |
258 | // reverse iterator creation methods. |
259 | reverse_iterator rbegin() { return reverse_iterator(end()); } |
260 | const_reverse_iterator rbegin() const{ return const_reverse_iterator(end()); } |
261 | reverse_iterator rend() { return reverse_iterator(begin()); } |
262 | const_reverse_iterator rend() const { return const_reverse_iterator(begin());} |
263 | |
264 | size_type size_in_bytes() const { return size() * sizeof(T); } |
265 | size_type max_size() const { |
266 | return std::min(this->SizeTypeMax(), size_type(-1) / sizeof(T)); |
267 | } |
268 | |
269 | size_t capacity_in_bytes() const { return capacity() * sizeof(T); } |
270 | |
271 | /// Return a pointer to the vector's buffer, even if empty(). |
272 | pointer data() { return pointer(begin()); } |
273 | /// Return a pointer to the vector's buffer, even if empty(). |
274 | const_pointer data() const { return const_pointer(begin()); } |
275 | |
276 | reference operator[](size_type idx) { |
277 | assert(idx < size())((void)0); |
278 | return begin()[idx]; |
279 | } |
280 | const_reference operator[](size_type idx) const { |
281 | assert(idx < size())((void)0); |
282 | return begin()[idx]; |
283 | } |
284 | |
285 | reference front() { |
286 | assert(!empty())((void)0); |
287 | return begin()[0]; |
288 | } |
289 | const_reference front() const { |
290 | assert(!empty())((void)0); |
291 | return begin()[0]; |
292 | } |
293 | |
294 | reference back() { |
295 | assert(!empty())((void)0); |
296 | return end()[-1]; |
297 | } |
298 | const_reference back() const { |
299 | assert(!empty())((void)0); |
300 | return end()[-1]; |
301 | } |
302 | }; |
303 | |
304 | /// SmallVectorTemplateBase<TriviallyCopyable = false> - This is where we put |
305 | /// method implementations that are designed to work with non-trivial T's. |
306 | /// |
307 | /// We approximate is_trivially_copyable with trivial move/copy construction and |
308 | /// trivial destruction. While the standard doesn't specify that you're allowed |
309 | /// copy these types with memcpy, there is no way for the type to observe this. |
310 | /// This catches the important case of std::pair<POD, POD>, which is not |
311 | /// trivially assignable. |
312 | template <typename T, bool = (is_trivially_copy_constructible<T>::value) && |
313 | (is_trivially_move_constructible<T>::value) && |
314 | std::is_trivially_destructible<T>::value> |
315 | class SmallVectorTemplateBase : public SmallVectorTemplateCommon<T> { |
316 | friend class SmallVectorTemplateCommon<T>; |
317 | |
318 | protected: |
319 | static constexpr bool TakesParamByValue = false; |
320 | using ValueParamT = const T &; |
321 | |
322 | SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {} |
323 | |
324 | static void destroy_range(T *S, T *E) { |
325 | while (S != E) { |
326 | --E; |
327 | E->~T(); |
328 | } |
329 | } |
330 | |
331 | /// Move the range [I, E) into the uninitialized memory starting with "Dest", |
332 | /// constructing elements as needed. |
333 | template<typename It1, typename It2> |
334 | static void uninitialized_move(It1 I, It1 E, It2 Dest) { |
335 | std::uninitialized_copy(std::make_move_iterator(I), |
336 | std::make_move_iterator(E), Dest); |
337 | } |
338 | |
339 | /// Copy the range [I, E) onto the uninitialized memory starting with "Dest", |
340 | /// constructing elements as needed. |
341 | template<typename It1, typename It2> |
342 | static void uninitialized_copy(It1 I, It1 E, It2 Dest) { |
343 | std::uninitialized_copy(I, E, Dest); |
344 | } |
345 | |
346 | /// Grow the allocated memory (without initializing new elements), doubling |
347 | /// the size of the allocated memory. Guarantees space for at least one more |
348 | /// element, or MinSize more elements if specified. |
349 | void grow(size_t MinSize = 0); |
350 | |
351 | /// Create a new allocation big enough for \p MinSize and pass back its size |
352 | /// in \p NewCapacity. This is the first section of \a grow(). |
353 | T *mallocForGrow(size_t MinSize, size_t &NewCapacity) { |
354 | return static_cast<T *>( |
355 | SmallVectorBase<SmallVectorSizeType<T>>::mallocForGrow( |
356 | MinSize, sizeof(T), NewCapacity)); |
357 | } |
358 | |
359 | /// Move existing elements over to the new allocation \p NewElts, the middle |
360 | /// section of \a grow(). |
361 | void moveElementsForGrow(T *NewElts); |
362 | |
363 | /// Transfer ownership of the allocation, finishing up \a grow(). |
364 | void takeAllocationForGrow(T *NewElts, size_t NewCapacity); |
365 | |
366 | /// Reserve enough space to add one element, and return the updated element |
367 | /// pointer in case it was a reference to the storage. |
368 | const T *reserveForParamAndGetAddress(const T &Elt, size_t N = 1) { |
369 | return this->reserveForParamAndGetAddressImpl(this, Elt, N); |
370 | } |
371 | |
372 | /// Reserve enough space to add one element, and return the updated element |
373 | /// pointer in case it was a reference to the storage. |
374 | T *reserveForParamAndGetAddress(T &Elt, size_t N = 1) { |
375 | return const_cast<T *>( |
376 | this->reserveForParamAndGetAddressImpl(this, Elt, N)); |
377 | } |
378 | |
379 | static T &&forward_value_param(T &&V) { return std::move(V); } |
380 | static const T &forward_value_param(const T &V) { return V; } |
381 | |
382 | void growAndAssign(size_t NumElts, const T &Elt) { |
383 | // Grow manually in case Elt is an internal reference. |
384 | size_t NewCapacity; |
385 | T *NewElts = mallocForGrow(NumElts, NewCapacity); |
386 | std::uninitialized_fill_n(NewElts, NumElts, Elt); |
387 | this->destroy_range(this->begin(), this->end()); |
388 | takeAllocationForGrow(NewElts, NewCapacity); |
389 | this->set_size(NumElts); |
390 | } |
391 | |
392 | template <typename... ArgTypes> T &growAndEmplaceBack(ArgTypes &&... Args) { |
393 | // Grow manually in case one of Args is an internal reference. |
394 | size_t NewCapacity; |
395 | T *NewElts = mallocForGrow(0, NewCapacity); |
396 | ::new ((void *)(NewElts + this->size())) T(std::forward<ArgTypes>(Args)...); |
397 | moveElementsForGrow(NewElts); |
398 | takeAllocationForGrow(NewElts, NewCapacity); |
399 | this->set_size(this->size() + 1); |
400 | return this->back(); |
401 | } |
402 | |
403 | public: |
404 | void push_back(const T &Elt) { |
405 | const T *EltPtr = reserveForParamAndGetAddress(Elt); |
406 | ::new ((void *)this->end()) T(*EltPtr); |
407 | this->set_size(this->size() + 1); |
408 | } |
409 | |
410 | void push_back(T &&Elt) { |
411 | T *EltPtr = reserveForParamAndGetAddress(Elt); |
412 | ::new ((void *)this->end()) T(::std::move(*EltPtr)); |
413 | this->set_size(this->size() + 1); |
414 | } |
415 | |
416 | void pop_back() { |
417 | this->set_size(this->size() - 1); |
418 | this->end()->~T(); |
419 | } |
420 | }; |
421 | |
422 | // Define this out-of-line to dissuade the C++ compiler from inlining it. |
423 | template <typename T, bool TriviallyCopyable> |
424 | void SmallVectorTemplateBase<T, TriviallyCopyable>::grow(size_t MinSize) { |
425 | size_t NewCapacity; |
426 | T *NewElts = mallocForGrow(MinSize, NewCapacity); |
427 | moveElementsForGrow(NewElts); |
428 | takeAllocationForGrow(NewElts, NewCapacity); |
429 | } |
430 | |
431 | // Define this out-of-line to dissuade the C++ compiler from inlining it. |
432 | template <typename T, bool TriviallyCopyable> |
433 | void SmallVectorTemplateBase<T, TriviallyCopyable>::moveElementsForGrow( |
434 | T *NewElts) { |
435 | // Move the elements over. |
436 | this->uninitialized_move(this->begin(), this->end(), NewElts); |
437 | |
438 | // Destroy the original elements. |
439 | destroy_range(this->begin(), this->end()); |
440 | } |
441 | |
442 | // Define this out-of-line to dissuade the C++ compiler from inlining it. |
443 | template <typename T, bool TriviallyCopyable> |
444 | void SmallVectorTemplateBase<T, TriviallyCopyable>::takeAllocationForGrow( |
445 | T *NewElts, size_t NewCapacity) { |
446 | // If this wasn't grown from the inline copy, deallocate the old space. |
447 | if (!this->isSmall()) |
448 | free(this->begin()); |
449 | |
450 | this->BeginX = NewElts; |
451 | this->Capacity = NewCapacity; |
452 | } |
453 | |
454 | /// SmallVectorTemplateBase<TriviallyCopyable = true> - This is where we put |
455 | /// method implementations that are designed to work with trivially copyable |
456 | /// T's. This allows using memcpy in place of copy/move construction and |
457 | /// skipping destruction. |
458 | template <typename T> |
459 | class SmallVectorTemplateBase<T, true> : public SmallVectorTemplateCommon<T> { |
460 | friend class SmallVectorTemplateCommon<T>; |
461 | |
462 | protected: |
463 | /// True if it's cheap enough to take parameters by value. Doing so avoids |
464 | /// overhead related to mitigations for reference invalidation. |
465 | static constexpr bool TakesParamByValue = sizeof(T) <= 2 * sizeof(void *); |
466 | |
467 | /// Either const T& or T, depending on whether it's cheap enough to take |
468 | /// parameters by value. |
469 | using ValueParamT = |
470 | typename std::conditional<TakesParamByValue, T, const T &>::type; |
471 | |
472 | SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {} |
473 | |
474 | // No need to do a destroy loop for POD's. |
475 | static void destroy_range(T *, T *) {} |
476 | |
477 | /// Move the range [I, E) onto the uninitialized memory |
478 | /// starting with "Dest", constructing elements into it as needed. |
479 | template<typename It1, typename It2> |
480 | static void uninitialized_move(It1 I, It1 E, It2 Dest) { |
481 | // Just do a copy. |
482 | uninitialized_copy(I, E, Dest); |
483 | } |
484 | |
485 | /// Copy the range [I, E) onto the uninitialized memory |
486 | /// starting with "Dest", constructing elements into it as needed. |
487 | template<typename It1, typename It2> |
488 | static void uninitialized_copy(It1 I, It1 E, It2 Dest) { |
489 | // Arbitrary iterator types; just use the basic implementation. |
490 | std::uninitialized_copy(I, E, Dest); |
491 | } |
492 | |
493 | /// Copy the range [I, E) onto the uninitialized memory |
494 | /// starting with "Dest", constructing elements into it as needed. |
495 | template <typename T1, typename T2> |
496 | static void uninitialized_copy( |
497 | T1 *I, T1 *E, T2 *Dest, |
498 | std::enable_if_t<std::is_same<typename std::remove_const<T1>::type, |
499 | T2>::value> * = nullptr) { |
500 | // Use memcpy for PODs iterated by pointers (which includes SmallVector |
501 | // iterators): std::uninitialized_copy optimizes to memmove, but we can |
502 | // use memcpy here. Note that I and E are iterators and thus might be |
503 | // invalid for memcpy if they are equal. |
504 | if (I != E) |
505 | memcpy(reinterpret_cast<void *>(Dest), I, (E - I) * sizeof(T)); |
506 | } |
507 | |
508 | /// Double the size of the allocated memory, guaranteeing space for at |
509 | /// least one more element or MinSize if specified. |
510 | void grow(size_t MinSize = 0) { this->grow_pod(MinSize, sizeof(T)); } |
511 | |
512 | /// Reserve enough space to add one element, and return the updated element |
513 | /// pointer in case it was a reference to the storage. |
514 | const T *reserveForParamAndGetAddress(const T &Elt, size_t N = 1) { |
515 | return this->reserveForParamAndGetAddressImpl(this, Elt, N); |
516 | } |
517 | |
518 | /// Reserve enough space to add one element, and return the updated element |
519 | /// pointer in case it was a reference to the storage. |
520 | T *reserveForParamAndGetAddress(T &Elt, size_t N = 1) { |
521 | return const_cast<T *>( |
522 | this->reserveForParamAndGetAddressImpl(this, Elt, N)); |
523 | } |
524 | |
525 | /// Copy \p V or return a reference, depending on \a ValueParamT. |
526 | static ValueParamT forward_value_param(ValueParamT V) { return V; } |
527 | |
528 | void growAndAssign(size_t NumElts, T Elt) { |
529 | // Elt has been copied in case it's an internal reference, side-stepping |
530 | // reference invalidation problems without losing the realloc optimization. |
531 | this->set_size(0); |
532 | this->grow(NumElts); |
533 | std::uninitialized_fill_n(this->begin(), NumElts, Elt); |
534 | this->set_size(NumElts); |
535 | } |
536 | |
537 | template <typename... ArgTypes> T &growAndEmplaceBack(ArgTypes &&... Args) { |
538 | // Use push_back with a copy in case Args has an internal reference, |
539 | // side-stepping reference invalidation problems without losing the realloc |
540 | // optimization. |
541 | push_back(T(std::forward<ArgTypes>(Args)...)); |
542 | return this->back(); |
543 | } |
544 | |
545 | public: |
546 | void push_back(ValueParamT Elt) { |
547 | const T *EltPtr = reserveForParamAndGetAddress(Elt); |
548 | memcpy(reinterpret_cast<void *>(this->end()), EltPtr, sizeof(T)); |
549 | this->set_size(this->size() + 1); |
550 | } |
551 | |
552 | void pop_back() { this->set_size(this->size() - 1); } |
553 | }; |
554 | |
555 | /// This class consists of common code factored out of the SmallVector class to |
556 | /// reduce code duplication based on the SmallVector 'N' template parameter. |
557 | template <typename T> |
558 | class SmallVectorImpl : public SmallVectorTemplateBase<T> { |
559 | using SuperClass = SmallVectorTemplateBase<T>; |
560 | |
561 | public: |
562 | using iterator = typename SuperClass::iterator; |
563 | using const_iterator = typename SuperClass::const_iterator; |
564 | using reference = typename SuperClass::reference; |
565 | using size_type = typename SuperClass::size_type; |
566 | |
567 | protected: |
568 | using SmallVectorTemplateBase<T>::TakesParamByValue; |
569 | using ValueParamT = typename SuperClass::ValueParamT; |
570 | |
571 | // Default ctor - Initialize to empty. |
572 | explicit SmallVectorImpl(unsigned N) |
573 | : SmallVectorTemplateBase<T>(N) {} |
574 | |
575 | public: |
576 | SmallVectorImpl(const SmallVectorImpl &) = delete; |
577 | |
578 | ~SmallVectorImpl() { |
579 | // Subclass has already destructed this vector's elements. |
580 | // If this wasn't grown from the inline copy, deallocate the old space. |
581 | if (!this->isSmall()) |
582 | free(this->begin()); |
583 | } |
584 | |
585 | void clear() { |
586 | this->destroy_range(this->begin(), this->end()); |
587 | this->Size = 0; |
588 | } |
589 | |
590 | private: |
591 | template <bool ForOverwrite> void resizeImpl(size_type N) { |
592 | if (N < this->size()) { |
593 | this->pop_back_n(this->size() - N); |
594 | } else if (N > this->size()) { |
595 | this->reserve(N); |
596 | for (auto I = this->end(), E = this->begin() + N; I != E; ++I) |
597 | if (ForOverwrite) |
598 | new (&*I) T; |
599 | else |
600 | new (&*I) T(); |
601 | this->set_size(N); |
602 | } |
603 | } |
604 | |
605 | public: |
606 | void resize(size_type N) { resizeImpl<false>(N); } |
607 | |
608 | /// Like resize, but \ref T is POD, the new values won't be initialized. |
609 | void resize_for_overwrite(size_type N) { resizeImpl<true>(N); } |
610 | |
611 | void resize(size_type N, ValueParamT NV) { |
612 | if (N == this->size()) |
613 | return; |
614 | |
615 | if (N < this->size()) { |
616 | this->pop_back_n(this->size() - N); |
617 | return; |
618 | } |
619 | |
620 | // N > this->size(). Defer to append. |
621 | this->append(N - this->size(), NV); |
622 | } |
623 | |
624 | void reserve(size_type N) { |
625 | if (this->capacity() < N) |
626 | this->grow(N); |
627 | } |
628 | |
629 | void pop_back_n(size_type NumItems) { |
630 | assert(this->size() >= NumItems)((void)0); |
631 | this->destroy_range(this->end() - NumItems, this->end()); |
632 | this->set_size(this->size() - NumItems); |
633 | } |
634 | |
635 | LLVM_NODISCARD[[clang::warn_unused_result]] T pop_back_val() { |
636 | T Result = ::std::move(this->back()); |
637 | this->pop_back(); |
638 | return Result; |
639 | } |
640 | |
641 | void swap(SmallVectorImpl &RHS); |
642 | |
643 | /// Add the specified range to the end of the SmallVector. |
644 | template <typename in_iter, |
645 | typename = std::enable_if_t<std::is_convertible< |
646 | typename std::iterator_traits<in_iter>::iterator_category, |
647 | std::input_iterator_tag>::value>> |
648 | void append(in_iter in_start, in_iter in_end) { |
649 | this->assertSafeToAddRange(in_start, in_end); |
650 | size_type NumInputs = std::distance(in_start, in_end); |
651 | this->reserve(this->size() + NumInputs); |
652 | this->uninitialized_copy(in_start, in_end, this->end()); |
653 | this->set_size(this->size() + NumInputs); |
654 | } |
655 | |
656 | /// Append \p NumInputs copies of \p Elt to the end. |
657 | void append(size_type NumInputs, ValueParamT Elt) { |
658 | const T *EltPtr = this->reserveForParamAndGetAddress(Elt, NumInputs); |
659 | std::uninitialized_fill_n(this->end(), NumInputs, *EltPtr); |
660 | this->set_size(this->size() + NumInputs); |
661 | } |
662 | |
663 | void append(std::initializer_list<T> IL) { |
664 | append(IL.begin(), IL.end()); |
665 | } |
666 | |
667 | void append(const SmallVectorImpl &RHS) { append(RHS.begin(), RHS.end()); } |
668 | |
669 | void assign(size_type NumElts, ValueParamT Elt) { |
670 | // Note that Elt could be an internal reference. |
671 | if (NumElts > this->capacity()) { |
672 | this->growAndAssign(NumElts, Elt); |
673 | return; |
674 | } |
675 | |
676 | // Assign over existing elements. |
677 | std::fill_n(this->begin(), std::min(NumElts, this->size()), Elt); |
678 | if (NumElts > this->size()) |
679 | std::uninitialized_fill_n(this->end(), NumElts - this->size(), Elt); |
680 | else if (NumElts < this->size()) |
681 | this->destroy_range(this->begin() + NumElts, this->end()); |
682 | this->set_size(NumElts); |
683 | } |
684 | |
685 | // FIXME: Consider assigning over existing elements, rather than clearing & |
686 | // re-initializing them - for all assign(...) variants. |
687 | |
688 | template <typename in_iter, |
689 | typename = std::enable_if_t<std::is_convertible< |
690 | typename std::iterator_traits<in_iter>::iterator_category, |
691 | std::input_iterator_tag>::value>> |
692 | void assign(in_iter in_start, in_iter in_end) { |
693 | this->assertSafeToReferenceAfterClear(in_start, in_end); |
694 | clear(); |
695 | append(in_start, in_end); |
696 | } |
697 | |
698 | void assign(std::initializer_list<T> IL) { |
699 | clear(); |
700 | append(IL); |
701 | } |
702 | |
703 | void assign(const SmallVectorImpl &RHS) { assign(RHS.begin(), RHS.end()); } |
704 | |
705 | iterator erase(const_iterator CI) { |
706 | // Just cast away constness because this is a non-const member function. |
707 | iterator I = const_cast<iterator>(CI); |
708 | |
709 | assert(this->isReferenceToStorage(CI) && "Iterator to erase is out of bounds.")((void)0); |
710 | |
711 | iterator N = I; |
712 | // Shift all elts down one. |
713 | std::move(I+1, this->end(), I); |
714 | // Drop the last elt. |
715 | this->pop_back(); |
716 | return(N); |
717 | } |
718 | |
719 | iterator erase(const_iterator CS, const_iterator CE) { |
720 | // Just cast away constness because this is a non-const member function. |
721 | iterator S = const_cast<iterator>(CS); |
722 | iterator E = const_cast<iterator>(CE); |
723 | |
724 | assert(this->isRangeInStorage(S, E) && "Range to erase is out of bounds.")((void)0); |
725 | |
726 | iterator N = S; |
727 | // Shift all elts down. |
728 | iterator I = std::move(E, this->end(), S); |
729 | // Drop the last elts. |
730 | this->destroy_range(I, this->end()); |
731 | this->set_size(I - this->begin()); |
732 | return(N); |
733 | } |
734 | |
735 | private: |
736 | template <class ArgType> iterator insert_one_impl(iterator I, ArgType &&Elt) { |
737 | // Callers ensure that ArgType is derived from T. |
738 | static_assert( |
739 | std::is_same<std::remove_const_t<std::remove_reference_t<ArgType>>, |
740 | T>::value, |
741 | "ArgType must be derived from T!"); |
742 | |
743 | if (I == this->end()) { // Important special case for empty vector. |
744 | this->push_back(::std::forward<ArgType>(Elt)); |
745 | return this->end()-1; |
746 | } |
747 | |
748 | assert(this->isReferenceToStorage(I) && "Insertion iterator is out of bounds.")((void)0); |
749 | |
750 | // Grow if necessary. |
751 | size_t Index = I - this->begin(); |
752 | std::remove_reference_t<ArgType> *EltPtr = |
753 | this->reserveForParamAndGetAddress(Elt); |
754 | I = this->begin() + Index; |
755 | |
756 | ::new ((void*) this->end()) T(::std::move(this->back())); |
757 | // Push everything else over. |
758 | std::move_backward(I, this->end()-1, this->end()); |
759 | this->set_size(this->size() + 1); |
760 | |
761 | // If we just moved the element we're inserting, be sure to update |
762 | // the reference (never happens if TakesParamByValue). |
763 | static_assert(!TakesParamByValue || std::is_same<ArgType, T>::value, |
764 | "ArgType must be 'T' when taking by value!"); |
765 | if (!TakesParamByValue && this->isReferenceToRange(EltPtr, I, this->end())) |
766 | ++EltPtr; |
767 | |
768 | *I = ::std::forward<ArgType>(*EltPtr); |
769 | return I; |
770 | } |
771 | |
772 | public: |
773 | iterator insert(iterator I, T &&Elt) { |
774 | return insert_one_impl(I, this->forward_value_param(std::move(Elt))); |
775 | } |
776 | |
777 | iterator insert(iterator I, const T &Elt) { |
778 | return insert_one_impl(I, this->forward_value_param(Elt)); |
779 | } |
780 | |
781 | iterator insert(iterator I, size_type NumToInsert, ValueParamT Elt) { |
782 | // Convert iterator to elt# to avoid invalidating iterator when we reserve() |
783 | size_t InsertElt = I - this->begin(); |
784 | |
785 | if (I == this->end()) { // Important special case for empty vector. |
786 | append(NumToInsert, Elt); |
787 | return this->begin()+InsertElt; |
788 | } |
789 | |
790 | assert(this->isReferenceToStorage(I) && "Insertion iterator is out of bounds.")((void)0); |
791 | |
792 | // Ensure there is enough space, and get the (maybe updated) address of |
793 | // Elt. |
794 | const T *EltPtr = this->reserveForParamAndGetAddress(Elt, NumToInsert); |
795 | |
796 | // Uninvalidate the iterator. |
797 | I = this->begin()+InsertElt; |
798 | |
799 | // If there are more elements between the insertion point and the end of the |
800 | // range than there are being inserted, we can use a simple approach to |
801 | // insertion. Since we already reserved space, we know that this won't |
802 | // reallocate the vector. |
803 | if (size_t(this->end()-I) >= NumToInsert) { |
804 | T *OldEnd = this->end(); |
805 | append(std::move_iterator<iterator>(this->end() - NumToInsert), |
806 | std::move_iterator<iterator>(this->end())); |
807 | |
808 | // Copy the existing elements that get replaced. |
809 | std::move_backward(I, OldEnd-NumToInsert, OldEnd); |
810 | |
811 | // If we just moved the element we're inserting, be sure to update |
812 | // the reference (never happens if TakesParamByValue). |
813 | if (!TakesParamByValue && I <= EltPtr && EltPtr < this->end()) |
814 | EltPtr += NumToInsert; |
815 | |
816 | std::fill_n(I, NumToInsert, *EltPtr); |
817 | return I; |
818 | } |
819 | |
820 | // Otherwise, we're inserting more elements than exist already, and we're |
821 | // not inserting at the end. |
822 | |
823 | // Move over the elements that we're about to overwrite. |
824 | T *OldEnd = this->end(); |
825 | this->set_size(this->size() + NumToInsert); |
826 | size_t NumOverwritten = OldEnd-I; |
827 | this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); |
828 | |
829 | // If we just moved the element we're inserting, be sure to update |
830 | // the reference (never happens if TakesParamByValue). |
831 | if (!TakesParamByValue && I <= EltPtr && EltPtr < this->end()) |
832 | EltPtr += NumToInsert; |
833 | |
834 | // Replace the overwritten part. |
835 | std::fill_n(I, NumOverwritten, *EltPtr); |
836 | |
837 | // Insert the non-overwritten middle part. |
838 | std::uninitialized_fill_n(OldEnd, NumToInsert - NumOverwritten, *EltPtr); |
839 | return I; |
840 | } |
841 | |
842 | template <typename ItTy, |
843 | typename = std::enable_if_t<std::is_convertible< |
844 | typename std::iterator_traits<ItTy>::iterator_category, |
845 | std::input_iterator_tag>::value>> |
846 | iterator insert(iterator I, ItTy From, ItTy To) { |
847 | // Convert iterator to elt# to avoid invalidating iterator when we reserve() |
848 | size_t InsertElt = I - this->begin(); |
849 | |
850 | if (I == this->end()) { // Important special case for empty vector. |
851 | append(From, To); |
852 | return this->begin()+InsertElt; |
853 | } |
854 | |
855 | assert(this->isReferenceToStorage(I) && "Insertion iterator is out of bounds.")((void)0); |
856 | |
857 | // Check that the reserve that follows doesn't invalidate the iterators. |
858 | this->assertSafeToAddRange(From, To); |
859 | |
860 | size_t NumToInsert = std::distance(From, To); |
861 | |
862 | // Ensure there is enough space. |
863 | reserve(this->size() + NumToInsert); |
864 | |
865 | // Uninvalidate the iterator. |
866 | I = this->begin()+InsertElt; |
867 | |
868 | // If there are more elements between the insertion point and the end of the |
869 | // range than there are being inserted, we can use a simple approach to |
870 | // insertion. Since we already reserved space, we know that this won't |
871 | // reallocate the vector. |
872 | if (size_t(this->end()-I) >= NumToInsert) { |
873 | T *OldEnd = this->end(); |
874 | append(std::move_iterator<iterator>(this->end() - NumToInsert), |
875 | std::move_iterator<iterator>(this->end())); |
876 | |
877 | // Copy the existing elements that get replaced. |
878 | std::move_backward(I, OldEnd-NumToInsert, OldEnd); |
879 | |
880 | std::copy(From, To, I); |
881 | return I; |
882 | } |
883 | |
884 | // Otherwise, we're inserting more elements than exist already, and we're |
885 | // not inserting at the end. |
886 | |
887 | // Move over the elements that we're about to overwrite. |
888 | T *OldEnd = this->end(); |
889 | this->set_size(this->size() + NumToInsert); |
890 | size_t NumOverwritten = OldEnd-I; |
891 | this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); |
892 | |
893 | // Replace the overwritten part. |
894 | for (T *J = I; NumOverwritten > 0; --NumOverwritten) { |
895 | *J = *From; |
896 | ++J; ++From; |
897 | } |
898 | |
899 | // Insert the non-overwritten middle part. |
900 | this->uninitialized_copy(From, To, OldEnd); |
901 | return I; |
902 | } |
903 | |
904 | void insert(iterator I, std::initializer_list<T> IL) { |
905 | insert(I, IL.begin(), IL.end()); |
906 | } |
907 | |
908 | template <typename... ArgTypes> reference emplace_back(ArgTypes &&... Args) { |
909 | if (LLVM_UNLIKELY(this->size() >= this->capacity())__builtin_expect((bool)(this->size() >= this->capacity ()), false)) |
910 | return this->growAndEmplaceBack(std::forward<ArgTypes>(Args)...); |
911 | |
912 | ::new ((void *)this->end()) T(std::forward<ArgTypes>(Args)...); |
913 | this->set_size(this->size() + 1); |
914 | return this->back(); |
915 | } |
916 | |
917 | SmallVectorImpl &operator=(const SmallVectorImpl &RHS); |
918 | |
919 | SmallVectorImpl &operator=(SmallVectorImpl &&RHS); |
920 | |
921 | bool operator==(const SmallVectorImpl &RHS) const { |
922 | if (this->size() != RHS.size()) return false; |
923 | return std::equal(this->begin(), this->end(), RHS.begin()); |
924 | } |
925 | bool operator!=(const SmallVectorImpl &RHS) const { |
926 | return !(*this == RHS); |
927 | } |
928 | |
929 | bool operator<(const SmallVectorImpl &RHS) const { |
930 | return std::lexicographical_compare(this->begin(), this->end(), |
931 | RHS.begin(), RHS.end()); |
932 | } |
933 | }; |
934 | |
935 | template <typename T> |
936 | void SmallVectorImpl<T>::swap(SmallVectorImpl<T> &RHS) { |
937 | if (this == &RHS) return; |
938 | |
939 | // We can only avoid copying elements if neither vector is small. |
940 | if (!this->isSmall() && !RHS.isSmall()) { |
941 | std::swap(this->BeginX, RHS.BeginX); |
942 | std::swap(this->Size, RHS.Size); |
943 | std::swap(this->Capacity, RHS.Capacity); |
944 | return; |
945 | } |
946 | this->reserve(RHS.size()); |
947 | RHS.reserve(this->size()); |
948 | |
949 | // Swap the shared elements. |
950 | size_t NumShared = this->size(); |
951 | if (NumShared > RHS.size()) NumShared = RHS.size(); |
952 | for (size_type i = 0; i != NumShared; ++i) |
953 | std::swap((*this)[i], RHS[i]); |
954 | |
955 | // Copy over the extra elts. |
956 | if (this->size() > RHS.size()) { |
957 | size_t EltDiff = this->size() - RHS.size(); |
958 | this->uninitialized_copy(this->begin()+NumShared, this->end(), RHS.end()); |
959 | RHS.set_size(RHS.size() + EltDiff); |
960 | this->destroy_range(this->begin()+NumShared, this->end()); |
961 | this->set_size(NumShared); |
962 | } else if (RHS.size() > this->size()) { |
963 | size_t EltDiff = RHS.size() - this->size(); |
964 | this->uninitialized_copy(RHS.begin()+NumShared, RHS.end(), this->end()); |
965 | this->set_size(this->size() + EltDiff); |
966 | this->destroy_range(RHS.begin()+NumShared, RHS.end()); |
967 | RHS.set_size(NumShared); |
968 | } |
969 | } |
970 | |
971 | template <typename T> |
972 | SmallVectorImpl<T> &SmallVectorImpl<T>:: |
973 | operator=(const SmallVectorImpl<T> &RHS) { |
974 | // Avoid self-assignment. |
975 | if (this == &RHS) return *this; |
976 | |
977 | // If we already have sufficient space, assign the common elements, then |
978 | // destroy any excess. |
979 | size_t RHSSize = RHS.size(); |
980 | size_t CurSize = this->size(); |
981 | if (CurSize >= RHSSize) { |
982 | // Assign common elements. |
983 | iterator NewEnd; |
984 | if (RHSSize) |
985 | NewEnd = std::copy(RHS.begin(), RHS.begin()+RHSSize, this->begin()); |
986 | else |
987 | NewEnd = this->begin(); |
988 | |
989 | // Destroy excess elements. |
990 | this->destroy_range(NewEnd, this->end()); |
991 | |
992 | // Trim. |
993 | this->set_size(RHSSize); |
994 | return *this; |
995 | } |
996 | |
997 | // If we have to grow to have enough elements, destroy the current elements. |
998 | // This allows us to avoid copying them during the grow. |
999 | // FIXME: don't do this if they're efficiently moveable. |
1000 | if (this->capacity() < RHSSize) { |
1001 | // Destroy current elements. |
1002 | this->clear(); |
1003 | CurSize = 0; |
1004 | this->grow(RHSSize); |
1005 | } else if (CurSize) { |
1006 | // Otherwise, use assignment for the already-constructed elements. |
1007 | std::copy(RHS.begin(), RHS.begin()+CurSize, this->begin()); |
1008 | } |
1009 | |
1010 | // Copy construct the new elements in place. |
1011 | this->uninitialized_copy(RHS.begin()+CurSize, RHS.end(), |
1012 | this->begin()+CurSize); |
1013 | |
1014 | // Set end. |
1015 | this->set_size(RHSSize); |
1016 | return *this; |
1017 | } |
1018 | |
1019 | template <typename T> |
1020 | SmallVectorImpl<T> &SmallVectorImpl<T>::operator=(SmallVectorImpl<T> &&RHS) { |
1021 | // Avoid self-assignment. |
1022 | if (this == &RHS) return *this; |
1023 | |
1024 | // If the RHS isn't small, clear this vector and then steal its buffer. |
1025 | if (!RHS.isSmall()) { |
1026 | this->destroy_range(this->begin(), this->end()); |
1027 | if (!this->isSmall()) free(this->begin()); |
1028 | this->BeginX = RHS.BeginX; |
1029 | this->Size = RHS.Size; |
1030 | this->Capacity = RHS.Capacity; |
1031 | RHS.resetToSmall(); |
1032 | return *this; |
1033 | } |
1034 | |
1035 | // If we already have sufficient space, assign the common elements, then |
1036 | // destroy any excess. |
1037 | size_t RHSSize = RHS.size(); |
1038 | size_t CurSize = this->size(); |
1039 | if (CurSize >= RHSSize) { |
1040 | // Assign common elements. |
1041 | iterator NewEnd = this->begin(); |
1042 | if (RHSSize) |
1043 | NewEnd = std::move(RHS.begin(), RHS.end(), NewEnd); |
1044 | |
1045 | // Destroy excess elements and trim the bounds. |
1046 | this->destroy_range(NewEnd, this->end()); |
1047 | this->set_size(RHSSize); |
1048 | |
1049 | // Clear the RHS. |
1050 | RHS.clear(); |
1051 | |
1052 | return *this; |
1053 | } |
1054 | |
1055 | // If we have to grow to have enough elements, destroy the current elements. |
1056 | // This allows us to avoid copying them during the grow. |
1057 | // FIXME: this may not actually make any sense if we can efficiently move |
1058 | // elements. |
1059 | if (this->capacity() < RHSSize) { |
1060 | // Destroy current elements. |
1061 | this->clear(); |
1062 | CurSize = 0; |
1063 | this->grow(RHSSize); |
1064 | } else if (CurSize) { |
1065 | // Otherwise, use assignment for the already-constructed elements. |
1066 | std::move(RHS.begin(), RHS.begin()+CurSize, this->begin()); |
1067 | } |
1068 | |
1069 | // Move-construct the new elements in place. |
1070 | this->uninitialized_move(RHS.begin()+CurSize, RHS.end(), |
1071 | this->begin()+CurSize); |
1072 | |
1073 | // Set end. |
1074 | this->set_size(RHSSize); |
1075 | |
1076 | RHS.clear(); |
1077 | return *this; |
1078 | } |
1079 | |
1080 | /// Storage for the SmallVector elements. This is specialized for the N=0 case |
1081 | /// to avoid allocating unnecessary storage. |
1082 | template <typename T, unsigned N> |
1083 | struct SmallVectorStorage { |
1084 | alignas(T) char InlineElts[N * sizeof(T)]; |
1085 | }; |
1086 | |
1087 | /// We need the storage to be properly aligned even for small-size of 0 so that |
1088 | /// the pointer math in \a SmallVectorTemplateCommon::getFirstEl() is |
1089 | /// well-defined. |
1090 | template <typename T> struct alignas(T) SmallVectorStorage<T, 0> {}; |
1091 | |
1092 | /// Forward declaration of SmallVector so that |
1093 | /// calculateSmallVectorDefaultInlinedElements can reference |
1094 | /// `sizeof(SmallVector<T, 0>)`. |
1095 | template <typename T, unsigned N> class LLVM_GSL_OWNER[[gsl::Owner]] SmallVector; |
1096 | |
1097 | /// Helper class for calculating the default number of inline elements for |
1098 | /// `SmallVector<T>`. |
1099 | /// |
1100 | /// This should be migrated to a constexpr function when our minimum |
1101 | /// compiler support is enough for multi-statement constexpr functions. |
1102 | template <typename T> struct CalculateSmallVectorDefaultInlinedElements { |
1103 | // Parameter controlling the default number of inlined elements |
1104 | // for `SmallVector<T>`. |
1105 | // |
1106 | // The default number of inlined elements ensures that |
1107 | // 1. There is at least one inlined element. |
1108 | // 2. `sizeof(SmallVector<T>) <= kPreferredSmallVectorSizeof` unless |
1109 | // it contradicts 1. |
1110 | static constexpr size_t kPreferredSmallVectorSizeof = 64; |
1111 | |
1112 | // static_assert that sizeof(T) is not "too big". |
1113 | // |
1114 | // Because our policy guarantees at least one inlined element, it is possible |
1115 | // for an arbitrarily large inlined element to allocate an arbitrarily large |
1116 | // amount of inline storage. We generally consider it an antipattern for a |
1117 | // SmallVector to allocate an excessive amount of inline storage, so we want |
1118 | // to call attention to these cases and make sure that users are making an |
1119 | // intentional decision if they request a lot of inline storage. |
1120 | // |
1121 | // We want this assertion to trigger in pathological cases, but otherwise |
1122 | // not be too easy to hit. To accomplish that, the cutoff is actually somewhat |
1123 | // larger than kPreferredSmallVectorSizeof (otherwise, |
1124 | // `SmallVector<SmallVector<T>>` would be one easy way to trip it, and that |
1125 | // pattern seems useful in practice). |
1126 | // |
1127 | // One wrinkle is that this assertion is in theory non-portable, since |
1128 | // sizeof(T) is in general platform-dependent. However, we don't expect this |
1129 | // to be much of an issue, because most LLVM development happens on 64-bit |
1130 | // hosts, and therefore sizeof(T) is expected to *decrease* when compiled for |
1131 | // 32-bit hosts, dodging the issue. The reverse situation, where development |
1132 | // happens on a 32-bit host and then fails due to sizeof(T) *increasing* on a |
1133 | // 64-bit host, is expected to be very rare. |
1134 | static_assert( |
1135 | sizeof(T) <= 256, |
1136 | "You are trying to use a default number of inlined elements for " |
1137 | "`SmallVector<T>` but `sizeof(T)` is really big! Please use an " |
1138 | "explicit number of inlined elements with `SmallVector<T, N>` to make " |
1139 | "sure you really want that much inline storage."); |
1140 | |
1141 | // Discount the size of the header itself when calculating the maximum inline |
1142 | // bytes. |
1143 | static constexpr size_t PreferredInlineBytes = |
1144 | kPreferredSmallVectorSizeof - sizeof(SmallVector<T, 0>); |
1145 | static constexpr size_t NumElementsThatFit = PreferredInlineBytes / sizeof(T); |
1146 | static constexpr size_t value = |
1147 | NumElementsThatFit == 0 ? 1 : NumElementsThatFit; |
1148 | }; |
1149 | |
1150 | /// This is a 'vector' (really, a variable-sized array), optimized |
1151 | /// for the case when the array is small. It contains some number of elements |
1152 | /// in-place, which allows it to avoid heap allocation when the actual number of |
1153 | /// elements is below that threshold. This allows normal "small" cases to be |
1154 | /// fast without losing generality for large inputs. |
1155 | /// |
1156 | /// \note |
1157 | /// In the absence of a well-motivated choice for the number of inlined |
1158 | /// elements \p N, it is recommended to use \c SmallVector<T> (that is, |
1159 | /// omitting the \p N). This will choose a default number of inlined elements |
1160 | /// reasonable for allocation on the stack (for example, trying to keep \c |
1161 | /// sizeof(SmallVector<T>) around 64 bytes). |
1162 | /// |
1163 | /// \warning This does not attempt to be exception safe. |
1164 | /// |
1165 | /// \see https://llvm.org/docs/ProgrammersManual.html#llvm-adt-smallvector-h |
1166 | template <typename T, |
1167 | unsigned N = CalculateSmallVectorDefaultInlinedElements<T>::value> |
1168 | class LLVM_GSL_OWNER[[gsl::Owner]] SmallVector : public SmallVectorImpl<T>, |
1169 | SmallVectorStorage<T, N> { |
1170 | public: |
1171 | SmallVector() : SmallVectorImpl<T>(N) {} |
1172 | |
1173 | ~SmallVector() { |
1174 | // Destroy the constructed elements in the vector. |
1175 | this->destroy_range(this->begin(), this->end()); |
1176 | } |
1177 | |
1178 | explicit SmallVector(size_t Size, const T &Value = T()) |
1179 | : SmallVectorImpl<T>(N) { |
1180 | this->assign(Size, Value); |
1181 | } |
1182 | |
1183 | template <typename ItTy, |
1184 | typename = std::enable_if_t<std::is_convertible< |
1185 | typename std::iterator_traits<ItTy>::iterator_category, |
1186 | std::input_iterator_tag>::value>> |
1187 | SmallVector(ItTy S, ItTy E) : SmallVectorImpl<T>(N) { |
1188 | this->append(S, E); |
1189 | } |
1190 | |
1191 | template <typename RangeTy> |
1192 | explicit SmallVector(const iterator_range<RangeTy> &R) |
1193 | : SmallVectorImpl<T>(N) { |
1194 | this->append(R.begin(), R.end()); |
1195 | } |
1196 | |
1197 | SmallVector(std::initializer_list<T> IL) : SmallVectorImpl<T>(N) { |
1198 | this->assign(IL); |
1199 | } |
1200 | |
1201 | SmallVector(const SmallVector &RHS) : SmallVectorImpl<T>(N) { |
1202 | if (!RHS.empty()) |
1203 | SmallVectorImpl<T>::operator=(RHS); |
1204 | } |
1205 | |
1206 | SmallVector &operator=(const SmallVector &RHS) { |
1207 | SmallVectorImpl<T>::operator=(RHS); |
1208 | return *this; |
1209 | } |
1210 | |
1211 | SmallVector(SmallVector &&RHS) : SmallVectorImpl<T>(N) { |
1212 | if (!RHS.empty()) |
1213 | SmallVectorImpl<T>::operator=(::std::move(RHS)); |
1214 | } |
1215 | |
1216 | SmallVector(SmallVectorImpl<T> &&RHS) : SmallVectorImpl<T>(N) { |
1217 | if (!RHS.empty()) |
1218 | SmallVectorImpl<T>::operator=(::std::move(RHS)); |
1219 | } |
1220 | |
1221 | SmallVector &operator=(SmallVector &&RHS) { |
1222 | SmallVectorImpl<T>::operator=(::std::move(RHS)); |
1223 | return *this; |
1224 | } |
1225 | |
1226 | SmallVector &operator=(SmallVectorImpl<T> &&RHS) { |
1227 | SmallVectorImpl<T>::operator=(::std::move(RHS)); |
1228 | return *this; |
1229 | } |
1230 | |
1231 | SmallVector &operator=(std::initializer_list<T> IL) { |
1232 | this->assign(IL); |
1233 | return *this; |
1234 | } |
1235 | }; |
1236 | |
1237 | template <typename T, unsigned N> |
1238 | inline size_t capacity_in_bytes(const SmallVector<T, N> &X) { |
1239 | return X.capacity_in_bytes(); |
1240 | } |
1241 | |
1242 | /// Given a range of type R, iterate the entire range and return a |
1243 | /// SmallVector with elements of the vector. This is useful, for example, |
1244 | /// when you want to iterate a range and then sort the results. |
1245 | template <unsigned Size, typename R> |
1246 | SmallVector<typename std::remove_const<typename std::remove_reference< |
1247 | decltype(*std::begin(std::declval<R &>()))>::type>::type, |
1248 | Size> |
1249 | to_vector(R &&Range) { |
1250 | return {std::begin(Range), std::end(Range)}; |
1251 | } |
1252 | |
1253 | } // end namespace llvm |
1254 | |
1255 | namespace std { |
1256 | |
1257 | /// Implement std::swap in terms of SmallVector swap. |
1258 | template<typename T> |
1259 | inline void |
1260 | swap(llvm::SmallVectorImpl<T> &LHS, llvm::SmallVectorImpl<T> &RHS) { |
1261 | LHS.swap(RHS); |
1262 | } |
1263 | |
1264 | /// Implement std::swap in terms of SmallVector swap. |
1265 | template<typename T, unsigned N> |
1266 | inline void |
1267 | swap(llvm::SmallVector<T, N> &LHS, llvm::SmallVector<T, N> &RHS) { |
1268 | LHS.swap(RHS); |
1269 | } |
1270 | |
1271 | } // end namespace std |
1272 | |
1273 | #endif // LLVM_ADT_SMALLVECTOR_H |