File: | src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/CodeGen/SelectionDAGNodes.h |
Warning: | line 1110, column 10 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This implements routines for translating from LLVM IR into SelectionDAG IR. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "SelectionDAGBuilder.h" |
14 | #include "SDNodeDbgValue.h" |
15 | #include "llvm/ADT/APFloat.h" |
16 | #include "llvm/ADT/APInt.h" |
17 | #include "llvm/ADT/BitVector.h" |
18 | #include "llvm/ADT/None.h" |
19 | #include "llvm/ADT/Optional.h" |
20 | #include "llvm/ADT/STLExtras.h" |
21 | #include "llvm/ADT/SmallPtrSet.h" |
22 | #include "llvm/ADT/SmallSet.h" |
23 | #include "llvm/ADT/StringRef.h" |
24 | #include "llvm/ADT/Triple.h" |
25 | #include "llvm/ADT/Twine.h" |
26 | #include "llvm/Analysis/AliasAnalysis.h" |
27 | #include "llvm/Analysis/BlockFrequencyInfo.h" |
28 | #include "llvm/Analysis/BranchProbabilityInfo.h" |
29 | #include "llvm/Analysis/ConstantFolding.h" |
30 | #include "llvm/Analysis/EHPersonalities.h" |
31 | #include "llvm/Analysis/Loads.h" |
32 | #include "llvm/Analysis/MemoryLocation.h" |
33 | #include "llvm/Analysis/ProfileSummaryInfo.h" |
34 | #include "llvm/Analysis/TargetLibraryInfo.h" |
35 | #include "llvm/Analysis/ValueTracking.h" |
36 | #include "llvm/Analysis/VectorUtils.h" |
37 | #include "llvm/CodeGen/Analysis.h" |
38 | #include "llvm/CodeGen/FunctionLoweringInfo.h" |
39 | #include "llvm/CodeGen/GCMetadata.h" |
40 | #include "llvm/CodeGen/MachineBasicBlock.h" |
41 | #include "llvm/CodeGen/MachineFrameInfo.h" |
42 | #include "llvm/CodeGen/MachineFunction.h" |
43 | #include "llvm/CodeGen/MachineInstr.h" |
44 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
45 | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
46 | #include "llvm/CodeGen/MachineMemOperand.h" |
47 | #include "llvm/CodeGen/MachineModuleInfo.h" |
48 | #include "llvm/CodeGen/MachineOperand.h" |
49 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
50 | #include "llvm/CodeGen/RuntimeLibcalls.h" |
51 | #include "llvm/CodeGen/SelectionDAG.h" |
52 | #include "llvm/CodeGen/SelectionDAGTargetInfo.h" |
53 | #include "llvm/CodeGen/StackMaps.h" |
54 | #include "llvm/CodeGen/SwiftErrorValueTracking.h" |
55 | #include "llvm/CodeGen/TargetFrameLowering.h" |
56 | #include "llvm/CodeGen/TargetInstrInfo.h" |
57 | #include "llvm/CodeGen/TargetOpcodes.h" |
58 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
59 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
60 | #include "llvm/CodeGen/WinEHFuncInfo.h" |
61 | #include "llvm/IR/Argument.h" |
62 | #include "llvm/IR/Attributes.h" |
63 | #include "llvm/IR/BasicBlock.h" |
64 | #include "llvm/IR/CFG.h" |
65 | #include "llvm/IR/CallingConv.h" |
66 | #include "llvm/IR/Constant.h" |
67 | #include "llvm/IR/ConstantRange.h" |
68 | #include "llvm/IR/Constants.h" |
69 | #include "llvm/IR/DataLayout.h" |
70 | #include "llvm/IR/DebugInfoMetadata.h" |
71 | #include "llvm/IR/DerivedTypes.h" |
72 | #include "llvm/IR/Function.h" |
73 | #include "llvm/IR/GetElementPtrTypeIterator.h" |
74 | #include "llvm/IR/InlineAsm.h" |
75 | #include "llvm/IR/InstrTypes.h" |
76 | #include "llvm/IR/Instructions.h" |
77 | #include "llvm/IR/IntrinsicInst.h" |
78 | #include "llvm/IR/Intrinsics.h" |
79 | #include "llvm/IR/IntrinsicsAArch64.h" |
80 | #include "llvm/IR/IntrinsicsWebAssembly.h" |
81 | #include "llvm/IR/LLVMContext.h" |
82 | #include "llvm/IR/Metadata.h" |
83 | #include "llvm/IR/Module.h" |
84 | #include "llvm/IR/Operator.h" |
85 | #include "llvm/IR/PatternMatch.h" |
86 | #include "llvm/IR/Statepoint.h" |
87 | #include "llvm/IR/Type.h" |
88 | #include "llvm/IR/User.h" |
89 | #include "llvm/IR/Value.h" |
90 | #include "llvm/MC/MCContext.h" |
91 | #include "llvm/MC/MCSymbol.h" |
92 | #include "llvm/Support/AtomicOrdering.h" |
93 | #include "llvm/Support/Casting.h" |
94 | #include "llvm/Support/CommandLine.h" |
95 | #include "llvm/Support/Compiler.h" |
96 | #include "llvm/Support/Debug.h" |
97 | #include "llvm/Support/MathExtras.h" |
98 | #include "llvm/Support/raw_ostream.h" |
99 | #include "llvm/Target/TargetIntrinsicInfo.h" |
100 | #include "llvm/Target/TargetMachine.h" |
101 | #include "llvm/Target/TargetOptions.h" |
102 | #include "llvm/Transforms/Utils/Local.h" |
103 | #include <cstddef> |
104 | #include <cstring> |
105 | #include <iterator> |
106 | #include <limits> |
107 | #include <numeric> |
108 | #include <tuple> |
109 | |
110 | using namespace llvm; |
111 | using namespace PatternMatch; |
112 | using namespace SwitchCG; |
113 | |
114 | #define DEBUG_TYPE"isel" "isel" |
115 | |
116 | /// LimitFloatPrecision - Generate low-precision inline sequences for |
117 | /// some float libcalls (6, 8 or 12 bits). |
118 | static unsigned LimitFloatPrecision; |
119 | |
120 | static cl::opt<bool> |
121 | InsertAssertAlign("insert-assert-align", cl::init(true), |
122 | cl::desc("Insert the experimental `assertalign` node."), |
123 | cl::ReallyHidden); |
124 | |
125 | static cl::opt<unsigned, true> |
126 | LimitFPPrecision("limit-float-precision", |
127 | cl::desc("Generate low-precision inline sequences " |
128 | "for some float libcalls"), |
129 | cl::location(LimitFloatPrecision), cl::Hidden, |
130 | cl::init(0)); |
131 | |
132 | static cl::opt<unsigned> SwitchPeelThreshold( |
133 | "switch-peel-threshold", cl::Hidden, cl::init(66), |
134 | cl::desc("Set the case probability threshold for peeling the case from a " |
135 | "switch statement. A value greater than 100 will void this " |
136 | "optimization")); |
137 | |
138 | // Limit the width of DAG chains. This is important in general to prevent |
139 | // DAG-based analysis from blowing up. For example, alias analysis and |
140 | // load clustering may not complete in reasonable time. It is difficult to |
141 | // recognize and avoid this situation within each individual analysis, and |
142 | // future analyses are likely to have the same behavior. Limiting DAG width is |
143 | // the safe approach and will be especially important with global DAGs. |
144 | // |
145 | // MaxParallelChains default is arbitrarily high to avoid affecting |
146 | // optimization, but could be lowered to improve compile time. Any ld-ld-st-st |
147 | // sequence over this should have been converted to llvm.memcpy by the |
148 | // frontend. It is easy to induce this behavior with .ll code such as: |
149 | // %buffer = alloca [4096 x i8] |
150 | // %data = load [4096 x i8]* %argPtr |
151 | // store [4096 x i8] %data, [4096 x i8]* %buffer |
152 | static const unsigned MaxParallelChains = 64; |
153 | |
154 | static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, |
155 | const SDValue *Parts, unsigned NumParts, |
156 | MVT PartVT, EVT ValueVT, const Value *V, |
157 | Optional<CallingConv::ID> CC); |
158 | |
159 | /// getCopyFromParts - Create a value that contains the specified legal parts |
160 | /// combined into the value they represent. If the parts combine to a type |
161 | /// larger than ValueVT then AssertOp can be used to specify whether the extra |
162 | /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT |
163 | /// (ISD::AssertSext). |
164 | static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, |
165 | const SDValue *Parts, unsigned NumParts, |
166 | MVT PartVT, EVT ValueVT, const Value *V, |
167 | Optional<CallingConv::ID> CC = None, |
168 | Optional<ISD::NodeType> AssertOp = None) { |
169 | // Let the target assemble the parts if it wants to |
170 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
171 | if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts, |
172 | PartVT, ValueVT, CC)) |
173 | return Val; |
174 | |
175 | if (ValueVT.isVector()) |
176 | return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V, |
177 | CC); |
178 | |
179 | assert(NumParts > 0 && "No parts to assemble!")((void)0); |
180 | SDValue Val = Parts[0]; |
181 | |
182 | if (NumParts > 1) { |
183 | // Assemble the value from multiple parts. |
184 | if (ValueVT.isInteger()) { |
185 | unsigned PartBits = PartVT.getSizeInBits(); |
186 | unsigned ValueBits = ValueVT.getSizeInBits(); |
187 | |
188 | // Assemble the power of 2 part. |
189 | unsigned RoundParts = |
190 | (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts; |
191 | unsigned RoundBits = PartBits * RoundParts; |
192 | EVT RoundVT = RoundBits == ValueBits ? |
193 | ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); |
194 | SDValue Lo, Hi; |
195 | |
196 | EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); |
197 | |
198 | if (RoundParts > 2) { |
199 | Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, |
200 | PartVT, HalfVT, V); |
201 | Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, |
202 | RoundParts / 2, PartVT, HalfVT, V); |
203 | } else { |
204 | Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); |
205 | Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); |
206 | } |
207 | |
208 | if (DAG.getDataLayout().isBigEndian()) |
209 | std::swap(Lo, Hi); |
210 | |
211 | Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); |
212 | |
213 | if (RoundParts < NumParts) { |
214 | // Assemble the trailing non-power-of-2 part. |
215 | unsigned OddParts = NumParts - RoundParts; |
216 | EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); |
217 | Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT, |
218 | OddVT, V, CC); |
219 | |
220 | // Combine the round and odd parts. |
221 | Lo = Val; |
222 | if (DAG.getDataLayout().isBigEndian()) |
223 | std::swap(Lo, Hi); |
224 | EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
225 | Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); |
226 | Hi = |
227 | DAG.getNode(ISD::SHL, DL, TotalVT, Hi, |
228 | DAG.getConstant(Lo.getValueSizeInBits(), DL, |
229 | TLI.getPointerTy(DAG.getDataLayout()))); |
230 | Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); |
231 | Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); |
232 | } |
233 | } else if (PartVT.isFloatingPoint()) { |
234 | // FP split into multiple FP parts (for ppcf128) |
235 | assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&((void)0) |
236 | "Unexpected split")((void)0); |
237 | SDValue Lo, Hi; |
238 | Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); |
239 | Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); |
240 | if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) |
241 | std::swap(Lo, Hi); |
242 | Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); |
243 | } else { |
244 | // FP split into integer parts (soft fp) |
245 | assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&((void)0) |
246 | !PartVT.isVector() && "Unexpected split")((void)0); |
247 | EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); |
248 | Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC); |
249 | } |
250 | } |
251 | |
252 | // There is now one part, held in Val. Correct it to match ValueVT. |
253 | // PartEVT is the type of the register class that holds the value. |
254 | // ValueVT is the type of the inline asm operation. |
255 | EVT PartEVT = Val.getValueType(); |
256 | |
257 | if (PartEVT == ValueVT) |
258 | return Val; |
259 | |
260 | if (PartEVT.isInteger() && ValueVT.isFloatingPoint() && |
261 | ValueVT.bitsLT(PartEVT)) { |
262 | // For an FP value in an integer part, we need to truncate to the right |
263 | // width first. |
264 | PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); |
265 | Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val); |
266 | } |
267 | |
268 | // Handle types that have the same size. |
269 | if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) |
270 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
271 | |
272 | // Handle types with different sizes. |
273 | if (PartEVT.isInteger() && ValueVT.isInteger()) { |
274 | if (ValueVT.bitsLT(PartEVT)) { |
275 | // For a truncate, see if we have any information to |
276 | // indicate whether the truncated bits will always be |
277 | // zero or sign-extension. |
278 | if (AssertOp.hasValue()) |
279 | Val = DAG.getNode(*AssertOp, DL, PartEVT, Val, |
280 | DAG.getValueType(ValueVT)); |
281 | return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); |
282 | } |
283 | return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); |
284 | } |
285 | |
286 | if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { |
287 | // FP_ROUND's are always exact here. |
288 | if (ValueVT.bitsLT(Val.getValueType())) |
289 | return DAG.getNode( |
290 | ISD::FP_ROUND, DL, ValueVT, Val, |
291 | DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()))); |
292 | |
293 | return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); |
294 | } |
295 | |
296 | // Handle MMX to a narrower integer type by bitcasting MMX to integer and |
297 | // then truncating. |
298 | if (PartEVT == MVT::x86mmx && ValueVT.isInteger() && |
299 | ValueVT.bitsLT(PartEVT)) { |
300 | Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val); |
301 | return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); |
302 | } |
303 | |
304 | report_fatal_error("Unknown mismatch in getCopyFromParts!"); |
305 | } |
306 | |
307 | static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, |
308 | const Twine &ErrMsg) { |
309 | const Instruction *I = dyn_cast_or_null<Instruction>(V); |
310 | if (!V) |
311 | return Ctx.emitError(ErrMsg); |
312 | |
313 | const char *AsmError = ", possible invalid constraint for vector type"; |
314 | if (const CallInst *CI = dyn_cast<CallInst>(I)) |
315 | if (CI->isInlineAsm()) |
316 | return Ctx.emitError(I, ErrMsg + AsmError); |
317 | |
318 | return Ctx.emitError(I, ErrMsg); |
319 | } |
320 | |
321 | /// getCopyFromPartsVector - Create a value that contains the specified legal |
322 | /// parts combined into the value they represent. If the parts combine to a |
323 | /// type larger than ValueVT then AssertOp can be used to specify whether the |
324 | /// extra bits are known to be zero (ISD::AssertZext) or sign extended from |
325 | /// ValueVT (ISD::AssertSext). |
326 | static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, |
327 | const SDValue *Parts, unsigned NumParts, |
328 | MVT PartVT, EVT ValueVT, const Value *V, |
329 | Optional<CallingConv::ID> CallConv) { |
330 | assert(ValueVT.isVector() && "Not a vector value")((void)0); |
331 | assert(NumParts > 0 && "No parts to assemble!")((void)0); |
332 | const bool IsABIRegCopy = CallConv.hasValue(); |
333 | |
334 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
335 | SDValue Val = Parts[0]; |
336 | |
337 | // Handle a multi-element vector. |
338 | if (NumParts > 1) { |
339 | EVT IntermediateVT; |
340 | MVT RegisterVT; |
341 | unsigned NumIntermediates; |
342 | unsigned NumRegs; |
343 | |
344 | if (IsABIRegCopy) { |
345 | NumRegs = TLI.getVectorTypeBreakdownForCallingConv( |
346 | *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT, |
347 | NumIntermediates, RegisterVT); |
348 | } else { |
349 | NumRegs = |
350 | TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, |
351 | NumIntermediates, RegisterVT); |
352 | } |
353 | |
354 | assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((void)0); |
355 | NumParts = NumRegs; // Silence a compiler warning. |
356 | assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((void)0); |
357 | assert(RegisterVT.getSizeInBits() ==((void)0) |
358 | Parts[0].getSimpleValueType().getSizeInBits() &&((void)0) |
359 | "Part type sizes don't match!")((void)0); |
360 | |
361 | // Assemble the parts into intermediate operands. |
362 | SmallVector<SDValue, 8> Ops(NumIntermediates); |
363 | if (NumIntermediates == NumParts) { |
364 | // If the register was not expanded, truncate or copy the value, |
365 | // as appropriate. |
366 | for (unsigned i = 0; i != NumParts; ++i) |
367 | Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, |
368 | PartVT, IntermediateVT, V, CallConv); |
369 | } else if (NumParts > 0) { |
370 | // If the intermediate type was expanded, build the intermediate |
371 | // operands from the parts. |
372 | assert(NumParts % NumIntermediates == 0 &&((void)0) |
373 | "Must expand into a divisible number of parts!")((void)0); |
374 | unsigned Factor = NumParts / NumIntermediates; |
375 | for (unsigned i = 0; i != NumIntermediates; ++i) |
376 | Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, |
377 | PartVT, IntermediateVT, V, CallConv); |
378 | } |
379 | |
380 | // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the |
381 | // intermediate operands. |
382 | EVT BuiltVectorTy = |
383 | IntermediateVT.isVector() |
384 | ? EVT::getVectorVT( |
385 | *DAG.getContext(), IntermediateVT.getScalarType(), |
386 | IntermediateVT.getVectorElementCount() * NumParts) |
387 | : EVT::getVectorVT(*DAG.getContext(), |
388 | IntermediateVT.getScalarType(), |
389 | NumIntermediates); |
390 | Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS |
391 | : ISD::BUILD_VECTOR, |
392 | DL, BuiltVectorTy, Ops); |
393 | } |
394 | |
395 | // There is now one part, held in Val. Correct it to match ValueVT. |
396 | EVT PartEVT = Val.getValueType(); |
397 | |
398 | if (PartEVT == ValueVT) |
399 | return Val; |
400 | |
401 | if (PartEVT.isVector()) { |
402 | // If the element type of the source/dest vectors are the same, but the |
403 | // parts vector has more elements than the value vector, then we have a |
404 | // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the |
405 | // elements we want. |
406 | if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) { |
407 | assert((PartEVT.getVectorElementCount().getKnownMinValue() >((void)0) |
408 | ValueVT.getVectorElementCount().getKnownMinValue()) &&((void)0) |
409 | (PartEVT.getVectorElementCount().isScalable() ==((void)0) |
410 | ValueVT.getVectorElementCount().isScalable()) &&((void)0) |
411 | "Cannot narrow, it would be a lossy transformation")((void)0); |
412 | return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, |
413 | DAG.getVectorIdxConstant(0, DL)); |
414 | } |
415 | |
416 | // Vector/Vector bitcast. |
417 | if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) |
418 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
419 | |
420 | assert(PartEVT.getVectorElementCount() == ValueVT.getVectorElementCount() &&((void)0) |
421 | "Cannot handle this kind of promotion")((void)0); |
422 | // Promoted vector extract |
423 | return DAG.getAnyExtOrTrunc(Val, DL, ValueVT); |
424 | |
425 | } |
426 | |
427 | // Trivial bitcast if the types are the same size and the destination |
428 | // vector type is legal. |
429 | if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && |
430 | TLI.isTypeLegal(ValueVT)) |
431 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
432 | |
433 | if (ValueVT.getVectorNumElements() != 1) { |
434 | // Certain ABIs require that vectors are passed as integers. For vectors |
435 | // are the same size, this is an obvious bitcast. |
436 | if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) { |
437 | return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
438 | } else if (ValueVT.bitsLT(PartEVT)) { |
439 | const uint64_t ValueSize = ValueVT.getFixedSizeInBits(); |
440 | EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); |
441 | // Drop the extra bits. |
442 | Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); |
443 | return DAG.getBitcast(ValueVT, Val); |
444 | } |
445 | |
446 | diagnosePossiblyInvalidConstraint( |
447 | *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); |
448 | return DAG.getUNDEF(ValueVT); |
449 | } |
450 | |
451 | // Handle cases such as i8 -> <1 x i1> |
452 | EVT ValueSVT = ValueVT.getVectorElementType(); |
453 | if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) { |
454 | if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits()) |
455 | Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val); |
456 | else |
457 | Val = ValueVT.isFloatingPoint() |
458 | ? DAG.getFPExtendOrRound(Val, DL, ValueSVT) |
459 | : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT); |
460 | } |
461 | |
462 | return DAG.getBuildVector(ValueVT, DL, Val); |
463 | } |
464 | |
465 | static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, |
466 | SDValue Val, SDValue *Parts, unsigned NumParts, |
467 | MVT PartVT, const Value *V, |
468 | Optional<CallingConv::ID> CallConv); |
469 | |
470 | /// getCopyToParts - Create a series of nodes that contain the specified value |
471 | /// split into legal parts. If the parts contain more bits than Val, then, for |
472 | /// integers, ExtendKind can be used to specify how to generate the extra bits. |
473 | static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, |
474 | SDValue *Parts, unsigned NumParts, MVT PartVT, |
475 | const Value *V, |
476 | Optional<CallingConv::ID> CallConv = None, |
477 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { |
478 | // Let the target split the parts if it wants to |
479 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
480 | if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT, |
481 | CallConv)) |
482 | return; |
483 | EVT ValueVT = Val.getValueType(); |
484 | |
485 | // Handle the vector case separately. |
486 | if (ValueVT.isVector()) |
487 | return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V, |
488 | CallConv); |
489 | |
490 | unsigned PartBits = PartVT.getSizeInBits(); |
491 | unsigned OrigNumParts = NumParts; |
492 | assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&((void)0) |
493 | "Copying to an illegal type!")((void)0); |
494 | |
495 | if (NumParts == 0) |
496 | return; |
497 | |
498 | assert(!ValueVT.isVector() && "Vector case handled elsewhere")((void)0); |
499 | EVT PartEVT = PartVT; |
500 | if (PartEVT == ValueVT) { |
501 | assert(NumParts == 1 && "No-op copy with multiple parts!")((void)0); |
502 | Parts[0] = Val; |
503 | return; |
504 | } |
505 | |
506 | if (NumParts * PartBits > ValueVT.getSizeInBits()) { |
507 | // If the parts cover more bits than the value has, promote the value. |
508 | if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { |
509 | assert(NumParts == 1 && "Do not know what to promote to!")((void)0); |
510 | Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); |
511 | } else { |
512 | if (ValueVT.isFloatingPoint()) { |
513 | // FP values need to be bitcast, then extended if they are being put |
514 | // into a larger container. |
515 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); |
516 | Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
517 | } |
518 | assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&((void)0) |
519 | ValueVT.isInteger() &&((void)0) |
520 | "Unknown mismatch!")((void)0); |
521 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
522 | Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); |
523 | if (PartVT == MVT::x86mmx) |
524 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
525 | } |
526 | } else if (PartBits == ValueVT.getSizeInBits()) { |
527 | // Different types of the same size. |
528 | assert(NumParts == 1 && PartEVT != ValueVT)((void)0); |
529 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
530 | } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { |
531 | // If the parts cover less bits than value has, truncate the value. |
532 | assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&((void)0) |
533 | ValueVT.isInteger() &&((void)0) |
534 | "Unknown mismatch!")((void)0); |
535 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
536 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); |
537 | if (PartVT == MVT::x86mmx) |
538 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
539 | } |
540 | |
541 | // The value may have changed - recompute ValueVT. |
542 | ValueVT = Val.getValueType(); |
543 | assert(NumParts * PartBits == ValueVT.getSizeInBits() &&((void)0) |
544 | "Failed to tile the value with PartVT!")((void)0); |
545 | |
546 | if (NumParts == 1) { |
547 | if (PartEVT != ValueVT) { |
548 | diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, |
549 | "scalar-to-vector conversion failed"); |
550 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
551 | } |
552 | |
553 | Parts[0] = Val; |
554 | return; |
555 | } |
556 | |
557 | // Expand the value into multiple parts. |
558 | if (NumParts & (NumParts - 1)) { |
559 | // The number of parts is not a power of 2. Split off and copy the tail. |
560 | assert(PartVT.isInteger() && ValueVT.isInteger() &&((void)0) |
561 | "Do not know what to expand to!")((void)0); |
562 | unsigned RoundParts = 1 << Log2_32(NumParts); |
563 | unsigned RoundBits = RoundParts * PartBits; |
564 | unsigned OddParts = NumParts - RoundParts; |
565 | SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, |
566 | DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false)); |
567 | |
568 | getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V, |
569 | CallConv); |
570 | |
571 | if (DAG.getDataLayout().isBigEndian()) |
572 | // The odd parts were reversed by getCopyToParts - unreverse them. |
573 | std::reverse(Parts + RoundParts, Parts + NumParts); |
574 | |
575 | NumParts = RoundParts; |
576 | ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
577 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); |
578 | } |
579 | |
580 | // The number of parts is a power of 2. Repeatedly bisect the value using |
581 | // EXTRACT_ELEMENT. |
582 | Parts[0] = DAG.getNode(ISD::BITCAST, DL, |
583 | EVT::getIntegerVT(*DAG.getContext(), |
584 | ValueVT.getSizeInBits()), |
585 | Val); |
586 | |
587 | for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { |
588 | for (unsigned i = 0; i < NumParts; i += StepSize) { |
589 | unsigned ThisBits = StepSize * PartBits / 2; |
590 | EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); |
591 | SDValue &Part0 = Parts[i]; |
592 | SDValue &Part1 = Parts[i+StepSize/2]; |
593 | |
594 | Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, |
595 | ThisVT, Part0, DAG.getIntPtrConstant(1, DL)); |
596 | Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, |
597 | ThisVT, Part0, DAG.getIntPtrConstant(0, DL)); |
598 | |
599 | if (ThisBits == PartBits && ThisVT != PartVT) { |
600 | Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); |
601 | Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); |
602 | } |
603 | } |
604 | } |
605 | |
606 | if (DAG.getDataLayout().isBigEndian()) |
607 | std::reverse(Parts, Parts + OrigNumParts); |
608 | } |
609 | |
610 | static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, |
611 | const SDLoc &DL, EVT PartVT) { |
612 | if (!PartVT.isVector()) |
613 | return SDValue(); |
614 | |
615 | EVT ValueVT = Val.getValueType(); |
616 | ElementCount PartNumElts = PartVT.getVectorElementCount(); |
617 | ElementCount ValueNumElts = ValueVT.getVectorElementCount(); |
618 | |
619 | // We only support widening vectors with equivalent element types and |
620 | // fixed/scalable properties. If a target needs to widen a fixed-length type |
621 | // to a scalable one, it should be possible to use INSERT_SUBVECTOR below. |
622 | if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) || |
623 | PartNumElts.isScalable() != ValueNumElts.isScalable() || |
624 | PartVT.getVectorElementType() != ValueVT.getVectorElementType()) |
625 | return SDValue(); |
626 | |
627 | // Widening a scalable vector to another scalable vector is done by inserting |
628 | // the vector into a larger undef one. |
629 | if (PartNumElts.isScalable()) |
630 | return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), |
631 | Val, DAG.getVectorIdxConstant(0, DL)); |
632 | |
633 | EVT ElementVT = PartVT.getVectorElementType(); |
634 | // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in |
635 | // undef elements. |
636 | SmallVector<SDValue, 16> Ops; |
637 | DAG.ExtractVectorElements(Val, Ops); |
638 | SDValue EltUndef = DAG.getUNDEF(ElementVT); |
639 | Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef); |
640 | |
641 | // FIXME: Use CONCAT for 2x -> 4x. |
642 | return DAG.getBuildVector(PartVT, DL, Ops); |
643 | } |
644 | |
645 | /// getCopyToPartsVector - Create a series of nodes that contain the specified |
646 | /// value split into legal parts. |
647 | static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL, |
648 | SDValue Val, SDValue *Parts, unsigned NumParts, |
649 | MVT PartVT, const Value *V, |
650 | Optional<CallingConv::ID> CallConv) { |
651 | EVT ValueVT = Val.getValueType(); |
652 | assert(ValueVT.isVector() && "Not a vector")((void)0); |
653 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
654 | const bool IsABIRegCopy = CallConv.hasValue(); |
655 | |
656 | if (NumParts == 1) { |
657 | EVT PartEVT = PartVT; |
658 | if (PartEVT == ValueVT) { |
659 | // Nothing to do. |
660 | } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { |
661 | // Bitconvert vector->vector case. |
662 | Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
663 | } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) { |
664 | Val = Widened; |
665 | } else if (PartVT.isVector() && |
666 | PartEVT.getVectorElementType().bitsGE( |
667 | ValueVT.getVectorElementType()) && |
668 | PartEVT.getVectorElementCount() == |
669 | ValueVT.getVectorElementCount()) { |
670 | |
671 | // Promoted vector extract |
672 | Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); |
673 | } else { |
674 | if (ValueVT.getVectorElementCount().isScalar()) { |
675 | Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val, |
676 | DAG.getVectorIdxConstant(0, DL)); |
677 | } else { |
678 | uint64_t ValueSize = ValueVT.getFixedSizeInBits(); |
679 | assert(PartVT.getFixedSizeInBits() > ValueSize &&((void)0) |
680 | "lossy conversion of vector to scalar type")((void)0); |
681 | EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); |
682 | Val = DAG.getBitcast(IntermediateType, Val); |
683 | Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); |
684 | } |
685 | } |
686 | |
687 | assert(Val.getValueType() == PartVT && "Unexpected vector part value type")((void)0); |
688 | Parts[0] = Val; |
689 | return; |
690 | } |
691 | |
692 | // Handle a multi-element vector. |
693 | EVT IntermediateVT; |
694 | MVT RegisterVT; |
695 | unsigned NumIntermediates; |
696 | unsigned NumRegs; |
697 | if (IsABIRegCopy) { |
698 | NumRegs = TLI.getVectorTypeBreakdownForCallingConv( |
699 | *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT, |
700 | NumIntermediates, RegisterVT); |
701 | } else { |
702 | NumRegs = |
703 | TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, |
704 | NumIntermediates, RegisterVT); |
705 | } |
706 | |
707 | assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((void)0); |
708 | NumParts = NumRegs; // Silence a compiler warning. |
709 | assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((void)0); |
710 | |
711 | assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&((void)0) |
712 | "Mixing scalable and fixed vectors when copying in parts")((void)0); |
713 | |
714 | Optional<ElementCount> DestEltCnt; |
715 | |
716 | if (IntermediateVT.isVector()) |
717 | DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates; |
718 | else |
719 | DestEltCnt = ElementCount::getFixed(NumIntermediates); |
720 | |
721 | EVT BuiltVectorTy = EVT::getVectorVT( |
722 | *DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue()); |
723 | |
724 | if (ValueVT == BuiltVectorTy) { |
725 | // Nothing to do. |
726 | } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) { |
727 | // Bitconvert vector->vector case. |
728 | Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val); |
729 | } else if (SDValue Widened = |
730 | widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) { |
731 | Val = Widened; |
732 | } else if (BuiltVectorTy.getVectorElementType().bitsGE( |
733 | ValueVT.getVectorElementType()) && |
734 | BuiltVectorTy.getVectorElementCount() == |
735 | ValueVT.getVectorElementCount()) { |
736 | // Promoted vector extract |
737 | Val = DAG.getAnyExtOrTrunc(Val, DL, BuiltVectorTy); |
738 | } |
739 | |
740 | assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")((void)0); |
741 | |
742 | // Split the vector into intermediate operands. |
743 | SmallVector<SDValue, 8> Ops(NumIntermediates); |
744 | for (unsigned i = 0; i != NumIntermediates; ++i) { |
745 | if (IntermediateVT.isVector()) { |
746 | // This does something sensible for scalable vectors - see the |
747 | // definition of EXTRACT_SUBVECTOR for further details. |
748 | unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements(); |
749 | Ops[i] = |
750 | DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val, |
751 | DAG.getVectorIdxConstant(i * IntermediateNumElts, DL)); |
752 | } else { |
753 | Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val, |
754 | DAG.getVectorIdxConstant(i, DL)); |
755 | } |
756 | } |
757 | |
758 | // Split the intermediate operands into legal parts. |
759 | if (NumParts == NumIntermediates) { |
760 | // If the register was not expanded, promote or copy the value, |
761 | // as appropriate. |
762 | for (unsigned i = 0; i != NumParts; ++i) |
763 | getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv); |
764 | } else if (NumParts > 0) { |
765 | // If the intermediate type was expanded, split each the value into |
766 | // legal parts. |
767 | assert(NumIntermediates != 0 && "division by zero")((void)0); |
768 | assert(NumParts % NumIntermediates == 0 &&((void)0) |
769 | "Must expand into a divisible number of parts!")((void)0); |
770 | unsigned Factor = NumParts / NumIntermediates; |
771 | for (unsigned i = 0; i != NumIntermediates; ++i) |
772 | getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V, |
773 | CallConv); |
774 | } |
775 | } |
776 | |
777 | RegsForValue::RegsForValue(const SmallVector<unsigned, 4> ®s, MVT regvt, |
778 | EVT valuevt, Optional<CallingConv::ID> CC) |
779 | : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs), |
780 | RegCount(1, regs.size()), CallConv(CC) {} |
781 | |
782 | RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI, |
783 | const DataLayout &DL, unsigned Reg, Type *Ty, |
784 | Optional<CallingConv::ID> CC) { |
785 | ComputeValueVTs(TLI, DL, Ty, ValueVTs); |
786 | |
787 | CallConv = CC; |
788 | |
789 | for (EVT ValueVT : ValueVTs) { |
790 | unsigned NumRegs = |
791 | isABIMangled() |
792 | ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT) |
793 | : TLI.getNumRegisters(Context, ValueVT); |
794 | MVT RegisterVT = |
795 | isABIMangled() |
796 | ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT) |
797 | : TLI.getRegisterType(Context, ValueVT); |
798 | for (unsigned i = 0; i != NumRegs; ++i) |
799 | Regs.push_back(Reg + i); |
800 | RegVTs.push_back(RegisterVT); |
801 | RegCount.push_back(NumRegs); |
802 | Reg += NumRegs; |
803 | } |
804 | } |
805 | |
806 | SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, |
807 | FunctionLoweringInfo &FuncInfo, |
808 | const SDLoc &dl, SDValue &Chain, |
809 | SDValue *Flag, const Value *V) const { |
810 | // A Value with type {} or [0 x %t] needs no registers. |
811 | if (ValueVTs.empty()) |
812 | return SDValue(); |
813 | |
814 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
815 | |
816 | // Assemble the legal parts into the final values. |
817 | SmallVector<SDValue, 4> Values(ValueVTs.size()); |
818 | SmallVector<SDValue, 8> Parts; |
819 | for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { |
820 | // Copy the legal parts from the registers. |
821 | EVT ValueVT = ValueVTs[Value]; |
822 | unsigned NumRegs = RegCount[Value]; |
823 | MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv( |
824 | *DAG.getContext(), |
825 | CallConv.getValue(), RegVTs[Value]) |
826 | : RegVTs[Value]; |
827 | |
828 | Parts.resize(NumRegs); |
829 | for (unsigned i = 0; i != NumRegs; ++i) { |
830 | SDValue P; |
831 | if (!Flag) { |
832 | P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); |
833 | } else { |
834 | P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); |
835 | *Flag = P.getValue(2); |
836 | } |
837 | |
838 | Chain = P.getValue(1); |
839 | Parts[i] = P; |
840 | |
841 | // If the source register was virtual and if we know something about it, |
842 | // add an assert node. |
843 | if (!Register::isVirtualRegister(Regs[Part + i]) || |
844 | !RegisterVT.isInteger()) |
845 | continue; |
846 | |
847 | const FunctionLoweringInfo::LiveOutInfo *LOI = |
848 | FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); |
849 | if (!LOI) |
850 | continue; |
851 | |
852 | unsigned RegSize = RegisterVT.getScalarSizeInBits(); |
853 | unsigned NumSignBits = LOI->NumSignBits; |
854 | unsigned NumZeroBits = LOI->Known.countMinLeadingZeros(); |
855 | |
856 | if (NumZeroBits == RegSize) { |
857 | // The current value is a zero. |
858 | // Explicitly express that as it would be easier for |
859 | // optimizations to kick in. |
860 | Parts[i] = DAG.getConstant(0, dl, RegisterVT); |
861 | continue; |
862 | } |
863 | |
864 | // FIXME: We capture more information than the dag can represent. For |
865 | // now, just use the tightest assertzext/assertsext possible. |
866 | bool isSExt; |
867 | EVT FromVT(MVT::Other); |
868 | if (NumZeroBits) { |
869 | FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits); |
870 | isSExt = false; |
871 | } else if (NumSignBits > 1) { |
872 | FromVT = |
873 | EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1); |
874 | isSExt = true; |
875 | } else { |
876 | continue; |
877 | } |
878 | // Add an assertion node. |
879 | assert(FromVT != MVT::Other)((void)0); |
880 | Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, |
881 | RegisterVT, P, DAG.getValueType(FromVT)); |
882 | } |
883 | |
884 | Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs, |
885 | RegisterVT, ValueVT, V, CallConv); |
886 | Part += NumRegs; |
887 | Parts.clear(); |
888 | } |
889 | |
890 | return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values); |
891 | } |
892 | |
893 | void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, |
894 | const SDLoc &dl, SDValue &Chain, SDValue *Flag, |
895 | const Value *V, |
896 | ISD::NodeType PreferredExtendType) const { |
897 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
898 | ISD::NodeType ExtendKind = PreferredExtendType; |
899 | |
900 | // Get the list of the values's legal parts. |
901 | unsigned NumRegs = Regs.size(); |
902 | SmallVector<SDValue, 8> Parts(NumRegs); |
903 | for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { |
904 | unsigned NumParts = RegCount[Value]; |
905 | |
906 | MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv( |
907 | *DAG.getContext(), |
908 | CallConv.getValue(), RegVTs[Value]) |
909 | : RegVTs[Value]; |
910 | |
911 | if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT)) |
912 | ExtendKind = ISD::ZERO_EXTEND; |
913 | |
914 | getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part], |
915 | NumParts, RegisterVT, V, CallConv, ExtendKind); |
916 | Part += NumParts; |
917 | } |
918 | |
919 | // Copy the parts into the registers. |
920 | SmallVector<SDValue, 8> Chains(NumRegs); |
921 | for (unsigned i = 0; i != NumRegs; ++i) { |
922 | SDValue Part; |
923 | if (!Flag) { |
924 | Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); |
925 | } else { |
926 | Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); |
927 | *Flag = Part.getValue(1); |
928 | } |
929 | |
930 | Chains[i] = Part.getValue(0); |
931 | } |
932 | |
933 | if (NumRegs == 1 || Flag) |
934 | // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is |
935 | // flagged to it. That is the CopyToReg nodes and the user are considered |
936 | // a single scheduling unit. If we create a TokenFactor and return it as |
937 | // chain, then the TokenFactor is both a predecessor (operand) of the |
938 | // user as well as a successor (the TF operands are flagged to the user). |
939 | // c1, f1 = CopyToReg |
940 | // c2, f2 = CopyToReg |
941 | // c3 = TokenFactor c1, c2 |
942 | // ... |
943 | // = op c3, ..., f2 |
944 | Chain = Chains[NumRegs-1]; |
945 | else |
946 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); |
947 | } |
948 | |
949 | void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, |
950 | unsigned MatchingIdx, const SDLoc &dl, |
951 | SelectionDAG &DAG, |
952 | std::vector<SDValue> &Ops) const { |
953 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
954 | |
955 | unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); |
956 | if (HasMatching) |
957 | Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); |
958 | else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) { |
959 | // Put the register class of the virtual registers in the flag word. That |
960 | // way, later passes can recompute register class constraints for inline |
961 | // assembly as well as normal instructions. |
962 | // Don't do this for tied operands that can use the regclass information |
963 | // from the def. |
964 | const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); |
965 | const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); |
966 | Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); |
967 | } |
968 | |
969 | SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32); |
970 | Ops.push_back(Res); |
971 | |
972 | if (Code == InlineAsm::Kind_Clobber) { |
973 | // Clobbers should always have a 1:1 mapping with registers, and may |
974 | // reference registers that have illegal (e.g. vector) types. Hence, we |
975 | // shouldn't try to apply any sort of splitting logic to them. |
976 | assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&((void)0) |
977 | "No 1:1 mapping from clobbers to regs?")((void)0); |
978 | Register SP = TLI.getStackPointerRegisterToSaveRestore(); |
979 | (void)SP; |
980 | for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) { |
981 | Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I])); |
982 | assert(((void)0) |
983 | (Regs[I] != SP ||((void)0) |
984 | DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&((void)0) |
985 | "If we clobbered the stack pointer, MFI should know about it.")((void)0); |
986 | } |
987 | return; |
988 | } |
989 | |
990 | for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { |
991 | MVT RegisterVT = RegVTs[Value]; |
992 | unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], |
993 | RegisterVT); |
994 | for (unsigned i = 0; i != NumRegs; ++i) { |
995 | assert(Reg < Regs.size() && "Mismatch in # registers expected")((void)0); |
996 | unsigned TheReg = Regs[Reg++]; |
997 | Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); |
998 | } |
999 | } |
1000 | } |
1001 | |
1002 | SmallVector<std::pair<unsigned, TypeSize>, 4> |
1003 | RegsForValue::getRegsAndSizes() const { |
1004 | SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec; |
1005 | unsigned I = 0; |
1006 | for (auto CountAndVT : zip_first(RegCount, RegVTs)) { |
1007 | unsigned RegCount = std::get<0>(CountAndVT); |
1008 | MVT RegisterVT = std::get<1>(CountAndVT); |
1009 | TypeSize RegisterSize = RegisterVT.getSizeInBits(); |
1010 | for (unsigned E = I + RegCount; I != E; ++I) |
1011 | OutVec.push_back(std::make_pair(Regs[I], RegisterSize)); |
1012 | } |
1013 | return OutVec; |
1014 | } |
1015 | |
1016 | void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa, |
1017 | const TargetLibraryInfo *li) { |
1018 | AA = aa; |
1019 | GFI = gfi; |
1020 | LibInfo = li; |
1021 | DL = &DAG.getDataLayout(); |
1022 | Context = DAG.getContext(); |
1023 | LPadToCallSiteMap.clear(); |
1024 | SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout()); |
1025 | } |
1026 | |
1027 | void SelectionDAGBuilder::clear() { |
1028 | NodeMap.clear(); |
1029 | UnusedArgNodeMap.clear(); |
1030 | PendingLoads.clear(); |
1031 | PendingExports.clear(); |
1032 | PendingConstrainedFP.clear(); |
1033 | PendingConstrainedFPStrict.clear(); |
1034 | CurInst = nullptr; |
1035 | HasTailCall = false; |
1036 | SDNodeOrder = LowestSDNodeOrder; |
1037 | StatepointLowering.clear(); |
1038 | } |
1039 | |
1040 | void SelectionDAGBuilder::clearDanglingDebugInfo() { |
1041 | DanglingDebugInfoMap.clear(); |
1042 | } |
1043 | |
1044 | // Update DAG root to include dependencies on Pending chains. |
1045 | SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) { |
1046 | SDValue Root = DAG.getRoot(); |
1047 | |
1048 | if (Pending.empty()) |
1049 | return Root; |
1050 | |
1051 | // Add current root to PendingChains, unless we already indirectly |
1052 | // depend on it. |
1053 | if (Root.getOpcode() != ISD::EntryToken) { |
1054 | unsigned i = 0, e = Pending.size(); |
1055 | for (; i != e; ++i) { |
1056 | assert(Pending[i].getNode()->getNumOperands() > 1)((void)0); |
1057 | if (Pending[i].getNode()->getOperand(0) == Root) |
1058 | break; // Don't add the root if we already indirectly depend on it. |
1059 | } |
1060 | |
1061 | if (i == e) |
1062 | Pending.push_back(Root); |
1063 | } |
1064 | |
1065 | if (Pending.size() == 1) |
1066 | Root = Pending[0]; |
1067 | else |
1068 | Root = DAG.getTokenFactor(getCurSDLoc(), Pending); |
1069 | |
1070 | DAG.setRoot(Root); |
1071 | Pending.clear(); |
1072 | return Root; |
1073 | } |
1074 | |
1075 | SDValue SelectionDAGBuilder::getMemoryRoot() { |
1076 | return updateRoot(PendingLoads); |
1077 | } |
1078 | |
1079 | SDValue SelectionDAGBuilder::getRoot() { |
1080 | // Chain up all pending constrained intrinsics together with all |
1081 | // pending loads, by simply appending them to PendingLoads and |
1082 | // then calling getMemoryRoot(). |
1083 | PendingLoads.reserve(PendingLoads.size() + |
1084 | PendingConstrainedFP.size() + |
1085 | PendingConstrainedFPStrict.size()); |
1086 | PendingLoads.append(PendingConstrainedFP.begin(), |
1087 | PendingConstrainedFP.end()); |
1088 | PendingLoads.append(PendingConstrainedFPStrict.begin(), |
1089 | PendingConstrainedFPStrict.end()); |
1090 | PendingConstrainedFP.clear(); |
1091 | PendingConstrainedFPStrict.clear(); |
1092 | return getMemoryRoot(); |
1093 | } |
1094 | |
1095 | SDValue SelectionDAGBuilder::getControlRoot() { |
1096 | // We need to emit pending fpexcept.strict constrained intrinsics, |
1097 | // so append them to the PendingExports list. |
1098 | PendingExports.append(PendingConstrainedFPStrict.begin(), |
1099 | PendingConstrainedFPStrict.end()); |
1100 | PendingConstrainedFPStrict.clear(); |
1101 | return updateRoot(PendingExports); |
1102 | } |
1103 | |
1104 | void SelectionDAGBuilder::visit(const Instruction &I) { |
1105 | // Set up outgoing PHI node register values before emitting the terminator. |
1106 | if (I.isTerminator()) { |
1107 | HandlePHINodesInSuccessorBlocks(I.getParent()); |
1108 | } |
1109 | |
1110 | // Increase the SDNodeOrder if dealing with a non-debug instruction. |
1111 | if (!isa<DbgInfoIntrinsic>(I)) |
1112 | ++SDNodeOrder; |
1113 | |
1114 | CurInst = &I; |
1115 | |
1116 | visit(I.getOpcode(), I); |
1117 | |
1118 | if (!I.isTerminator() && !HasTailCall && |
1119 | !isa<GCStatepointInst>(I)) // statepoints handle their exports internally |
1120 | CopyToExportRegsIfNeeded(&I); |
1121 | |
1122 | CurInst = nullptr; |
1123 | } |
1124 | |
1125 | void SelectionDAGBuilder::visitPHI(const PHINode &) { |
1126 | llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")__builtin_unreachable(); |
1127 | } |
1128 | |
1129 | void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { |
1130 | // Note: this doesn't use InstVisitor, because it has to work with |
1131 | // ConstantExpr's in addition to instructions. |
1132 | switch (Opcode) { |
1133 | default: llvm_unreachable("Unknown instruction type encountered!")__builtin_unreachable(); |
1134 | // Build the switch statement using the Instruction.def file. |
1135 | #define HANDLE_INST(NUM, OPCODE, CLASS) \ |
1136 | case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; |
1137 | #include "llvm/IR/Instruction.def" |
1138 | } |
1139 | } |
1140 | |
1141 | void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI, |
1142 | DebugLoc DL, unsigned Order) { |
1143 | // We treat variadic dbg_values differently at this stage. |
1144 | if (DI->hasArgList()) { |
1145 | // For variadic dbg_values we will now insert an undef. |
1146 | // FIXME: We can potentially recover these! |
1147 | SmallVector<SDDbgOperand, 2> Locs; |
1148 | for (const Value *V : DI->getValues()) { |
1149 | auto Undef = UndefValue::get(V->getType()); |
1150 | Locs.push_back(SDDbgOperand::fromConst(Undef)); |
1151 | } |
1152 | SDDbgValue *SDV = DAG.getDbgValueList( |
1153 | DI->getVariable(), DI->getExpression(), Locs, {}, |
1154 | /*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true); |
1155 | DAG.AddDbgValue(SDV, /*isParameter=*/false); |
1156 | } else { |
1157 | // TODO: Dangling debug info will eventually either be resolved or produce |
1158 | // an Undef DBG_VALUE. However in the resolution case, a gap may appear |
1159 | // between the original dbg.value location and its resolved DBG_VALUE, |
1160 | // which we should ideally fill with an extra Undef DBG_VALUE. |
1161 | assert(DI->getNumVariableLocationOps() == 1 &&((void)0) |
1162 | "DbgValueInst without an ArgList should have a single location "((void)0) |
1163 | "operand.")((void)0); |
1164 | DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order); |
1165 | } |
1166 | } |
1167 | |
1168 | void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable, |
1169 | const DIExpression *Expr) { |
1170 | auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) { |
1171 | const DbgValueInst *DI = DDI.getDI(); |
1172 | DIVariable *DanglingVariable = DI->getVariable(); |
1173 | DIExpression *DanglingExpr = DI->getExpression(); |
1174 | if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) { |
1175 | LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { } while (false); |
1176 | return true; |
1177 | } |
1178 | return false; |
1179 | }; |
1180 | |
1181 | for (auto &DDIMI : DanglingDebugInfoMap) { |
1182 | DanglingDebugInfoVector &DDIV = DDIMI.second; |
1183 | |
1184 | // If debug info is to be dropped, run it through final checks to see |
1185 | // whether it can be salvaged. |
1186 | for (auto &DDI : DDIV) |
1187 | if (isMatchingDbgValue(DDI)) |
1188 | salvageUnresolvedDbgValue(DDI); |
1189 | |
1190 | erase_if(DDIV, isMatchingDbgValue); |
1191 | } |
1192 | } |
1193 | |
1194 | // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, |
1195 | // generate the debug data structures now that we've seen its definition. |
1196 | void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, |
1197 | SDValue Val) { |
1198 | auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V); |
1199 | if (DanglingDbgInfoIt == DanglingDebugInfoMap.end()) |
1200 | return; |
1201 | |
1202 | DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second; |
1203 | for (auto &DDI : DDIV) { |
1204 | const DbgValueInst *DI = DDI.getDI(); |
1205 | assert(!DI->hasArgList() && "Not implemented for variadic dbg_values")((void)0); |
1206 | assert(DI && "Ill-formed DanglingDebugInfo")((void)0); |
1207 | DebugLoc dl = DDI.getdl(); |
1208 | unsigned ValSDNodeOrder = Val.getNode()->getIROrder(); |
1209 | unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); |
1210 | DILocalVariable *Variable = DI->getVariable(); |
1211 | DIExpression *Expr = DI->getExpression(); |
1212 | assert(Variable->isValidLocationForIntrinsic(dl) &&((void)0) |
1213 | "Expected inlined-at fields to agree")((void)0); |
1214 | SDDbgValue *SDV; |
1215 | if (Val.getNode()) { |
1216 | // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a |
1217 | // FuncArgumentDbgValue (it would be hoisted to the function entry, and if |
1218 | // we couldn't resolve it directly when examining the DbgValue intrinsic |
1219 | // in the first place we should not be more successful here). Unless we |
1220 | // have some test case that prove this to be correct we should avoid |
1221 | // calling EmitFuncArgumentDbgValue here. |
1222 | if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) { |
1223 | LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { } while (false) |
1224 | << DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { } while (false); |
1225 | LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { } while (false); |
1226 | // Increase the SDNodeOrder for the DbgValue here to make sure it is |
1227 | // inserted after the definition of Val when emitting the instructions |
1228 | // after ISel. An alternative could be to teach |
1229 | // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly. |
1230 | LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { } while (false) |
1231 | << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { } while (false) |
1232 | << ValSDNodeOrder << "\n")do { } while (false); |
1233 | SDV = getDbgValue(Val, Variable, Expr, dl, |
1234 | std::max(DbgSDNodeOrder, ValSDNodeOrder)); |
1235 | DAG.AddDbgValue(SDV, false); |
1236 | } else |
1237 | LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { } while (false) |
1238 | << "in EmitFuncArgumentDbgValue\n")do { } while (false); |
1239 | } else { |
1240 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { } while (false); |
1241 | auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType()); |
1242 | auto SDV = |
1243 | DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder); |
1244 | DAG.AddDbgValue(SDV, false); |
1245 | } |
1246 | } |
1247 | DDIV.clear(); |
1248 | } |
1249 | |
1250 | void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) { |
1251 | // TODO: For the variadic implementation, instead of only checking the fail |
1252 | // state of `handleDebugValue`, we need know specifically which values were |
1253 | // invalid, so that we attempt to salvage only those values when processing |
1254 | // a DIArgList. |
1255 | assert(!DDI.getDI()->hasArgList() &&((void)0) |
1256 | "Not implemented for variadic dbg_values")((void)0); |
1257 | Value *V = DDI.getDI()->getValue(0); |
1258 | DILocalVariable *Var = DDI.getDI()->getVariable(); |
1259 | DIExpression *Expr = DDI.getDI()->getExpression(); |
1260 | DebugLoc DL = DDI.getdl(); |
1261 | DebugLoc InstDL = DDI.getDI()->getDebugLoc(); |
1262 | unsigned SDOrder = DDI.getSDNodeOrder(); |
1263 | // Currently we consider only dbg.value intrinsics -- we tell the salvager |
1264 | // that DW_OP_stack_value is desired. |
1265 | assert(isa<DbgValueInst>(DDI.getDI()))((void)0); |
1266 | bool StackValue = true; |
1267 | |
1268 | // Can this Value can be encoded without any further work? |
1269 | if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false)) |
1270 | return; |
1271 | |
1272 | // Attempt to salvage back through as many instructions as possible. Bail if |
1273 | // a non-instruction is seen, such as a constant expression or global |
1274 | // variable. FIXME: Further work could recover those too. |
1275 | while (isa<Instruction>(V)) { |
1276 | Instruction &VAsInst = *cast<Instruction>(V); |
1277 | // Temporary "0", awaiting real implementation. |
1278 | SmallVector<Value *, 4> AdditionalValues; |
1279 | DIExpression *SalvagedExpr = |
1280 | salvageDebugInfoImpl(VAsInst, Expr, StackValue, 0, AdditionalValues); |
1281 | |
1282 | // If we cannot salvage any further, and haven't yet found a suitable debug |
1283 | // expression, bail out. |
1284 | // TODO: If AdditionalValues isn't empty, then the salvage can only be |
1285 | // represented with a DBG_VALUE_LIST, so we give up. When we have support |
1286 | // here for variadic dbg_values, remove that condition. |
1287 | if (!SalvagedExpr || !AdditionalValues.empty()) |
1288 | break; |
1289 | |
1290 | // New value and expr now represent this debuginfo. |
1291 | V = VAsInst.getOperand(0); |
1292 | Expr = SalvagedExpr; |
1293 | |
1294 | // Some kind of simplification occurred: check whether the operand of the |
1295 | // salvaged debug expression can be encoded in this DAG. |
1296 | if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, |
1297 | /*IsVariadic=*/false)) { |
1298 | LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n "do { } while (false) |
1299 | << DDI.getDI() << "\nBy stripping back to:\n " << V)do { } while (false); |
1300 | return; |
1301 | } |
1302 | } |
1303 | |
1304 | // This was the final opportunity to salvage this debug information, and it |
1305 | // couldn't be done. Place an undef DBG_VALUE at this location to terminate |
1306 | // any earlier variable location. |
1307 | auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType()); |
1308 | auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder); |
1309 | DAG.AddDbgValue(SDV, false); |
1310 | |
1311 | LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI()do { } while (false) |
1312 | << "\n")do { } while (false); |
1313 | LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0)do { } while (false) |
1314 | << "\n")do { } while (false); |
1315 | } |
1316 | |
1317 | bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values, |
1318 | DILocalVariable *Var, |
1319 | DIExpression *Expr, DebugLoc dl, |
1320 | DebugLoc InstDL, unsigned Order, |
1321 | bool IsVariadic) { |
1322 | if (Values.empty()) |
1323 | return true; |
1324 | SmallVector<SDDbgOperand> LocationOps; |
1325 | SmallVector<SDNode *> Dependencies; |
1326 | for (const Value *V : Values) { |
1327 | // Constant value. |
1328 | if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) || |
1329 | isa<ConstantPointerNull>(V)) { |
1330 | LocationOps.emplace_back(SDDbgOperand::fromConst(V)); |
1331 | continue; |
1332 | } |
1333 | |
1334 | // If the Value is a frame index, we can create a FrameIndex debug value |
1335 | // without relying on the DAG at all. |
1336 | if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { |
1337 | auto SI = FuncInfo.StaticAllocaMap.find(AI); |
1338 | if (SI != FuncInfo.StaticAllocaMap.end()) { |
1339 | LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second)); |
1340 | continue; |
1341 | } |
1342 | } |
1343 | |
1344 | // Do not use getValue() in here; we don't want to generate code at |
1345 | // this point if it hasn't been done yet. |
1346 | SDValue N = NodeMap[V]; |
1347 | if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map. |
1348 | N = UnusedArgNodeMap[V]; |
1349 | if (N.getNode()) { |
1350 | // Only emit func arg dbg value for non-variadic dbg.values for now. |
1351 | if (!IsVariadic && EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N)) |
1352 | return true; |
1353 | if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { |
1354 | // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can |
1355 | // describe stack slot locations. |
1356 | // |
1357 | // Consider "int x = 0; int *px = &x;". There are two kinds of |
1358 | // interesting debug values here after optimization: |
1359 | // |
1360 | // dbg.value(i32* %px, !"int *px", !DIExpression()), and |
1361 | // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) |
1362 | // |
1363 | // Both describe the direct values of their associated variables. |
1364 | Dependencies.push_back(N.getNode()); |
1365 | LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex())); |
1366 | continue; |
1367 | } |
1368 | LocationOps.emplace_back( |
1369 | SDDbgOperand::fromNode(N.getNode(), N.getResNo())); |
1370 | continue; |
1371 | } |
1372 | |
1373 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
1374 | // Special rules apply for the first dbg.values of parameter variables in a |
1375 | // function. Identify them by the fact they reference Argument Values, that |
1376 | // they're parameters, and they are parameters of the current function. We |
1377 | // need to let them dangle until they get an SDNode. |
1378 | bool IsParamOfFunc = |
1379 | isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt(); |
1380 | if (IsParamOfFunc) |
1381 | return false; |
1382 | |
1383 | // The value is not used in this block yet (or it would have an SDNode). |
1384 | // We still want the value to appear for the user if possible -- if it has |
1385 | // an associated VReg, we can refer to that instead. |
1386 | auto VMI = FuncInfo.ValueMap.find(V); |
1387 | if (VMI != FuncInfo.ValueMap.end()) { |
1388 | unsigned Reg = VMI->second; |
1389 | // If this is a PHI node, it may be split up into several MI PHI nodes |
1390 | // (in FunctionLoweringInfo::set). |
1391 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, |
1392 | V->getType(), None); |
1393 | if (RFV.occupiesMultipleRegs()) { |
1394 | // FIXME: We could potentially support variadic dbg_values here. |
1395 | if (IsVariadic) |
1396 | return false; |
1397 | unsigned Offset = 0; |
1398 | unsigned BitsToDescribe = 0; |
1399 | if (auto VarSize = Var->getSizeInBits()) |
1400 | BitsToDescribe = *VarSize; |
1401 | if (auto Fragment = Expr->getFragmentInfo()) |
1402 | BitsToDescribe = Fragment->SizeInBits; |
1403 | for (auto RegAndSize : RFV.getRegsAndSizes()) { |
1404 | // Bail out if all bits are described already. |
1405 | if (Offset >= BitsToDescribe) |
1406 | break; |
1407 | // TODO: handle scalable vectors. |
1408 | unsigned RegisterSize = RegAndSize.second; |
1409 | unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe) |
1410 | ? BitsToDescribe - Offset |
1411 | : RegisterSize; |
1412 | auto FragmentExpr = DIExpression::createFragmentExpression( |
1413 | Expr, Offset, FragmentSize); |
1414 | if (!FragmentExpr) |
1415 | continue; |
1416 | SDDbgValue *SDV = DAG.getVRegDbgValue( |
1417 | Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder); |
1418 | DAG.AddDbgValue(SDV, false); |
1419 | Offset += RegisterSize; |
1420 | } |
1421 | return true; |
1422 | } |
1423 | // We can use simple vreg locations for variadic dbg_values as well. |
1424 | LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg)); |
1425 | continue; |
1426 | } |
1427 | // We failed to create a SDDbgOperand for V. |
1428 | return false; |
1429 | } |
1430 | |
1431 | // We have created a SDDbgOperand for each Value in Values. |
1432 | // Should use Order instead of SDNodeOrder? |
1433 | assert(!LocationOps.empty())((void)0); |
1434 | SDDbgValue *SDV = |
1435 | DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies, |
1436 | /*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic); |
1437 | DAG.AddDbgValue(SDV, /*isParameter=*/false); |
1438 | return true; |
1439 | } |
1440 | |
1441 | void SelectionDAGBuilder::resolveOrClearDbgInfo() { |
1442 | // Try to fixup any remaining dangling debug info -- and drop it if we can't. |
1443 | for (auto &Pair : DanglingDebugInfoMap) |
1444 | for (auto &DDI : Pair.second) |
1445 | salvageUnresolvedDbgValue(DDI); |
1446 | clearDanglingDebugInfo(); |
1447 | } |
1448 | |
1449 | /// getCopyFromRegs - If there was virtual register allocated for the value V |
1450 | /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. |
1451 | SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { |
1452 | DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V); |
1453 | SDValue Result; |
1454 | |
1455 | if (It != FuncInfo.ValueMap.end()) { |
1456 | Register InReg = It->second; |
1457 | |
1458 | RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), |
1459 | DAG.getDataLayout(), InReg, Ty, |
1460 | None); // This is not an ABI copy. |
1461 | SDValue Chain = DAG.getEntryNode(); |
1462 | Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, |
1463 | V); |
1464 | resolveDanglingDebugInfo(V, Result); |
1465 | } |
1466 | |
1467 | return Result; |
1468 | } |
1469 | |
1470 | /// getValue - Return an SDValue for the given Value. |
1471 | SDValue SelectionDAGBuilder::getValue(const Value *V) { |
1472 | // If we already have an SDValue for this value, use it. It's important |
1473 | // to do this first, so that we don't create a CopyFromReg if we already |
1474 | // have a regular SDValue. |
1475 | SDValue &N = NodeMap[V]; |
1476 | if (N.getNode()) return N; |
1477 | |
1478 | // If there's a virtual register allocated and initialized for this |
1479 | // value, use it. |
1480 | if (SDValue copyFromReg = getCopyFromRegs(V, V->getType())) |
1481 | return copyFromReg; |
1482 | |
1483 | // Otherwise create a new SDValue and remember it. |
1484 | SDValue Val = getValueImpl(V); |
1485 | NodeMap[V] = Val; |
1486 | resolveDanglingDebugInfo(V, Val); |
1487 | return Val; |
1488 | } |
1489 | |
1490 | /// getNonRegisterValue - Return an SDValue for the given Value, but |
1491 | /// don't look in FuncInfo.ValueMap for a virtual register. |
1492 | SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { |
1493 | // If we already have an SDValue for this value, use it. |
1494 | SDValue &N = NodeMap[V]; |
1495 | if (N.getNode()) { |
1496 | if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) { |
1497 | // Remove the debug location from the node as the node is about to be used |
1498 | // in a location which may differ from the original debug location. This |
1499 | // is relevant to Constant and ConstantFP nodes because they can appear |
1500 | // as constant expressions inside PHI nodes. |
1501 | N->setDebugLoc(DebugLoc()); |
1502 | } |
1503 | return N; |
1504 | } |
1505 | |
1506 | // Otherwise create a new SDValue and remember it. |
1507 | SDValue Val = getValueImpl(V); |
1508 | NodeMap[V] = Val; |
1509 | resolveDanglingDebugInfo(V, Val); |
1510 | return Val; |
1511 | } |
1512 | |
1513 | /// getValueImpl - Helper function for getValue and getNonRegisterValue. |
1514 | /// Create an SDValue for the given value. |
1515 | SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { |
1516 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
1517 | |
1518 | if (const Constant *C = dyn_cast<Constant>(V)) { |
1519 | EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true); |
1520 | |
1521 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) |
1522 | return DAG.getConstant(*CI, getCurSDLoc(), VT); |
1523 | |
1524 | if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) |
1525 | return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); |
1526 | |
1527 | if (isa<ConstantPointerNull>(C)) { |
1528 | unsigned AS = V->getType()->getPointerAddressSpace(); |
1529 | return DAG.getConstant(0, getCurSDLoc(), |
1530 | TLI.getPointerTy(DAG.getDataLayout(), AS)); |
1531 | } |
1532 | |
1533 | if (match(C, m_VScale(DAG.getDataLayout()))) |
1534 | return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)); |
1535 | |
1536 | if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) |
1537 | return DAG.getConstantFP(*CFP, getCurSDLoc(), VT); |
1538 | |
1539 | if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) |
1540 | return DAG.getUNDEF(VT); |
1541 | |
1542 | if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { |
1543 | visit(CE->getOpcode(), *CE); |
1544 | SDValue N1 = NodeMap[V]; |
1545 | assert(N1.getNode() && "visit didn't populate the NodeMap!")((void)0); |
1546 | return N1; |
1547 | } |
1548 | |
1549 | if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { |
1550 | SmallVector<SDValue, 4> Constants; |
1551 | for (const Use &U : C->operands()) { |
1552 | SDNode *Val = getValue(U).getNode(); |
1553 | // If the operand is an empty aggregate, there are no values. |
1554 | if (!Val) continue; |
1555 | // Add each leaf value from the operand to the Constants list |
1556 | // to form a flattened list of all the values. |
1557 | for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) |
1558 | Constants.push_back(SDValue(Val, i)); |
1559 | } |
1560 | |
1561 | return DAG.getMergeValues(Constants, getCurSDLoc()); |
1562 | } |
1563 | |
1564 | if (const ConstantDataSequential *CDS = |
1565 | dyn_cast<ConstantDataSequential>(C)) { |
1566 | SmallVector<SDValue, 4> Ops; |
1567 | for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { |
1568 | SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); |
1569 | // Add each leaf value from the operand to the Constants list |
1570 | // to form a flattened list of all the values. |
1571 | for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) |
1572 | Ops.push_back(SDValue(Val, i)); |
1573 | } |
1574 | |
1575 | if (isa<ArrayType>(CDS->getType())) |
1576 | return DAG.getMergeValues(Ops, getCurSDLoc()); |
1577 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); |
1578 | } |
1579 | |
1580 | if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { |
1581 | assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&((void)0) |
1582 | "Unknown struct or array constant!")((void)0); |
1583 | |
1584 | SmallVector<EVT, 4> ValueVTs; |
1585 | ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs); |
1586 | unsigned NumElts = ValueVTs.size(); |
1587 | if (NumElts == 0) |
1588 | return SDValue(); // empty struct |
1589 | SmallVector<SDValue, 4> Constants(NumElts); |
1590 | for (unsigned i = 0; i != NumElts; ++i) { |
1591 | EVT EltVT = ValueVTs[i]; |
1592 | if (isa<UndefValue>(C)) |
1593 | Constants[i] = DAG.getUNDEF(EltVT); |
1594 | else if (EltVT.isFloatingPoint()) |
1595 | Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT); |
1596 | else |
1597 | Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT); |
1598 | } |
1599 | |
1600 | return DAG.getMergeValues(Constants, getCurSDLoc()); |
1601 | } |
1602 | |
1603 | if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) |
1604 | return DAG.getBlockAddress(BA, VT); |
1605 | |
1606 | if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C)) |
1607 | return getValue(Equiv->getGlobalValue()); |
1608 | |
1609 | VectorType *VecTy = cast<VectorType>(V->getType()); |
1610 | |
1611 | // Now that we know the number and type of the elements, get that number of |
1612 | // elements into the Ops array based on what kind of constant it is. |
1613 | if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { |
1614 | SmallVector<SDValue, 16> Ops; |
1615 | unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements(); |
1616 | for (unsigned i = 0; i != NumElements; ++i) |
1617 | Ops.push_back(getValue(CV->getOperand(i))); |
1618 | |
1619 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); |
1620 | } else if (isa<ConstantAggregateZero>(C)) { |
1621 | EVT EltVT = |
1622 | TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType()); |
1623 | |
1624 | SDValue Op; |
1625 | if (EltVT.isFloatingPoint()) |
1626 | Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT); |
1627 | else |
1628 | Op = DAG.getConstant(0, getCurSDLoc(), EltVT); |
1629 | |
1630 | if (isa<ScalableVectorType>(VecTy)) |
1631 | return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op); |
1632 | else { |
1633 | SmallVector<SDValue, 16> Ops; |
1634 | Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op); |
1635 | return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); |
1636 | } |
1637 | } |
1638 | llvm_unreachable("Unknown vector constant")__builtin_unreachable(); |
1639 | } |
1640 | |
1641 | // If this is a static alloca, generate it as the frameindex instead of |
1642 | // computation. |
1643 | if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { |
1644 | DenseMap<const AllocaInst*, int>::iterator SI = |
1645 | FuncInfo.StaticAllocaMap.find(AI); |
1646 | if (SI != FuncInfo.StaticAllocaMap.end()) |
1647 | return DAG.getFrameIndex(SI->second, |
1648 | TLI.getFrameIndexTy(DAG.getDataLayout())); |
1649 | } |
1650 | |
1651 | // If this is an instruction which fast-isel has deferred, select it now. |
1652 | if (const Instruction *Inst = dyn_cast<Instruction>(V)) { |
1653 | unsigned InReg = FuncInfo.InitializeRegForValue(Inst); |
1654 | |
1655 | RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg, |
1656 | Inst->getType(), None); |
1657 | SDValue Chain = DAG.getEntryNode(); |
1658 | return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); |
1659 | } |
1660 | |
1661 | if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) { |
1662 | return DAG.getMDNode(cast<MDNode>(MD->getMetadata())); |
1663 | } |
1664 | llvm_unreachable("Can't get register for value!")__builtin_unreachable(); |
1665 | } |
1666 | |
1667 | void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) { |
1668 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); |
1669 | bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX; |
1670 | bool IsCoreCLR = Pers == EHPersonality::CoreCLR; |
1671 | bool IsSEH = isAsynchronousEHPersonality(Pers); |
1672 | MachineBasicBlock *CatchPadMBB = FuncInfo.MBB; |
1673 | if (!IsSEH) |
1674 | CatchPadMBB->setIsEHScopeEntry(); |
1675 | // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues. |
1676 | if (IsMSVCCXX || IsCoreCLR) |
1677 | CatchPadMBB->setIsEHFuncletEntry(); |
1678 | } |
1679 | |
1680 | void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) { |
1681 | // Update machine-CFG edge. |
1682 | MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()]; |
1683 | FuncInfo.MBB->addSuccessor(TargetMBB); |
1684 | TargetMBB->setIsEHCatchretTarget(true); |
1685 | DAG.getMachineFunction().setHasEHCatchret(true); |
1686 | |
1687 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); |
1688 | bool IsSEH = isAsynchronousEHPersonality(Pers); |
1689 | if (IsSEH) { |
1690 | // If this is not a fall-through branch or optimizations are switched off, |
1691 | // emit the branch. |
1692 | if (TargetMBB != NextBlock(FuncInfo.MBB) || |
1693 | TM.getOptLevel() == CodeGenOpt::None) |
1694 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, |
1695 | getControlRoot(), DAG.getBasicBlock(TargetMBB))); |
1696 | return; |
1697 | } |
1698 | |
1699 | // Figure out the funclet membership for the catchret's successor. |
1700 | // This will be used by the FuncletLayout pass to determine how to order the |
1701 | // BB's. |
1702 | // A 'catchret' returns to the outer scope's color. |
1703 | Value *ParentPad = I.getCatchSwitchParentPad(); |
1704 | const BasicBlock *SuccessorColor; |
1705 | if (isa<ConstantTokenNone>(ParentPad)) |
1706 | SuccessorColor = &FuncInfo.Fn->getEntryBlock(); |
1707 | else |
1708 | SuccessorColor = cast<Instruction>(ParentPad)->getParent(); |
1709 | assert(SuccessorColor && "No parent funclet for catchret!")((void)0); |
1710 | MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor]; |
1711 | assert(SuccessorColorMBB && "No MBB for SuccessorColor!")((void)0); |
1712 | |
1713 | // Create the terminator node. |
1714 | SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other, |
1715 | getControlRoot(), DAG.getBasicBlock(TargetMBB), |
1716 | DAG.getBasicBlock(SuccessorColorMBB)); |
1717 | DAG.setRoot(Ret); |
1718 | } |
1719 | |
1720 | void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) { |
1721 | // Don't emit any special code for the cleanuppad instruction. It just marks |
1722 | // the start of an EH scope/funclet. |
1723 | FuncInfo.MBB->setIsEHScopeEntry(); |
1724 | auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); |
1725 | if (Pers != EHPersonality::Wasm_CXX) { |
1726 | FuncInfo.MBB->setIsEHFuncletEntry(); |
1727 | FuncInfo.MBB->setIsCleanupFuncletEntry(); |
1728 | } |
1729 | } |
1730 | |
1731 | // In wasm EH, even though a catchpad may not catch an exception if a tag does |
1732 | // not match, it is OK to add only the first unwind destination catchpad to the |
1733 | // successors, because there will be at least one invoke instruction within the |
1734 | // catch scope that points to the next unwind destination, if one exists, so |
1735 | // CFGSort cannot mess up with BB sorting order. |
1736 | // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic |
1737 | // call within them, and catchpads only consisting of 'catch (...)' have a |
1738 | // '__cxa_end_catch' call within them, both of which generate invokes in case |
1739 | // the next unwind destination exists, i.e., the next unwind destination is not |
1740 | // the caller.) |
1741 | // |
1742 | // Having at most one EH pad successor is also simpler and helps later |
1743 | // transformations. |
1744 | // |
1745 | // For example, |
1746 | // current: |
1747 | // invoke void @foo to ... unwind label %catch.dispatch |
1748 | // catch.dispatch: |
1749 | // %0 = catchswitch within ... [label %catch.start] unwind label %next |
1750 | // catch.start: |
1751 | // ... |
1752 | // ... in this BB or some other child BB dominated by this BB there will be an |
1753 | // invoke that points to 'next' BB as an unwind destination |
1754 | // |
1755 | // next: ; We don't need to add this to 'current' BB's successor |
1756 | // ... |
1757 | static void findWasmUnwindDestinations( |
1758 | FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, |
1759 | BranchProbability Prob, |
1760 | SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> |
1761 | &UnwindDests) { |
1762 | while (EHPadBB) { |
1763 | const Instruction *Pad = EHPadBB->getFirstNonPHI(); |
1764 | if (isa<CleanupPadInst>(Pad)) { |
1765 | // Stop on cleanup pads. |
1766 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); |
1767 | UnwindDests.back().first->setIsEHScopeEntry(); |
1768 | break; |
1769 | } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { |
1770 | // Add the catchpad handlers to the possible destinations. We don't |
1771 | // continue to the unwind destination of the catchswitch for wasm. |
1772 | for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { |
1773 | UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); |
1774 | UnwindDests.back().first->setIsEHScopeEntry(); |
1775 | } |
1776 | break; |
1777 | } else { |
1778 | continue; |
1779 | } |
1780 | } |
1781 | } |
1782 | |
1783 | /// When an invoke or a cleanupret unwinds to the next EH pad, there are |
1784 | /// many places it could ultimately go. In the IR, we have a single unwind |
1785 | /// destination, but in the machine CFG, we enumerate all the possible blocks. |
1786 | /// This function skips over imaginary basic blocks that hold catchswitch |
1787 | /// instructions, and finds all the "real" machine |
1788 | /// basic block destinations. As those destinations may not be successors of |
1789 | /// EHPadBB, here we also calculate the edge probability to those destinations. |
1790 | /// The passed-in Prob is the edge probability to EHPadBB. |
1791 | static void findUnwindDestinations( |
1792 | FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, |
1793 | BranchProbability Prob, |
1794 | SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> |
1795 | &UnwindDests) { |
1796 | EHPersonality Personality = |
1797 | classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); |
1798 | bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; |
1799 | bool IsCoreCLR = Personality == EHPersonality::CoreCLR; |
1800 | bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX; |
1801 | bool IsSEH = isAsynchronousEHPersonality(Personality); |
1802 | |
1803 | if (IsWasmCXX) { |
1804 | findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests); |
1805 | assert(UnwindDests.size() <= 1 &&((void)0) |
1806 | "There should be at most one unwind destination for wasm")((void)0); |
1807 | return; |
1808 | } |
1809 | |
1810 | while (EHPadBB) { |
1811 | const Instruction *Pad = EHPadBB->getFirstNonPHI(); |
1812 | BasicBlock *NewEHPadBB = nullptr; |
1813 | if (isa<LandingPadInst>(Pad)) { |
1814 | // Stop on landingpads. They are not funclets. |
1815 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); |
1816 | break; |
1817 | } else if (isa<CleanupPadInst>(Pad)) { |
1818 | // Stop on cleanup pads. Cleanups are always funclet entries for all known |
1819 | // personalities. |
1820 | UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); |
1821 | UnwindDests.back().first->setIsEHScopeEntry(); |
1822 | UnwindDests.back().first->setIsEHFuncletEntry(); |
1823 | break; |
1824 | } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { |
1825 | // Add the catchpad handlers to the possible destinations. |
1826 | for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { |
1827 | UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); |
1828 | // For MSVC++ and the CLR, catchblocks are funclets and need prologues. |
1829 | if (IsMSVCCXX || IsCoreCLR) |
1830 | UnwindDests.back().first->setIsEHFuncletEntry(); |
1831 | if (!IsSEH) |
1832 | UnwindDests.back().first->setIsEHScopeEntry(); |
1833 | } |
1834 | NewEHPadBB = CatchSwitch->getUnwindDest(); |
1835 | } else { |
1836 | continue; |
1837 | } |
1838 | |
1839 | BranchProbabilityInfo *BPI = FuncInfo.BPI; |
1840 | if (BPI && NewEHPadBB) |
1841 | Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB); |
1842 | EHPadBB = NewEHPadBB; |
1843 | } |
1844 | } |
1845 | |
1846 | void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) { |
1847 | // Update successor info. |
1848 | SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; |
1849 | auto UnwindDest = I.getUnwindDest(); |
1850 | BranchProbabilityInfo *BPI = FuncInfo.BPI; |
1851 | BranchProbability UnwindDestProb = |
1852 | (BPI && UnwindDest) |
1853 | ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest) |
1854 | : BranchProbability::getZero(); |
1855 | findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests); |
1856 | for (auto &UnwindDest : UnwindDests) { |
1857 | UnwindDest.first->setIsEHPad(); |
1858 | addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second); |
1859 | } |
1860 | FuncInfo.MBB->normalizeSuccProbs(); |
1861 | |
1862 | // Create the terminator node. |
1863 | SDValue Ret = |
1864 | DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot()); |
1865 | DAG.setRoot(Ret); |
1866 | } |
1867 | |
1868 | void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) { |
1869 | report_fatal_error("visitCatchSwitch not yet implemented!"); |
1870 | } |
1871 | |
1872 | void SelectionDAGBuilder::visitRet(const ReturnInst &I) { |
1873 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
1874 | auto &DL = DAG.getDataLayout(); |
1875 | SDValue Chain = getControlRoot(); |
1876 | SmallVector<ISD::OutputArg, 8> Outs; |
1877 | SmallVector<SDValue, 8> OutVals; |
1878 | |
1879 | // Calls to @llvm.experimental.deoptimize don't generate a return value, so |
1880 | // lower |
1881 | // |
1882 | // %val = call <ty> @llvm.experimental.deoptimize() |
1883 | // ret <ty> %val |
1884 | // |
1885 | // differently. |
1886 | if (I.getParent()->getTerminatingDeoptimizeCall()) { |
1887 | LowerDeoptimizingReturn(); |
1888 | return; |
1889 | } |
1890 | |
1891 | if (!FuncInfo.CanLowerReturn) { |
1892 | unsigned DemoteReg = FuncInfo.DemoteRegister; |
1893 | const Function *F = I.getParent()->getParent(); |
1894 | |
1895 | // Emit a store of the return value through the virtual register. |
1896 | // Leave Outs empty so that LowerReturn won't try to load return |
1897 | // registers the usual way. |
1898 | SmallVector<EVT, 1> PtrValueVTs; |
1899 | ComputeValueVTs(TLI, DL, |
1900 | F->getReturnType()->getPointerTo( |
1901 | DAG.getDataLayout().getAllocaAddrSpace()), |
1902 | PtrValueVTs); |
1903 | |
1904 | SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), |
1905 | DemoteReg, PtrValueVTs[0]); |
1906 | SDValue RetOp = getValue(I.getOperand(0)); |
1907 | |
1908 | SmallVector<EVT, 4> ValueVTs, MemVTs; |
1909 | SmallVector<uint64_t, 4> Offsets; |
1910 | ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs, |
1911 | &Offsets); |
1912 | unsigned NumValues = ValueVTs.size(); |
1913 | |
1914 | SmallVector<SDValue, 4> Chains(NumValues); |
1915 | Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType()); |
1916 | for (unsigned i = 0; i != NumValues; ++i) { |
1917 | // An aggregate return value cannot wrap around the address space, so |
1918 | // offsets to its parts don't wrap either. |
1919 | SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, |
1920 | TypeSize::Fixed(Offsets[i])); |
1921 | |
1922 | SDValue Val = RetOp.getValue(RetOp.getResNo() + i); |
1923 | if (MemVTs[i] != ValueVTs[i]) |
1924 | Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]); |
1925 | Chains[i] = DAG.getStore( |
1926 | Chain, getCurSDLoc(), Val, |
1927 | // FIXME: better loc info would be nice. |
1928 | Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), |
1929 | commonAlignment(BaseAlign, Offsets[i])); |
1930 | } |
1931 | |
1932 | Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), |
1933 | MVT::Other, Chains); |
1934 | } else if (I.getNumOperands() != 0) { |
1935 | SmallVector<EVT, 4> ValueVTs; |
1936 | ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs); |
1937 | unsigned NumValues = ValueVTs.size(); |
1938 | if (NumValues) { |
1939 | SDValue RetOp = getValue(I.getOperand(0)); |
1940 | |
1941 | const Function *F = I.getParent()->getParent(); |
1942 | |
1943 | bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( |
1944 | I.getOperand(0)->getType(), F->getCallingConv(), |
1945 | /*IsVarArg*/ false, DL); |
1946 | |
1947 | ISD::NodeType ExtendKind = ISD::ANY_EXTEND; |
1948 | if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex, |
1949 | Attribute::SExt)) |
1950 | ExtendKind = ISD::SIGN_EXTEND; |
1951 | else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex, |
1952 | Attribute::ZExt)) |
1953 | ExtendKind = ISD::ZERO_EXTEND; |
1954 | |
1955 | LLVMContext &Context = F->getContext(); |
1956 | bool RetInReg = F->getAttributes().hasAttribute( |
1957 | AttributeList::ReturnIndex, Attribute::InReg); |
1958 | |
1959 | for (unsigned j = 0; j != NumValues; ++j) { |
1960 | EVT VT = ValueVTs[j]; |
1961 | |
1962 | if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) |
1963 | VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind); |
1964 | |
1965 | CallingConv::ID CC = F->getCallingConv(); |
1966 | |
1967 | unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT); |
1968 | MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT); |
1969 | SmallVector<SDValue, 4> Parts(NumParts); |
1970 | getCopyToParts(DAG, getCurSDLoc(), |
1971 | SDValue(RetOp.getNode(), RetOp.getResNo() + j), |
1972 | &Parts[0], NumParts, PartVT, &I, CC, ExtendKind); |
1973 | |
1974 | // 'inreg' on function refers to return value |
1975 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); |
1976 | if (RetInReg) |
1977 | Flags.setInReg(); |
1978 | |
1979 | if (I.getOperand(0)->getType()->isPointerTy()) { |
1980 | Flags.setPointer(); |
1981 | Flags.setPointerAddrSpace( |
1982 | cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace()); |
1983 | } |
1984 | |
1985 | if (NeedsRegBlock) { |
1986 | Flags.setInConsecutiveRegs(); |
1987 | if (j == NumValues - 1) |
1988 | Flags.setInConsecutiveRegsLast(); |
1989 | } |
1990 | |
1991 | // Propagate extension type if any |
1992 | if (ExtendKind == ISD::SIGN_EXTEND) |
1993 | Flags.setSExt(); |
1994 | else if (ExtendKind == ISD::ZERO_EXTEND) |
1995 | Flags.setZExt(); |
1996 | |
1997 | for (unsigned i = 0; i < NumParts; ++i) { |
1998 | Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(), |
1999 | VT, /*isfixed=*/true, 0, 0)); |
2000 | OutVals.push_back(Parts[i]); |
2001 | } |
2002 | } |
2003 | } |
2004 | } |
2005 | |
2006 | // Push in swifterror virtual register as the last element of Outs. This makes |
2007 | // sure swifterror virtual register will be returned in the swifterror |
2008 | // physical register. |
2009 | const Function *F = I.getParent()->getParent(); |
2010 | if (TLI.supportSwiftError() && |
2011 | F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) { |
2012 | assert(SwiftError.getFunctionArg() && "Need a swift error argument")((void)0); |
2013 | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); |
2014 | Flags.setSwiftError(); |
2015 | Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/, |
2016 | EVT(TLI.getPointerTy(DL)) /*argvt*/, |
2017 | true /*isfixed*/, 1 /*origidx*/, |
2018 | 0 /*partOffs*/)); |
2019 | // Create SDNode for the swifterror virtual register. |
2020 | OutVals.push_back( |
2021 | DAG.getRegister(SwiftError.getOrCreateVRegUseAt( |
2022 | &I, FuncInfo.MBB, SwiftError.getFunctionArg()), |
2023 | EVT(TLI.getPointerTy(DL)))); |
2024 | } |
2025 | |
2026 | bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg(); |
2027 | CallingConv::ID CallConv = |
2028 | DAG.getMachineFunction().getFunction().getCallingConv(); |
2029 | Chain = DAG.getTargetLoweringInfo().LowerReturn( |
2030 | Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG); |
2031 | |
2032 | // Verify that the target's LowerReturn behaved as expected. |
2033 | assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&((void)0) |
2034 | "LowerReturn didn't return a valid chain!")((void)0); |
2035 | |
2036 | // Update the DAG with the new chain value resulting from return lowering. |
2037 | DAG.setRoot(Chain); |
2038 | } |
2039 | |
2040 | /// CopyToExportRegsIfNeeded - If the given value has virtual registers |
2041 | /// created for it, emit nodes to copy the value into the virtual |
2042 | /// registers. |
2043 | void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { |
2044 | // Skip empty types |
2045 | if (V->getType()->isEmptyTy()) |
2046 | return; |
2047 | |
2048 | DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V); |
2049 | if (VMI != FuncInfo.ValueMap.end()) { |
2050 | assert(!V->use_empty() && "Unused value assigned virtual registers!")((void)0); |
2051 | CopyValueToVirtualRegister(V, VMI->second); |
2052 | } |
2053 | } |
2054 | |
2055 | /// ExportFromCurrentBlock - If this condition isn't known to be exported from |
2056 | /// the current basic block, add it to ValueMap now so that we'll get a |
2057 | /// CopyTo/FromReg. |
2058 | void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { |
2059 | // No need to export constants. |
2060 | if (!isa<Instruction>(V) && !isa<Argument>(V)) return; |
2061 | |
2062 | // Already exported? |
2063 | if (FuncInfo.isExportedInst(V)) return; |
2064 | |
2065 | unsigned Reg = FuncInfo.InitializeRegForValue(V); |
2066 | CopyValueToVirtualRegister(V, Reg); |
2067 | } |
2068 | |
2069 | bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, |
2070 | const BasicBlock *FromBB) { |
2071 | // The operands of the setcc have to be in this block. We don't know |
2072 | // how to export them from some other block. |
2073 | if (const Instruction *VI = dyn_cast<Instruction>(V)) { |
2074 | // Can export from current BB. |
2075 | if (VI->getParent() == FromBB) |
2076 | return true; |
2077 | |
2078 | // Is already exported, noop. |
2079 | return FuncInfo.isExportedInst(V); |
2080 | } |
2081 | |
2082 | // If this is an argument, we can export it if the BB is the entry block or |
2083 | // if it is already exported. |
2084 | if (isa<Argument>(V)) { |
2085 | if (FromBB->isEntryBlock()) |
2086 | return true; |
2087 | |
2088 | // Otherwise, can only export this if it is already exported. |
2089 | return FuncInfo.isExportedInst(V); |
2090 | } |
2091 | |
2092 | // Otherwise, constants can always be exported. |
2093 | return true; |
2094 | } |
2095 | |
2096 | /// Return branch probability calculated by BranchProbabilityInfo for IR blocks. |
2097 | BranchProbability |
2098 | SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src, |
2099 | const MachineBasicBlock *Dst) const { |
2100 | BranchProbabilityInfo *BPI = FuncInfo.BPI; |
2101 | const BasicBlock *SrcBB = Src->getBasicBlock(); |
2102 | const BasicBlock *DstBB = Dst->getBasicBlock(); |
2103 | if (!BPI) { |
2104 | // If BPI is not available, set the default probability as 1 / N, where N is |
2105 | // the number of successors. |
2106 | auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1); |
2107 | return BranchProbability(1, SuccSize); |
2108 | } |
2109 | return BPI->getEdgeProbability(SrcBB, DstBB); |
2110 | } |
2111 | |
2112 | void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src, |
2113 | MachineBasicBlock *Dst, |
2114 | BranchProbability Prob) { |
2115 | if (!FuncInfo.BPI) |
2116 | Src->addSuccessorWithoutProb(Dst); |
2117 | else { |
2118 | if (Prob.isUnknown()) |
2119 | Prob = getEdgeProbability(Src, Dst); |
2120 | Src->addSuccessor(Dst, Prob); |
2121 | } |
2122 | } |
2123 | |
2124 | static bool InBlock(const Value *V, const BasicBlock *BB) { |
2125 | if (const Instruction *I = dyn_cast<Instruction>(V)) |
2126 | return I->getParent() == BB; |
2127 | return true; |
2128 | } |
2129 | |
2130 | /// EmitBranchForMergedCondition - Helper method for FindMergedConditions. |
2131 | /// This function emits a branch and is used at the leaves of an OR or an |
2132 | /// AND operator tree. |
2133 | void |
2134 | SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, |
2135 | MachineBasicBlock *TBB, |
2136 | MachineBasicBlock *FBB, |
2137 | MachineBasicBlock *CurBB, |
2138 | MachineBasicBlock *SwitchBB, |
2139 | BranchProbability TProb, |
2140 | BranchProbability FProb, |
2141 | bool InvertCond) { |
2142 | const BasicBlock *BB = CurBB->getBasicBlock(); |
2143 | |
2144 | // If the leaf of the tree is a comparison, merge the condition into |
2145 | // the caseblock. |
2146 | if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { |
2147 | // The operands of the cmp have to be in this block. We don't know |
2148 | // how to export them from some other block. If this is the first block |
2149 | // of the sequence, no exporting is needed. |
2150 | if (CurBB == SwitchBB || |
2151 | (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && |
2152 | isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { |
2153 | ISD::CondCode Condition; |
2154 | if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { |
2155 | ICmpInst::Predicate Pred = |
2156 | InvertCond ? IC->getInversePredicate() : IC->getPredicate(); |
2157 | Condition = getICmpCondCode(Pred); |
2158 | } else { |
2159 | const FCmpInst *FC = cast<FCmpInst>(Cond); |
2160 | FCmpInst::Predicate Pred = |
2161 | InvertCond ? FC->getInversePredicate() : FC->getPredicate(); |
2162 | Condition = getFCmpCondCode(Pred); |
2163 | if (TM.Options.NoNaNsFPMath) |
2164 | Condition = getFCmpCodeWithoutNaN(Condition); |
2165 | } |
2166 | |
2167 | CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, |
2168 | TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); |
2169 | SL->SwitchCases.push_back(CB); |
2170 | return; |
2171 | } |
2172 | } |
2173 | |
2174 | // Create a CaseBlock record representing this branch. |
2175 | ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ; |
2176 | CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), |
2177 | nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); |
2178 | SL->SwitchCases.push_back(CB); |
2179 | } |
2180 | |
2181 | void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, |
2182 | MachineBasicBlock *TBB, |
2183 | MachineBasicBlock *FBB, |
2184 | MachineBasicBlock *CurBB, |
2185 | MachineBasicBlock *SwitchBB, |
2186 | Instruction::BinaryOps Opc, |
2187 | BranchProbability TProb, |
2188 | BranchProbability FProb, |
2189 | bool InvertCond) { |
2190 | // Skip over not part of the tree and remember to invert op and operands at |
2191 | // next level. |
2192 | Value *NotCond; |
2193 | if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) && |
2194 | InBlock(NotCond, CurBB->getBasicBlock())) { |
2195 | FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb, |
2196 | !InvertCond); |
2197 | return; |
2198 | } |
2199 | |
2200 | const Instruction *BOp = dyn_cast<Instruction>(Cond); |
2201 | const Value *BOpOp0, *BOpOp1; |
2202 | // Compute the effective opcode for Cond, taking into account whether it needs |
2203 | // to be inverted, e.g. |
2204 | // and (not (or A, B)), C |
2205 | // gets lowered as |
2206 | // and (and (not A, not B), C) |
2207 | Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0; |
2208 | if (BOp) { |
2209 | BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1))) |
2210 | ? Instruction::And |
2211 | : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1))) |
2212 | ? Instruction::Or |
2213 | : (Instruction::BinaryOps)0); |
2214 | if (InvertCond) { |
2215 | if (BOpc == Instruction::And) |
2216 | BOpc = Instruction::Or; |
2217 | else if (BOpc == Instruction::Or) |
2218 | BOpc = Instruction::And; |
2219 | } |
2220 | } |
2221 | |
2222 | // If this node is not part of the or/and tree, emit it as a branch. |
2223 | // Note that all nodes in the tree should have same opcode. |
2224 | bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse(); |
2225 | if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() || |
2226 | !InBlock(BOpOp0, CurBB->getBasicBlock()) || |
2227 | !InBlock(BOpOp1, CurBB->getBasicBlock())) { |
2228 | EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, |
2229 | TProb, FProb, InvertCond); |
2230 | return; |
2231 | } |
2232 | |
2233 | // Create TmpBB after CurBB. |
2234 | MachineFunction::iterator BBI(CurBB); |
2235 | MachineFunction &MF = DAG.getMachineFunction(); |
2236 | MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); |
2237 | CurBB->getParent()->insert(++BBI, TmpBB); |
2238 | |
2239 | if (Opc == Instruction::Or) { |
2240 | // Codegen X | Y as: |
2241 | // BB1: |
2242 | // jmp_if_X TBB |
2243 | // jmp TmpBB |
2244 | // TmpBB: |
2245 | // jmp_if_Y TBB |
2246 | // jmp FBB |
2247 | // |
2248 | |
2249 | // We have flexibility in setting Prob for BB1 and Prob for TmpBB. |
2250 | // The requirement is that |
2251 | // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) |
2252 | // = TrueProb for original BB. |
2253 | // Assuming the original probabilities are A and B, one choice is to set |
2254 | // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to |
2255 | // A/(1+B) and 2B/(1+B). This choice assumes that |
2256 | // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. |
2257 | // Another choice is to assume TrueProb for BB1 equals to TrueProb for |
2258 | // TmpBB, but the math is more complicated. |
2259 | |
2260 | auto NewTrueProb = TProb / 2; |
2261 | auto NewFalseProb = TProb / 2 + FProb; |
2262 | // Emit the LHS condition. |
2263 | FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb, |
2264 | NewFalseProb, InvertCond); |
2265 | |
2266 | // Normalize A/2 and B to get A/(1+B) and 2B/(1+B). |
2267 | SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb}; |
2268 | BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); |
2269 | // Emit the RHS condition into TmpBB. |
2270 | FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], |
2271 | Probs[1], InvertCond); |
2272 | } else { |
2273 | assert(Opc == Instruction::And && "Unknown merge op!")((void)0); |
2274 | // Codegen X & Y as: |
2275 | // BB1: |
2276 | // jmp_if_X TmpBB |
2277 | // jmp FBB |
2278 | // TmpBB: |
2279 | // jmp_if_Y TBB |
2280 | // jmp FBB |
2281 | // |
2282 | // This requires creation of TmpBB after CurBB. |
2283 | |
2284 | // We have flexibility in setting Prob for BB1 and Prob for TmpBB. |
2285 | // The requirement is that |
2286 | // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) |
2287 | // = FalseProb for original BB. |
2288 | // Assuming the original probabilities are A and B, one choice is to set |
2289 | // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to |
2290 | // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 == |
2291 | // TrueProb for BB1 * FalseProb for TmpBB. |
2292 | |
2293 | auto NewTrueProb = TProb + FProb / 2; |
2294 | auto NewFalseProb = FProb / 2; |
2295 | // Emit the LHS condition. |
2296 | FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb, |
2297 | NewFalseProb, InvertCond); |
2298 | |
2299 | // Normalize A and B/2 to get 2A/(1+A) and B/(1+A). |
2300 | SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2}; |
2301 | BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); |
2302 | // Emit the RHS condition into TmpBB. |
2303 | FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], |
2304 | Probs[1], InvertCond); |
2305 | } |
2306 | } |
2307 | |
2308 | /// If the set of cases should be emitted as a series of branches, return true. |
2309 | /// If we should emit this as a bunch of and/or'd together conditions, return |
2310 | /// false. |
2311 | bool |
2312 | SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { |
2313 | if (Cases.size() != 2) return true; |
2314 | |
2315 | // If this is two comparisons of the same values or'd or and'd together, they |
2316 | // will get folded into a single comparison, so don't emit two blocks. |
2317 | if ((Cases[0].CmpLHS == Cases[1].CmpLHS && |
2318 | Cases[0].CmpRHS == Cases[1].CmpRHS) || |
2319 | (Cases[0].CmpRHS == Cases[1].CmpLHS && |
2320 | Cases[0].CmpLHS == Cases[1].CmpRHS)) { |
2321 | return false; |
2322 | } |
2323 | |
2324 | // Handle: (X != null) | (Y != null) --> (X|Y) != 0 |
2325 | // Handle: (X == null) & (Y == null) --> (X|Y) == 0 |
2326 | if (Cases[0].CmpRHS == Cases[1].CmpRHS && |
2327 | Cases[0].CC == Cases[1].CC && |
2328 | isa<Constant>(Cases[0].CmpRHS) && |
2329 | cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { |
2330 | if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) |
2331 | return false; |
2332 | if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) |
2333 | return false; |
2334 | } |
2335 | |
2336 | return true; |
2337 | } |
2338 | |
2339 | void SelectionDAGBuilder::visitBr(const BranchInst &I) { |
2340 | MachineBasicBlock *BrMBB = FuncInfo.MBB; |
2341 | |
2342 | // Update machine-CFG edges. |
2343 | MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; |
2344 | |
2345 | if (I.isUnconditional()) { |
2346 | // Update machine-CFG edges. |
2347 | BrMBB->addSuccessor(Succ0MBB); |
2348 | |
2349 | // If this is not a fall-through branch or optimizations are switched off, |
2350 | // emit the branch. |
2351 | if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None) |
2352 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), |
2353 | MVT::Other, getControlRoot(), |
2354 | DAG.getBasicBlock(Succ0MBB))); |
2355 | |
2356 | return; |
2357 | } |
2358 | |
2359 | // If this condition is one of the special cases we handle, do special stuff |
2360 | // now. |
2361 | const Value *CondVal = I.getCondition(); |
2362 | MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; |
2363 | |
2364 | // If this is a series of conditions that are or'd or and'd together, emit |
2365 | // this as a sequence of branches instead of setcc's with and/or operations. |
2366 | // As long as jumps are not expensive (exceptions for multi-use logic ops, |
2367 | // unpredictable branches, and vector extracts because those jumps are likely |
2368 | // expensive for any target), this should improve performance. |
2369 | // For example, instead of something like: |
2370 | // cmp A, B |
2371 | // C = seteq |
2372 | // cmp D, E |
2373 | // F = setle |
2374 | // or C, F |
2375 | // jnz foo |
2376 | // Emit: |
2377 | // cmp A, B |
2378 | // je foo |
2379 | // cmp D, E |
2380 | // jle foo |
2381 | const Instruction *BOp = dyn_cast<Instruction>(CondVal); |
2382 | if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp && |
2383 | BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) { |
2384 | Value *Vec; |
2385 | const Value *BOp0, *BOp1; |
2386 | Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0; |
2387 | if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1)))) |
2388 | Opcode = Instruction::And; |
2389 | else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1)))) |
2390 | Opcode = Instruction::Or; |
2391 | |
2392 | if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) && |
2393 | match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) { |
2394 | FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode, |
2395 | getEdgeProbability(BrMBB, Succ0MBB), |
2396 | getEdgeProbability(BrMBB, Succ1MBB), |
2397 | /*InvertCond=*/false); |
2398 | // If the compares in later blocks need to use values not currently |
2399 | // exported from this block, export them now. This block should always |
2400 | // be the first entry. |
2401 | assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")((void)0); |
2402 | |
2403 | // Allow some cases to be rejected. |
2404 | if (ShouldEmitAsBranches(SL->SwitchCases)) { |
2405 | for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) { |
2406 | ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS); |
2407 | ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS); |
2408 | } |
2409 | |
2410 | // Emit the branch for this block. |
2411 | visitSwitchCase(SL->SwitchCases[0], BrMBB); |
2412 | SL->SwitchCases.erase(SL->SwitchCases.begin()); |
2413 | return; |
2414 | } |
2415 | |
2416 | // Okay, we decided not to do this, remove any inserted MBB's and clear |
2417 | // SwitchCases. |
2418 | for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) |
2419 | FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB); |
2420 | |
2421 | SL->SwitchCases.clear(); |
2422 | } |
2423 | } |
2424 | |
2425 | // Create a CaseBlock record representing this branch. |
2426 | CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), |
2427 | nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc()); |
2428 | |
2429 | // Use visitSwitchCase to actually insert the fast branch sequence for this |
2430 | // cond branch. |
2431 | visitSwitchCase(CB, BrMBB); |
2432 | } |
2433 | |
2434 | /// visitSwitchCase - Emits the necessary code to represent a single node in |
2435 | /// the binary search tree resulting from lowering a switch instruction. |
2436 | void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, |
2437 | MachineBasicBlock *SwitchBB) { |
2438 | SDValue Cond; |
2439 | SDValue CondLHS = getValue(CB.CmpLHS); |
2440 | SDLoc dl = CB.DL; |
2441 | |
2442 | if (CB.CC == ISD::SETTRUE) { |
2443 | // Branch or fall through to TrueBB. |
2444 | addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); |
2445 | SwitchBB->normalizeSuccProbs(); |
2446 | if (CB.TrueBB != NextBlock(SwitchBB)) { |
2447 | DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(), |
2448 | DAG.getBasicBlock(CB.TrueBB))); |
2449 | } |
2450 | return; |
2451 | } |
2452 | |
2453 | auto &TLI = DAG.getTargetLoweringInfo(); |
2454 | EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType()); |
2455 | |
2456 | // Build the setcc now. |
2457 | if (!CB.CmpMHS) { |
2458 | // Fold "(X == true)" to X and "(X == false)" to !X to |
2459 | // handle common cases produced by branch lowering. |
2460 | if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && |
2461 | CB.CC == ISD::SETEQ) |
2462 | Cond = CondLHS; |
2463 | else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && |
2464 | CB.CC == ISD::SETEQ) { |
2465 | SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType()); |
2466 | Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); |
2467 | } else { |
2468 | SDValue CondRHS = getValue(CB.CmpRHS); |
2469 | |
2470 | // If a pointer's DAG type is larger than its memory type then the DAG |
2471 | // values are zero-extended. This breaks signed comparisons so truncate |
2472 | // back to the underlying type before doing the compare. |
2473 | if (CondLHS.getValueType() != MemVT) { |
2474 | CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT); |
2475 | CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT); |
2476 | } |
2477 | Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC); |
2478 | } |
2479 | } else { |
2480 | assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")((void)0); |
2481 | |
2482 | const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); |
2483 | const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); |
2484 | |
2485 | SDValue CmpOp = getValue(CB.CmpMHS); |
2486 | EVT VT = CmpOp.getValueType(); |
2487 | |
2488 | if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { |
2489 | Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT), |
2490 | ISD::SETLE); |
2491 | } else { |
2492 | SDValue SUB = DAG.getNode(ISD::SUB, dl, |
2493 | VT, CmpOp, DAG.getConstant(Low, dl, VT)); |
2494 | Cond = DAG.getSetCC(dl, MVT::i1, SUB, |
2495 | DAG.getConstant(High-Low, dl, VT), ISD::SETULE); |
2496 | } |
2497 | } |
2498 | |
2499 | // Update successor info |
2500 | addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); |
2501 | // TrueBB and FalseBB are always different unless the incoming IR is |
2502 | // degenerate. This only happens when running llc on weird IR. |
2503 | if (CB.TrueBB != CB.FalseBB) |
2504 | addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb); |
2505 | SwitchBB->normalizeSuccProbs(); |
2506 | |
2507 | // If the lhs block is the next block, invert the condition so that we can |
2508 | // fall through to the lhs instead of the rhs block. |
2509 | if (CB.TrueBB == NextBlock(SwitchBB)) { |
2510 | std::swap(CB.TrueBB, CB.FalseBB); |
2511 | SDValue True = DAG.getConstant(1, dl, Cond.getValueType()); |
2512 | Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); |
2513 | } |
2514 | |
2515 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, |
2516 | MVT::Other, getControlRoot(), Cond, |
2517 | DAG.getBasicBlock(CB.TrueBB)); |
2518 | |
2519 | // Insert the false branch. Do this even if it's a fall through branch, |
2520 | // this makes it easier to do DAG optimizations which require inverting |
2521 | // the branch condition. |
2522 | BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, |
2523 | DAG.getBasicBlock(CB.FalseBB)); |
2524 | |
2525 | DAG.setRoot(BrCond); |
2526 | } |
2527 | |
2528 | /// visitJumpTable - Emit JumpTable node in the current MBB |
2529 | void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) { |
2530 | // Emit the code for the jump table |
2531 | assert(JT.Reg != -1U && "Should lower JT Header first!")((void)0); |
2532 | EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); |
2533 | SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), |
2534 | JT.Reg, PTy); |
2535 | SDValue Table = DAG.getJumpTable(JT.JTI, PTy); |
2536 | SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), |
2537 | MVT::Other, Index.getValue(1), |
2538 | Table, Index); |
2539 | DAG.setRoot(BrJumpTable); |
2540 | } |
2541 | |
2542 | /// visitJumpTableHeader - This function emits necessary code to produce index |
2543 | /// in the JumpTable from switch case. |
2544 | void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT, |
2545 | JumpTableHeader &JTH, |
2546 | MachineBasicBlock *SwitchBB) { |
2547 | SDLoc dl = getCurSDLoc(); |
2548 | |
2549 | // Subtract the lowest switch case value from the value being switched on. |
2550 | SDValue SwitchOp = getValue(JTH.SValue); |
2551 | EVT VT = SwitchOp.getValueType(); |
2552 | SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, |
2553 | DAG.getConstant(JTH.First, dl, VT)); |
2554 | |
2555 | // The SDNode we just created, which holds the value being switched on minus |
2556 | // the smallest case value, needs to be copied to a virtual register so it |
2557 | // can be used as an index into the jump table in a subsequent basic block. |
2558 | // This value may be smaller or larger than the target's pointer type, and |
2559 | // therefore require extension or truncating. |
2560 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
2561 | SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout())); |
2562 | |
2563 | unsigned JumpTableReg = |
2564 | FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout())); |
2565 | SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, |
2566 | JumpTableReg, SwitchOp); |
2567 | JT.Reg = JumpTableReg; |
2568 | |
2569 | if (!JTH.OmitRangeCheck) { |
2570 | // Emit the range check for the jump table, and branch to the default block |
2571 | // for the switch statement if the value being switched on exceeds the |
2572 | // largest case in the switch. |
2573 | SDValue CMP = DAG.getSetCC( |
2574 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), |
2575 | Sub.getValueType()), |
2576 | Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); |
2577 | |
2578 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, |
2579 | MVT::Other, CopyTo, CMP, |
2580 | DAG.getBasicBlock(JT.Default)); |
2581 | |
2582 | // Avoid emitting unnecessary branches to the next block. |
2583 | if (JT.MBB != NextBlock(SwitchBB)) |
2584 | BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, |
2585 | DAG.getBasicBlock(JT.MBB)); |
2586 | |
2587 | DAG.setRoot(BrCond); |
2588 | } else { |
2589 | // Avoid emitting unnecessary branches to the next block. |
2590 | if (JT.MBB != NextBlock(SwitchBB)) |
2591 | DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo, |
2592 | DAG.getBasicBlock(JT.MBB))); |
2593 | else |
2594 | DAG.setRoot(CopyTo); |
2595 | } |
2596 | } |
2597 | |
2598 | /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global |
2599 | /// variable if there exists one. |
2600 | static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, |
2601 | SDValue &Chain) { |
2602 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
2603 | EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); |
2604 | EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); |
2605 | MachineFunction &MF = DAG.getMachineFunction(); |
2606 | Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent()); |
2607 | MachineSDNode *Node = |
2608 | DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain); |
2609 | if (Global) { |
2610 | MachinePointerInfo MPInfo(Global); |
2611 | auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | |
2612 | MachineMemOperand::MODereferenceable; |
2613 | MachineMemOperand *MemRef = MF.getMachineMemOperand( |
2614 | MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy)); |
2615 | DAG.setNodeMemRefs(Node, {MemRef}); |
2616 | } |
2617 | if (PtrTy != PtrMemTy) |
2618 | return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy); |
2619 | return SDValue(Node, 0); |
2620 | } |
2621 | |
2622 | /// Codegen a new tail for a stack protector check ParentMBB which has had its |
2623 | /// tail spliced into a stack protector check success bb. |
2624 | /// |
2625 | /// For a high level explanation of how this fits into the stack protector |
2626 | /// generation see the comment on the declaration of class |
2627 | /// StackProtectorDescriptor. |
2628 | void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, |
2629 | MachineBasicBlock *ParentBB) { |
2630 | |
2631 | // First create the loads to the guard/stack slot for the comparison. |
2632 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
2633 | EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); |
2634 | EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); |
2635 | |
2636 | MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo(); |
2637 | int FI = MFI.getStackProtectorIndex(); |
2638 | |
2639 | SDValue Guard; |
2640 | SDLoc dl = getCurSDLoc(); |
2641 | SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); |
2642 | const Module &M = *ParentBB->getParent()->getFunction().getParent(); |
2643 | Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext())); |
2644 | |
2645 | // Generate code to load the content of the guard slot. |
2646 | SDValue GuardVal = DAG.getLoad( |
2647 | PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr, |
2648 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align, |
2649 | MachineMemOperand::MOVolatile); |
2650 | |
2651 | if (TLI.useStackGuardXorFP()) |
2652 | GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl); |
2653 | |
2654 | // Retrieve guard check function, nullptr if instrumentation is inlined. |
2655 | if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) { |
2656 | // The target provides a guard check function to validate the guard value. |
2657 | // Generate a call to that function with the content of the guard slot as |
2658 | // argument. |
2659 | FunctionType *FnTy = GuardCheckFn->getFunctionType(); |
2660 | assert(FnTy->getNumParams() == 1 && "Invalid function signature")((void)0); |
2661 | |
2662 | TargetLowering::ArgListTy Args; |
2663 | TargetLowering::ArgListEntry Entry; |
2664 | Entry.Node = GuardVal; |
2665 | Entry.Ty = FnTy->getParamType(0); |
2666 | if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg)) |
2667 | Entry.IsInReg = true; |
2668 | Args.push_back(Entry); |
2669 | |
2670 | TargetLowering::CallLoweringInfo CLI(DAG); |
2671 | CLI.setDebugLoc(getCurSDLoc()) |
2672 | .setChain(DAG.getEntryNode()) |
2673 | .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(), |
2674 | getValue(GuardCheckFn), std::move(Args)); |
2675 | |
2676 | std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); |
2677 | DAG.setRoot(Result.second); |
2678 | return; |
2679 | } |
2680 | |
2681 | // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD. |
2682 | // Otherwise, emit a volatile load to retrieve the stack guard value. |
2683 | SDValue Chain = DAG.getEntryNode(); |
2684 | if (TLI.useLoadStackGuardNode()) { |
2685 | Guard = getLoadStackGuard(DAG, dl, Chain); |
2686 | } else { |
2687 | const Value *IRGuard = TLI.getSDagStackGuard(M); |
2688 | SDValue GuardPtr = getValue(IRGuard); |
2689 | |
2690 | Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, |
2691 | MachinePointerInfo(IRGuard, 0), Align, |
2692 | MachineMemOperand::MOVolatile); |
2693 | } |
2694 | |
2695 | // Perform the comparison via a getsetcc. |
2696 | SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(), |
2697 | *DAG.getContext(), |
2698 | Guard.getValueType()), |
2699 | Guard, GuardVal, ISD::SETNE); |
2700 | |
2701 | // If the guard/stackslot do not equal, branch to failure MBB. |
2702 | SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, |
2703 | MVT::Other, GuardVal.getOperand(0), |
2704 | Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); |
2705 | // Otherwise branch to success MBB. |
2706 | SDValue Br = DAG.getNode(ISD::BR, dl, |
2707 | MVT::Other, BrCond, |
2708 | DAG.getBasicBlock(SPD.getSuccessMBB())); |
2709 | |
2710 | DAG.setRoot(Br); |
2711 | } |
2712 | |
2713 | /// Codegen the failure basic block for a stack protector check. |
2714 | /// |
2715 | /// A failure stack protector machine basic block consists simply of a call to |
2716 | /// __stack_chk_fail(). |
2717 | /// |
2718 | /// For a high level explanation of how this fits into the stack protector |
2719 | /// generation see the comment on the declaration of class |
2720 | /// StackProtectorDescriptor. |
2721 | void |
2722 | SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { |
2723 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
2724 | TargetLowering::MakeLibCallOptions CallOptions; |
2725 | CallOptions.setDiscardResult(true); |
2726 | SDValue Chain = |
2727 | TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid, |
2728 | None, CallOptions, getCurSDLoc()).second; |
2729 | // On PS4, the "return address" must still be within the calling function, |
2730 | // even if it's at the very end, so emit an explicit TRAP here. |
2731 | // Passing 'true' for doesNotReturn above won't generate the trap for us. |
2732 | if (TM.getTargetTriple().isPS4CPU()) |
2733 | Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); |
2734 | // WebAssembly needs an unreachable instruction after a non-returning call, |
2735 | // because the function return type can be different from __stack_chk_fail's |
2736 | // return type (void). |
2737 | if (TM.getTargetTriple().isWasm()) |
2738 | Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); |
2739 | |
2740 | DAG.setRoot(Chain); |
2741 | } |
2742 | |
2743 | /// visitBitTestHeader - This function emits necessary code to produce value |
2744 | /// suitable for "bit tests" |
2745 | void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, |
2746 | MachineBasicBlock *SwitchBB) { |
2747 | SDLoc dl = getCurSDLoc(); |
2748 | |
2749 | // Subtract the minimum value. |
2750 | SDValue SwitchOp = getValue(B.SValue); |
2751 | EVT VT = SwitchOp.getValueType(); |
2752 | SDValue RangeSub = |
2753 | DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT)); |
2754 | |
2755 | // Determine the type of the test operands. |
2756 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
2757 | bool UsePtrType = false; |
2758 | if (!TLI.isTypeLegal(VT)) { |
2759 | UsePtrType = true; |
2760 | } else { |
2761 | for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) |
2762 | if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { |
2763 | // Switch table case range are encoded into series of masks. |
2764 | // Just use pointer type, it's guaranteed to fit. |
2765 | UsePtrType = true; |
2766 | break; |
2767 | } |
2768 | } |
2769 | SDValue Sub = RangeSub; |
2770 | if (UsePtrType) { |
2771 | VT = TLI.getPointerTy(DAG.getDataLayout()); |
2772 | Sub = DAG.getZExtOrTrunc(Sub, dl, VT); |
2773 | } |
2774 | |
2775 | B.RegVT = VT.getSimpleVT(); |
2776 | B.Reg = FuncInfo.CreateReg(B.RegVT); |
2777 | SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub); |
2778 | |
2779 | MachineBasicBlock* MBB = B.Cases[0].ThisBB; |
2780 | |
2781 | if (!B.OmitRangeCheck) |
2782 | addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb); |
2783 | addSuccessorWithProb(SwitchBB, MBB, B.Prob); |
2784 | SwitchBB->normalizeSuccProbs(); |
2785 | |
2786 | SDValue Root = CopyTo; |
2787 | if (!B.OmitRangeCheck) { |
2788 | // Conditional branch to the default block. |
2789 | SDValue RangeCmp = DAG.getSetCC(dl, |
2790 | TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), |
2791 | RangeSub.getValueType()), |
2792 | RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()), |
2793 | ISD::SETUGT); |
2794 | |
2795 | Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp, |
2796 | DAG.getBasicBlock(B.Default)); |
2797 | } |
2798 | |
2799 | // Avoid emitting unnecessary branches to the next block. |
2800 | if (MBB != NextBlock(SwitchBB)) |
2801 | Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB)); |
2802 | |
2803 | DAG.setRoot(Root); |
2804 | } |
2805 | |
2806 | /// visitBitTestCase - this function produces one "bit test" |
2807 | void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, |
2808 | MachineBasicBlock* NextMBB, |
2809 | BranchProbability BranchProbToNext, |
2810 | unsigned Reg, |
2811 | BitTestCase &B, |
2812 | MachineBasicBlock *SwitchBB) { |
2813 | SDLoc dl = getCurSDLoc(); |
2814 | MVT VT = BB.RegVT; |
2815 | SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT); |
2816 | SDValue Cmp; |
2817 | unsigned PopCount = countPopulation(B.Mask); |
2818 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
2819 | if (PopCount == 1) { |
2820 | // Testing for a single bit; just compare the shift count with what it |
2821 | // would need to be to shift a 1 bit in that position. |
2822 | Cmp = DAG.getSetCC( |
2823 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), |
2824 | ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT), |
2825 | ISD::SETEQ); |
2826 | } else if (PopCount == BB.Range) { |
2827 | // There is only one zero bit in the range, test for it directly. |
2828 | Cmp = DAG.getSetCC( |
2829 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), |
2830 | ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT), |
2831 | ISD::SETNE); |
2832 | } else { |
2833 | // Make desired shift |
2834 | SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT, |
2835 | DAG.getConstant(1, dl, VT), ShiftOp); |
2836 | |
2837 | // Emit bit tests and jumps |
2838 | SDValue AndOp = DAG.getNode(ISD::AND, dl, |
2839 | VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT)); |
2840 | Cmp = DAG.getSetCC( |
2841 | dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), |
2842 | AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE); |
2843 | } |
2844 | |
2845 | // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb. |
2846 | addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb); |
2847 | // The branch probability from SwitchBB to NextMBB is BranchProbToNext. |
2848 | addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext); |
2849 | // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is |
2850 | // one as they are relative probabilities (and thus work more like weights), |
2851 | // and hence we need to normalize them to let the sum of them become one. |
2852 | SwitchBB->normalizeSuccProbs(); |
2853 | |
2854 | SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, |
2855 | MVT::Other, getControlRoot(), |
2856 | Cmp, DAG.getBasicBlock(B.TargetBB)); |
2857 | |
2858 | // Avoid emitting unnecessary branches to the next block. |
2859 | if (NextMBB != NextBlock(SwitchBB)) |
2860 | BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, |
2861 | DAG.getBasicBlock(NextMBB)); |
2862 | |
2863 | DAG.setRoot(BrAnd); |
2864 | } |
2865 | |
2866 | void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { |
2867 | MachineBasicBlock *InvokeMBB = FuncInfo.MBB; |
2868 | |
2869 | // Retrieve successors. Look through artificial IR level blocks like |
2870 | // catchswitch for successors. |
2871 | MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; |
2872 | const BasicBlock *EHPadBB = I.getSuccessor(1); |
2873 | |
2874 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't |
2875 | // have to do anything here to lower funclet bundles. |
2876 | assert(!I.hasOperandBundlesOtherThan(((void)0) |
2877 | {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,((void)0) |
2878 | LLVMContext::OB_gc_live, LLVMContext::OB_funclet,((void)0) |
2879 | LLVMContext::OB_cfguardtarget,((void)0) |
2880 | LLVMContext::OB_clang_arc_attachedcall}) &&((void)0) |
2881 | "Cannot lower invokes with arbitrary operand bundles yet!")((void)0); |
2882 | |
2883 | const Value *Callee(I.getCalledOperand()); |
2884 | const Function *Fn = dyn_cast<Function>(Callee); |
2885 | if (isa<InlineAsm>(Callee)) |
2886 | visitInlineAsm(I, EHPadBB); |
2887 | else if (Fn && Fn->isIntrinsic()) { |
2888 | switch (Fn->getIntrinsicID()) { |
2889 | default: |
2890 | llvm_unreachable("Cannot invoke this intrinsic")__builtin_unreachable(); |
2891 | case Intrinsic::donothing: |
2892 | // Ignore invokes to @llvm.donothing: jump directly to the next BB. |
2893 | case Intrinsic::seh_try_begin: |
2894 | case Intrinsic::seh_scope_begin: |
2895 | case Intrinsic::seh_try_end: |
2896 | case Intrinsic::seh_scope_end: |
2897 | break; |
2898 | case Intrinsic::experimental_patchpoint_void: |
2899 | case Intrinsic::experimental_patchpoint_i64: |
2900 | visitPatchpoint(I, EHPadBB); |
2901 | break; |
2902 | case Intrinsic::experimental_gc_statepoint: |
2903 | LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB); |
2904 | break; |
2905 | case Intrinsic::wasm_rethrow: { |
2906 | // This is usually done in visitTargetIntrinsic, but this intrinsic is |
2907 | // special because it can be invoked, so we manually lower it to a DAG |
2908 | // node here. |
2909 | SmallVector<SDValue, 8> Ops; |
2910 | Ops.push_back(getRoot()); // inchain |
2911 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
2912 | Ops.push_back( |
2913 | DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(), |
2914 | TLI.getPointerTy(DAG.getDataLayout()))); |
2915 | SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain |
2916 | DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops)); |
2917 | break; |
2918 | } |
2919 | } |
2920 | } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) { |
2921 | // Currently we do not lower any intrinsic calls with deopt operand bundles. |
2922 | // Eventually we will support lowering the @llvm.experimental.deoptimize |
2923 | // intrinsic, and right now there are no plans to support other intrinsics |
2924 | // with deopt state. |
2925 | LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB); |
2926 | } else { |
2927 | LowerCallTo(I, getValue(Callee), false, false, EHPadBB); |
2928 | } |
2929 | |
2930 | // If the value of the invoke is used outside of its defining block, make it |
2931 | // available as a virtual register. |
2932 | // We already took care of the exported value for the statepoint instruction |
2933 | // during call to the LowerStatepoint. |
2934 | if (!isa<GCStatepointInst>(I)) { |
2935 | CopyToExportRegsIfNeeded(&I); |
2936 | } |
2937 | |
2938 | SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; |
2939 | BranchProbabilityInfo *BPI = FuncInfo.BPI; |
2940 | BranchProbability EHPadBBProb = |
2941 | BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB) |
2942 | : BranchProbability::getZero(); |
2943 | findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests); |
2944 | |
2945 | // Update successor info. |
2946 | addSuccessorWithProb(InvokeMBB, Return); |
2947 | for (auto &UnwindDest : UnwindDests) { |
2948 | UnwindDest.first->setIsEHPad(); |
2949 | addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second); |
2950 | } |
2951 | InvokeMBB->normalizeSuccProbs(); |
2952 | |
2953 | // Drop into normal successor. |
2954 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), |
2955 | DAG.getBasicBlock(Return))); |
2956 | } |
2957 | |
2958 | void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { |
2959 | MachineBasicBlock *CallBrMBB = FuncInfo.MBB; |
2960 | |
2961 | // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't |
2962 | // have to do anything here to lower funclet bundles. |
2963 | assert(!I.hasOperandBundlesOtherThan(((void)0) |
2964 | {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&((void)0) |
2965 | "Cannot lower callbrs with arbitrary operand bundles yet!")((void)0); |
2966 | |
2967 | assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")((void)0); |
2968 | visitInlineAsm(I); |
2969 | CopyToExportRegsIfNeeded(&I); |
2970 | |
2971 | // Retrieve successors. |
2972 | MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()]; |
2973 | |
2974 | // Update successor info. |
2975 | addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); |
2976 | for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { |
2977 | MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)]; |
2978 | addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero()); |
2979 | Target->setIsInlineAsmBrIndirectTarget(); |
2980 | } |
2981 | CallBrMBB->normalizeSuccProbs(); |
2982 | |
2983 | // Drop into default successor. |
2984 | DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), |
2985 | MVT::Other, getControlRoot(), |
2986 | DAG.getBasicBlock(Return))); |
2987 | } |
2988 | |
2989 | void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { |
2990 | llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")__builtin_unreachable(); |
2991 | } |
2992 | |
2993 | void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { |
2994 | assert(FuncInfo.MBB->isEHPad() &&((void)0) |
2995 | "Call to landingpad not in landing pad!")((void)0); |
2996 | |
2997 | // If there aren't registers to copy the values into (e.g., during SjLj |
2998 | // exceptions), then don't bother to create these DAG nodes. |
2999 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3000 | const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn(); |
3001 | if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 && |
3002 | TLI.getExceptionSelectorRegister(PersonalityFn) == 0) |
3003 | return; |
3004 | |
3005 | // If landingpad's return type is token type, we don't create DAG nodes |
3006 | // for its exception pointer and selector value. The extraction of exception |
3007 | // pointer or selector value from token type landingpads is not currently |
3008 | // supported. |
3009 | if (LP.getType()->isTokenTy()) |
3010 | return; |
3011 | |
3012 | SmallVector<EVT, 2> ValueVTs; |
3013 | SDLoc dl = getCurSDLoc(); |
3014 | ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs); |
3015 | assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")((void)0); |
3016 | |
3017 | // Get the two live-in registers as SDValues. The physregs have already been |
3018 | // copied into virtual registers. |
3019 | SDValue Ops[2]; |
3020 | if (FuncInfo.ExceptionPointerVirtReg) { |
3021 | Ops[0] = DAG.getZExtOrTrunc( |
3022 | DAG.getCopyFromReg(DAG.getEntryNode(), dl, |
3023 | FuncInfo.ExceptionPointerVirtReg, |
3024 | TLI.getPointerTy(DAG.getDataLayout())), |
3025 | dl, ValueVTs[0]); |
3026 | } else { |
3027 | Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())); |
3028 | } |
3029 | Ops[1] = DAG.getZExtOrTrunc( |
3030 | DAG.getCopyFromReg(DAG.getEntryNode(), dl, |
3031 | FuncInfo.ExceptionSelectorVirtReg, |
3032 | TLI.getPointerTy(DAG.getDataLayout())), |
3033 | dl, ValueVTs[1]); |
3034 | |
3035 | // Merge into one. |
3036 | SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, |
3037 | DAG.getVTList(ValueVTs), Ops); |
3038 | setValue(&LP, Res); |
3039 | } |
3040 | |
3041 | void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, |
3042 | MachineBasicBlock *Last) { |
3043 | // Update JTCases. |
3044 | for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i) |
3045 | if (SL->JTCases[i].first.HeaderBB == First) |
3046 | SL->JTCases[i].first.HeaderBB = Last; |
3047 | |
3048 | // Update BitTestCases. |
3049 | for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i) |
3050 | if (SL->BitTestCases[i].Parent == First) |
3051 | SL->BitTestCases[i].Parent = Last; |
3052 | } |
3053 | |
3054 | void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { |
3055 | MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB; |
3056 | |
3057 | // Update machine-CFG edges with unique successors. |
3058 | SmallSet<BasicBlock*, 32> Done; |
3059 | for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) { |
3060 | BasicBlock *BB = I.getSuccessor(i); |
3061 | bool Inserted = Done.insert(BB).second; |
3062 | if (!Inserted) |
3063 | continue; |
3064 | |
3065 | MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; |
3066 | addSuccessorWithProb(IndirectBrMBB, Succ); |
3067 | } |
3068 | IndirectBrMBB->normalizeSuccProbs(); |
3069 | |
3070 | DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), |
3071 | MVT::Other, getControlRoot(), |
3072 | getValue(I.getAddress()))); |
3073 | } |
3074 | |
3075 | void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) { |
3076 | if (!DAG.getTarget().Options.TrapUnreachable) |
3077 | return; |
3078 | |
3079 | // We may be able to ignore unreachable behind a noreturn call. |
3080 | if (DAG.getTarget().Options.NoTrapAfterNoreturn) { |
3081 | const BasicBlock &BB = *I.getParent(); |
3082 | if (&I != &BB.front()) { |
3083 | BasicBlock::const_iterator PredI = |
3084 | std::prev(BasicBlock::const_iterator(&I)); |
3085 | if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) { |
3086 | if (Call->doesNotReturn()) |
3087 | return; |
3088 | } |
3089 | } |
3090 | } |
3091 | |
3092 | DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot())); |
3093 | } |
3094 | |
3095 | void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) { |
3096 | SDNodeFlags Flags; |
3097 | |
3098 | SDValue Op = getValue(I.getOperand(0)); |
3099 | SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), |
3100 | Op, Flags); |
3101 | setValue(&I, UnNodeValue); |
3102 | } |
3103 | |
3104 | void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) { |
3105 | SDNodeFlags Flags; |
3106 | if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) { |
3107 | Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap()); |
3108 | Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap()); |
3109 | } |
3110 | if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) |
3111 | Flags.setExact(ExactOp->isExact()); |
3112 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) |
3113 | Flags.copyFMF(*FPOp); |
3114 | |
3115 | SDValue Op1 = getValue(I.getOperand(0)); |
3116 | SDValue Op2 = getValue(I.getOperand(1)); |
3117 | SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), |
3118 | Op1, Op2, Flags); |
3119 | setValue(&I, BinNodeValue); |
3120 | } |
3121 | |
3122 | void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { |
3123 | SDValue Op1 = getValue(I.getOperand(0)); |
3124 | SDValue Op2 = getValue(I.getOperand(1)); |
3125 | |
3126 | EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy( |
3127 | Op1.getValueType(), DAG.getDataLayout()); |
3128 | |
3129 | // Coerce the shift amount to the right type if we can. |
3130 | if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { |
3131 | unsigned ShiftSize = ShiftTy.getSizeInBits(); |
3132 | unsigned Op2Size = Op2.getValueSizeInBits(); |
3133 | SDLoc DL = getCurSDLoc(); |
3134 | |
3135 | // If the operand is smaller than the shift count type, promote it. |
3136 | if (ShiftSize > Op2Size) |
3137 | Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2); |
3138 | |
3139 | // If the operand is larger than the shift count type but the shift |
3140 | // count type has enough bits to represent any shift value, truncate |
3141 | // it now. This is a common case and it exposes the truncate to |
3142 | // optimization early. |
3143 | else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits())) |
3144 | Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2); |
3145 | // Otherwise we'll need to temporarily settle for some other convenient |
3146 | // type. Type legalization will make adjustments once the shiftee is split. |
3147 | else |
3148 | Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32); |
3149 | } |
3150 | |
3151 | bool nuw = false; |
3152 | bool nsw = false; |
3153 | bool exact = false; |
3154 | |
3155 | if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) { |
3156 | |
3157 | if (const OverflowingBinaryOperator *OFBinOp = |
3158 | dyn_cast<const OverflowingBinaryOperator>(&I)) { |
3159 | nuw = OFBinOp->hasNoUnsignedWrap(); |
3160 | nsw = OFBinOp->hasNoSignedWrap(); |
3161 | } |
3162 | if (const PossiblyExactOperator *ExactOp = |
3163 | dyn_cast<const PossiblyExactOperator>(&I)) |
3164 | exact = ExactOp->isExact(); |
3165 | } |
3166 | SDNodeFlags Flags; |
3167 | Flags.setExact(exact); |
3168 | Flags.setNoSignedWrap(nsw); |
3169 | Flags.setNoUnsignedWrap(nuw); |
3170 | SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, |
3171 | Flags); |
3172 | setValue(&I, Res); |
3173 | } |
3174 | |
3175 | void SelectionDAGBuilder::visitSDiv(const User &I) { |
3176 | SDValue Op1 = getValue(I.getOperand(0)); |
3177 | SDValue Op2 = getValue(I.getOperand(1)); |
3178 | |
3179 | SDNodeFlags Flags; |
3180 | Flags.setExact(isa<PossiblyExactOperator>(&I) && |
3181 | cast<PossiblyExactOperator>(&I)->isExact()); |
3182 | setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1, |
3183 | Op2, Flags)); |
3184 | } |
3185 | |
3186 | void SelectionDAGBuilder::visitICmp(const User &I) { |
3187 | ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; |
3188 | if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) |
3189 | predicate = IC->getPredicate(); |
3190 | else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) |
3191 | predicate = ICmpInst::Predicate(IC->getPredicate()); |
3192 | SDValue Op1 = getValue(I.getOperand(0)); |
3193 | SDValue Op2 = getValue(I.getOperand(1)); |
3194 | ISD::CondCode Opcode = getICmpCondCode(predicate); |
3195 | |
3196 | auto &TLI = DAG.getTargetLoweringInfo(); |
3197 | EVT MemVT = |
3198 | TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); |
3199 | |
3200 | // If a pointer's DAG type is larger than its memory type then the DAG values |
3201 | // are zero-extended. This breaks signed comparisons so truncate back to the |
3202 | // underlying type before doing the compare. |
3203 | if (Op1.getValueType() != MemVT) { |
3204 | Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT); |
3205 | Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT); |
3206 | } |
3207 | |
3208 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3209 | I.getType()); |
3210 | setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); |
3211 | } |
3212 | |
3213 | void SelectionDAGBuilder::visitFCmp(const User &I) { |
3214 | FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; |
3215 | if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) |
3216 | predicate = FC->getPredicate(); |
3217 | else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) |
3218 | predicate = FCmpInst::Predicate(FC->getPredicate()); |
3219 | SDValue Op1 = getValue(I.getOperand(0)); |
3220 | SDValue Op2 = getValue(I.getOperand(1)); |
3221 | |
3222 | ISD::CondCode Condition = getFCmpCondCode(predicate); |
3223 | auto *FPMO = cast<FPMathOperator>(&I); |
3224 | if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath) |
3225 | Condition = getFCmpCodeWithoutNaN(Condition); |
3226 | |
3227 | SDNodeFlags Flags; |
3228 | Flags.copyFMF(*FPMO); |
3229 | SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); |
3230 | |
3231 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3232 | I.getType()); |
3233 | setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); |
3234 | } |
3235 | |
3236 | // Check if the condition of the select has one use or two users that are both |
3237 | // selects with the same condition. |
3238 | static bool hasOnlySelectUsers(const Value *Cond) { |
3239 | return llvm::all_of(Cond->users(), [](const Value *V) { |
3240 | return isa<SelectInst>(V); |
3241 | }); |
3242 | } |
3243 | |
3244 | void SelectionDAGBuilder::visitSelect(const User &I) { |
3245 | SmallVector<EVT, 4> ValueVTs; |
3246 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), |
3247 | ValueVTs); |
3248 | unsigned NumValues = ValueVTs.size(); |
3249 | if (NumValues == 0) return; |
3250 | |
3251 | SmallVector<SDValue, 4> Values(NumValues); |
3252 | SDValue Cond = getValue(I.getOperand(0)); |
3253 | SDValue LHSVal = getValue(I.getOperand(1)); |
3254 | SDValue RHSVal = getValue(I.getOperand(2)); |
3255 | SmallVector<SDValue, 1> BaseOps(1, Cond); |
3256 | ISD::NodeType OpCode = |
3257 | Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; |
3258 | |
3259 | bool IsUnaryAbs = false; |
3260 | bool Negate = false; |
3261 | |
3262 | SDNodeFlags Flags; |
3263 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) |
3264 | Flags.copyFMF(*FPOp); |
3265 | |
3266 | // Min/max matching is only viable if all output VTs are the same. |
3267 | if (is_splat(ValueVTs)) { |
3268 | EVT VT = ValueVTs[0]; |
3269 | LLVMContext &Ctx = *DAG.getContext(); |
3270 | auto &TLI = DAG.getTargetLoweringInfo(); |
3271 | |
3272 | // We care about the legality of the operation after it has been type |
3273 | // legalized. |
3274 | while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal) |
3275 | VT = TLI.getTypeToTransformTo(Ctx, VT); |
3276 | |
3277 | // If the vselect is legal, assume we want to leave this as a vector setcc + |
3278 | // vselect. Otherwise, if this is going to be scalarized, we want to see if |
3279 | // min/max is legal on the scalar type. |
3280 | bool UseScalarMinMax = VT.isVector() && |
3281 | !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT); |
3282 | |
3283 | Value *LHS, *RHS; |
3284 | auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS); |
3285 | ISD::NodeType Opc = ISD::DELETED_NODE; |
3286 | switch (SPR.Flavor) { |
3287 | case SPF_UMAX: Opc = ISD::UMAX; break; |
3288 | case SPF_UMIN: Opc = ISD::UMIN; break; |
3289 | case SPF_SMAX: Opc = ISD::SMAX; break; |
3290 | case SPF_SMIN: Opc = ISD::SMIN; break; |
3291 | case SPF_FMINNUM: |
3292 | switch (SPR.NaNBehavior) { |
3293 | case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")__builtin_unreachable(); |
3294 | case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break; |
3295 | case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break; |
3296 | case SPNB_RETURNS_ANY: { |
3297 | if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT)) |
3298 | Opc = ISD::FMINNUM; |
3299 | else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT)) |
3300 | Opc = ISD::FMINIMUM; |
3301 | else if (UseScalarMinMax) |
3302 | Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ? |
3303 | ISD::FMINNUM : ISD::FMINIMUM; |
3304 | break; |
3305 | } |
3306 | } |
3307 | break; |
3308 | case SPF_FMAXNUM: |
3309 | switch (SPR.NaNBehavior) { |
3310 | case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")__builtin_unreachable(); |
3311 | case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break; |
3312 | case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break; |
3313 | case SPNB_RETURNS_ANY: |
3314 | |
3315 | if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT)) |
3316 | Opc = ISD::FMAXNUM; |
3317 | else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT)) |
3318 | Opc = ISD::FMAXIMUM; |
3319 | else if (UseScalarMinMax) |
3320 | Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ? |
3321 | ISD::FMAXNUM : ISD::FMAXIMUM; |
3322 | break; |
3323 | } |
3324 | break; |
3325 | case SPF_NABS: |
3326 | Negate = true; |
3327 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; |
3328 | case SPF_ABS: |
3329 | IsUnaryAbs = true; |
3330 | Opc = ISD::ABS; |
3331 | break; |
3332 | default: break; |
3333 | } |
3334 | |
3335 | if (!IsUnaryAbs && Opc != ISD::DELETED_NODE && |
3336 | (TLI.isOperationLegalOrCustom(Opc, VT) || |
3337 | (UseScalarMinMax && |
3338 | TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) && |
3339 | // If the underlying comparison instruction is used by any other |
3340 | // instruction, the consumed instructions won't be destroyed, so it is |
3341 | // not profitable to convert to a min/max. |
3342 | hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) { |
3343 | OpCode = Opc; |
3344 | LHSVal = getValue(LHS); |
3345 | RHSVal = getValue(RHS); |
3346 | BaseOps.clear(); |
3347 | } |
3348 | |
3349 | if (IsUnaryAbs) { |
3350 | OpCode = Opc; |
3351 | LHSVal = getValue(LHS); |
3352 | BaseOps.clear(); |
3353 | } |
3354 | } |
3355 | |
3356 | if (IsUnaryAbs) { |
3357 | for (unsigned i = 0; i != NumValues; ++i) { |
3358 | SDLoc dl = getCurSDLoc(); |
3359 | EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i); |
3360 | Values[i] = |
3361 | DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i)); |
3362 | if (Negate) |
3363 | Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), |
3364 | Values[i]); |
3365 | } |
3366 | } else { |
3367 | for (unsigned i = 0; i != NumValues; ++i) { |
3368 | SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end()); |
3369 | Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i)); |
3370 | Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i)); |
3371 | Values[i] = DAG.getNode( |
3372 | OpCode, getCurSDLoc(), |
3373 | LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags); |
3374 | } |
3375 | } |
3376 | |
3377 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), |
3378 | DAG.getVTList(ValueVTs), Values)); |
3379 | } |
3380 | |
3381 | void SelectionDAGBuilder::visitTrunc(const User &I) { |
3382 | // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). |
3383 | SDValue N = getValue(I.getOperand(0)); |
3384 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3385 | I.getType()); |
3386 | setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); |
3387 | } |
3388 | |
3389 | void SelectionDAGBuilder::visitZExt(const User &I) { |
3390 | // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). |
3391 | // ZExt also can't be a cast to bool for same reason. So, nothing much to do |
3392 | SDValue N = getValue(I.getOperand(0)); |
3393 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3394 | I.getType()); |
3395 | setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N)); |
3396 | } |
3397 | |
3398 | void SelectionDAGBuilder::visitSExt(const User &I) { |
3399 | // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). |
3400 | // SExt also can't be a cast to bool for same reason. So, nothing much to do |
3401 | SDValue N = getValue(I.getOperand(0)); |
3402 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3403 | I.getType()); |
3404 | setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); |
3405 | } |
3406 | |
3407 | void SelectionDAGBuilder::visitFPTrunc(const User &I) { |
3408 | // FPTrunc is never a no-op cast, no need to check |
3409 | SDValue N = getValue(I.getOperand(0)); |
3410 | SDLoc dl = getCurSDLoc(); |
3411 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3412 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
3413 | setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N, |
3414 | DAG.getTargetConstant( |
3415 | 0, dl, TLI.getPointerTy(DAG.getDataLayout())))); |
3416 | } |
3417 | |
3418 | void SelectionDAGBuilder::visitFPExt(const User &I) { |
3419 | // FPExt is never a no-op cast, no need to check |
3420 | SDValue N = getValue(I.getOperand(0)); |
3421 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3422 | I.getType()); |
3423 | setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); |
3424 | } |
3425 | |
3426 | void SelectionDAGBuilder::visitFPToUI(const User &I) { |
3427 | // FPToUI is never a no-op cast, no need to check |
3428 | SDValue N = getValue(I.getOperand(0)); |
3429 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3430 | I.getType()); |
3431 | setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); |
3432 | } |
3433 | |
3434 | void SelectionDAGBuilder::visitFPToSI(const User &I) { |
3435 | // FPToSI is never a no-op cast, no need to check |
3436 | SDValue N = getValue(I.getOperand(0)); |
3437 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3438 | I.getType()); |
3439 | setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); |
3440 | } |
3441 | |
3442 | void SelectionDAGBuilder::visitUIToFP(const User &I) { |
3443 | // UIToFP is never a no-op cast, no need to check |
3444 | SDValue N = getValue(I.getOperand(0)); |
3445 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3446 | I.getType()); |
3447 | setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N)); |
3448 | } |
3449 | |
3450 | void SelectionDAGBuilder::visitSIToFP(const User &I) { |
3451 | // SIToFP is never a no-op cast, no need to check |
3452 | SDValue N = getValue(I.getOperand(0)); |
3453 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3454 | I.getType()); |
3455 | setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); |
3456 | } |
3457 | |
3458 | void SelectionDAGBuilder::visitPtrToInt(const User &I) { |
3459 | // What to do depends on the size of the integer and the size of the pointer. |
3460 | // We can either truncate, zero extend, or no-op, accordingly. |
3461 | SDValue N = getValue(I.getOperand(0)); |
3462 | auto &TLI = DAG.getTargetLoweringInfo(); |
3463 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3464 | I.getType()); |
3465 | EVT PtrMemVT = |
3466 | TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); |
3467 | N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT); |
3468 | N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT); |
3469 | setValue(&I, N); |
3470 | } |
3471 | |
3472 | void SelectionDAGBuilder::visitIntToPtr(const User &I) { |
3473 | // What to do depends on the size of the integer and the size of the pointer. |
3474 | // We can either truncate, zero extend, or no-op, accordingly. |
3475 | SDValue N = getValue(I.getOperand(0)); |
3476 | auto &TLI = DAG.getTargetLoweringInfo(); |
3477 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
3478 | EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); |
3479 | N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT); |
3480 | N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT); |
3481 | setValue(&I, N); |
3482 | } |
3483 | |
3484 | void SelectionDAGBuilder::visitBitCast(const User &I) { |
3485 | SDValue N = getValue(I.getOperand(0)); |
3486 | SDLoc dl = getCurSDLoc(); |
3487 | EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), |
3488 | I.getType()); |
3489 | |
3490 | // BitCast assures us that source and destination are the same size so this is |
3491 | // either a BITCAST or a no-op. |
3492 | if (DestVT != N.getValueType()) |
3493 | setValue(&I, DAG.getNode(ISD::BITCAST, dl, |
3494 | DestVT, N)); // convert types. |
3495 | // Check if the original LLVM IR Operand was a ConstantInt, because getValue() |
3496 | // might fold any kind of constant expression to an integer constant and that |
3497 | // is not what we are looking for. Only recognize a bitcast of a genuine |
3498 | // constant integer as an opaque constant. |
3499 | else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0))) |
3500 | setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false, |
3501 | /*isOpaque*/true)); |
3502 | else |
3503 | setValue(&I, N); // noop cast. |
3504 | } |
3505 | |
3506 | void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { |
3507 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3508 | const Value *SV = I.getOperand(0); |
3509 | SDValue N = getValue(SV); |
3510 | EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
3511 | |
3512 | unsigned SrcAS = SV->getType()->getPointerAddressSpace(); |
3513 | unsigned DestAS = I.getType()->getPointerAddressSpace(); |
3514 | |
3515 | if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS)) |
3516 | N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); |
3517 | |
3518 | setValue(&I, N); |
3519 | } |
3520 | |
3521 | void SelectionDAGBuilder::visitInsertElement(const User &I) { |
3522 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3523 | SDValue InVec = getValue(I.getOperand(0)); |
3524 | SDValue InVal = getValue(I.getOperand(1)); |
3525 | SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(), |
3526 | TLI.getVectorIdxTy(DAG.getDataLayout())); |
3527 | setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), |
3528 | TLI.getValueType(DAG.getDataLayout(), I.getType()), |
3529 | InVec, InVal, InIdx)); |
3530 | } |
3531 | |
3532 | void SelectionDAGBuilder::visitExtractElement(const User &I) { |
3533 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3534 | SDValue InVec = getValue(I.getOperand(0)); |
3535 | SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(), |
3536 | TLI.getVectorIdxTy(DAG.getDataLayout())); |
3537 | setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), |
3538 | TLI.getValueType(DAG.getDataLayout(), I.getType()), |
3539 | InVec, InIdx)); |
3540 | } |
3541 | |
3542 | void SelectionDAGBuilder::visitShuffleVector(const User &I) { |
3543 | SDValue Src1 = getValue(I.getOperand(0)); |
3544 | SDValue Src2 = getValue(I.getOperand(1)); |
3545 | ArrayRef<int> Mask; |
3546 | if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) |
3547 | Mask = SVI->getShuffleMask(); |
3548 | else |
3549 | Mask = cast<ConstantExpr>(I).getShuffleMask(); |
3550 | SDLoc DL = getCurSDLoc(); |
3551 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3552 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
3553 | EVT SrcVT = Src1.getValueType(); |
3554 | |
3555 | if (all_of(Mask, [](int Elem) { return Elem == 0; }) && |
3556 | VT.isScalableVector()) { |
3557 | // Canonical splat form of first element of first input vector. |
3558 | SDValue FirstElt = |
3559 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1, |
3560 | DAG.getVectorIdxConstant(0, DL)); |
3561 | setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt)); |
3562 | return; |
3563 | } |
3564 | |
3565 | // For now, we only handle splats for scalable vectors. |
3566 | // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation |
3567 | // for targets that support a SPLAT_VECTOR for non-scalable vector types. |
3568 | assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")((void)0); |
3569 | |
3570 | unsigned SrcNumElts = SrcVT.getVectorNumElements(); |
3571 | unsigned MaskNumElts = Mask.size(); |
3572 | |
3573 | if (SrcNumElts == MaskNumElts) { |
3574 | setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask)); |
3575 | return; |
3576 | } |
3577 | |
3578 | // Normalize the shuffle vector since mask and vector length don't match. |
3579 | if (SrcNumElts < MaskNumElts) { |
3580 | // Mask is longer than the source vectors. We can use concatenate vector to |
3581 | // make the mask and vectors lengths match. |
3582 | |
3583 | if (MaskNumElts % SrcNumElts == 0) { |
3584 | // Mask length is a multiple of the source vector length. |
3585 | // Check if the shuffle is some kind of concatenation of the input |
3586 | // vectors. |
3587 | unsigned NumConcat = MaskNumElts / SrcNumElts; |
3588 | bool IsConcat = true; |
3589 | SmallVector<int, 8> ConcatSrcs(NumConcat, -1); |
3590 | for (unsigned i = 0; i != MaskNumElts; ++i) { |
3591 | int Idx = Mask[i]; |
3592 | if (Idx < 0) |
3593 | continue; |
3594 | // Ensure the indices in each SrcVT sized piece are sequential and that |
3595 | // the same source is used for the whole piece. |
3596 | if ((Idx % SrcNumElts != (i % SrcNumElts)) || |
3597 | (ConcatSrcs[i / SrcNumElts] >= 0 && |
3598 | ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) { |
3599 | IsConcat = false; |
3600 | break; |
3601 | } |
3602 | // Remember which source this index came from. |
3603 | ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts; |
3604 | } |
3605 | |
3606 | // The shuffle is concatenating multiple vectors together. Just emit |
3607 | // a CONCAT_VECTORS operation. |
3608 | if (IsConcat) { |
3609 | SmallVector<SDValue, 8> ConcatOps; |
3610 | for (auto Src : ConcatSrcs) { |
3611 | if (Src < 0) |
3612 | ConcatOps.push_back(DAG.getUNDEF(SrcVT)); |
3613 | else if (Src == 0) |
3614 | ConcatOps.push_back(Src1); |
3615 | else |
3616 | ConcatOps.push_back(Src2); |
3617 | } |
3618 | setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps)); |
3619 | return; |
3620 | } |
3621 | } |
3622 | |
3623 | unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts); |
3624 | unsigned NumConcat = PaddedMaskNumElts / SrcNumElts; |
3625 | EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), |
3626 | PaddedMaskNumElts); |
3627 | |
3628 | // Pad both vectors with undefs to make them the same length as the mask. |
3629 | SDValue UndefVal = DAG.getUNDEF(SrcVT); |
3630 | |
3631 | SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal); |
3632 | SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); |
3633 | MOps1[0] = Src1; |
3634 | MOps2[0] = Src2; |
3635 | |
3636 | Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1); |
3637 | Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2); |
3638 | |
3639 | // Readjust mask for new input vector length. |
3640 | SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1); |
3641 | for (unsigned i = 0; i != MaskNumElts; ++i) { |
3642 | int Idx = Mask[i]; |
3643 | if (Idx >= (int)SrcNumElts) |
3644 | Idx -= SrcNumElts - PaddedMaskNumElts; |
3645 | MappedOps[i] = Idx; |
3646 | } |
3647 | |
3648 | SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps); |
3649 | |
3650 | // If the concatenated vector was padded, extract a subvector with the |
3651 | // correct number of elements. |
3652 | if (MaskNumElts != PaddedMaskNumElts) |
3653 | Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result, |
3654 | DAG.getVectorIdxConstant(0, DL)); |
3655 | |
3656 | setValue(&I, Result); |
3657 | return; |
3658 | } |
3659 | |
3660 | if (SrcNumElts > MaskNumElts) { |
3661 | // Analyze the access pattern of the vector to see if we can extract |
3662 | // two subvectors and do the shuffle. |
3663 | int StartIdx[2] = { -1, -1 }; // StartIdx to extract from |
3664 | bool CanExtract = true; |
3665 | for (int Idx : Mask) { |
3666 | unsigned Input = 0; |
3667 | if (Idx < 0) |
3668 | continue; |
3669 | |
3670 | if (Idx >= (int)SrcNumElts) { |
3671 | Input = 1; |
3672 | Idx -= SrcNumElts; |
3673 | } |
3674 | |
3675 | // If all the indices come from the same MaskNumElts sized portion of |
3676 | // the sources we can use extract. Also make sure the extract wouldn't |
3677 | // extract past the end of the source. |
3678 | int NewStartIdx = alignDown(Idx, MaskNumElts); |
3679 | if (NewStartIdx + MaskNumElts > SrcNumElts || |
3680 | (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx)) |
3681 | CanExtract = false; |
3682 | // Make sure we always update StartIdx as we use it to track if all |
3683 | // elements are undef. |
3684 | StartIdx[Input] = NewStartIdx; |
3685 | } |
3686 | |
3687 | if (StartIdx[0] < 0 && StartIdx[1] < 0) { |
3688 | setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. |
3689 | return; |
3690 | } |
3691 | if (CanExtract) { |
3692 | // Extract appropriate subvector and generate a vector shuffle |
3693 | for (unsigned Input = 0; Input < 2; ++Input) { |
3694 | SDValue &Src = Input == 0 ? Src1 : Src2; |
3695 | if (StartIdx[Input] < 0) |
3696 | Src = DAG.getUNDEF(VT); |
3697 | else { |
3698 | Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src, |
3699 | DAG.getVectorIdxConstant(StartIdx[Input], DL)); |
3700 | } |
3701 | } |
3702 | |
3703 | // Calculate new mask. |
3704 | SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end()); |
3705 | for (int &Idx : MappedOps) { |
3706 | if (Idx >= (int)SrcNumElts) |
3707 | Idx -= SrcNumElts + StartIdx[1] - MaskNumElts; |
3708 | else if (Idx >= 0) |
3709 | Idx -= StartIdx[0]; |
3710 | } |
3711 | |
3712 | setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps)); |
3713 | return; |
3714 | } |
3715 | } |
3716 | |
3717 | // We can't use either concat vectors or extract subvectors so fall back to |
3718 | // replacing the shuffle with extract and build vector. |
3719 | // to insert and build vector. |
3720 | EVT EltVT = VT.getVectorElementType(); |
3721 | SmallVector<SDValue,8> Ops; |
3722 | for (int Idx : Mask) { |
3723 | SDValue Res; |
3724 | |
3725 | if (Idx < 0) { |
3726 | Res = DAG.getUNDEF(EltVT); |
3727 | } else { |
3728 | SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; |
3729 | if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; |
3730 | |
3731 | Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src, |
3732 | DAG.getVectorIdxConstant(Idx, DL)); |
3733 | } |
3734 | |
3735 | Ops.push_back(Res); |
3736 | } |
3737 | |
3738 | setValue(&I, DAG.getBuildVector(VT, DL, Ops)); |
3739 | } |
3740 | |
3741 | void SelectionDAGBuilder::visitInsertValue(const User &I) { |
3742 | ArrayRef<unsigned> Indices; |
3743 | if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I)) |
3744 | Indices = IV->getIndices(); |
3745 | else |
3746 | Indices = cast<ConstantExpr>(&I)->getIndices(); |
3747 | |
3748 | const Value *Op0 = I.getOperand(0); |
3749 | const Value *Op1 = I.getOperand(1); |
3750 | Type *AggTy = I.getType(); |
3751 | Type *ValTy = Op1->getType(); |
3752 | bool IntoUndef = isa<UndefValue>(Op0); |
3753 | bool FromUndef = isa<UndefValue>(Op1); |
3754 | |
3755 | unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); |
3756 | |
3757 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3758 | SmallVector<EVT, 4> AggValueVTs; |
3759 | ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs); |
3760 | SmallVector<EVT, 4> ValValueVTs; |
3761 | ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); |
3762 | |
3763 | unsigned NumAggValues = AggValueVTs.size(); |
3764 | unsigned NumValValues = ValValueVTs.size(); |
3765 | SmallVector<SDValue, 4> Values(NumAggValues); |
3766 | |
3767 | // Ignore an insertvalue that produces an empty object |
3768 | if (!NumAggValues) { |
3769 | setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); |
3770 | return; |
3771 | } |
3772 | |
3773 | SDValue Agg = getValue(Op0); |
3774 | unsigned i = 0; |
3775 | // Copy the beginning value(s) from the original aggregate. |
3776 | for (; i != LinearIndex; ++i) |
3777 | Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : |
3778 | SDValue(Agg.getNode(), Agg.getResNo() + i); |
3779 | // Copy values from the inserted value(s). |
3780 | if (NumValValues) { |
3781 | SDValue Val = getValue(Op1); |
3782 | for (; i != LinearIndex + NumValValues; ++i) |
3783 | Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : |
3784 | SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); |
3785 | } |
3786 | // Copy remaining value(s) from the original aggregate. |
3787 | for (; i != NumAggValues; ++i) |
3788 | Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : |
3789 | SDValue(Agg.getNode(), Agg.getResNo() + i); |
3790 | |
3791 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), |
3792 | DAG.getVTList(AggValueVTs), Values)); |
3793 | } |
3794 | |
3795 | void SelectionDAGBuilder::visitExtractValue(const User &I) { |
3796 | ArrayRef<unsigned> Indices; |
3797 | if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I)) |
3798 | Indices = EV->getIndices(); |
3799 | else |
3800 | Indices = cast<ConstantExpr>(&I)->getIndices(); |
3801 | |
3802 | const Value *Op0 = I.getOperand(0); |
3803 | Type *AggTy = Op0->getType(); |
3804 | Type *ValTy = I.getType(); |
3805 | bool OutOfUndef = isa<UndefValue>(Op0); |
3806 | |
3807 | unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); |
3808 | |
3809 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3810 | SmallVector<EVT, 4> ValValueVTs; |
3811 | ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); |
3812 | |
3813 | unsigned NumValValues = ValValueVTs.size(); |
3814 | |
3815 | // Ignore a extractvalue that produces an empty object |
3816 | if (!NumValValues) { |
3817 | setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); |
3818 | return; |
3819 | } |
3820 | |
3821 | SmallVector<SDValue, 4> Values(NumValValues); |
3822 | |
3823 | SDValue Agg = getValue(Op0); |
3824 | // Copy out the selected value(s). |
3825 | for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) |
3826 | Values[i - LinearIndex] = |
3827 | OutOfUndef ? |
3828 | DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : |
3829 | SDValue(Agg.getNode(), Agg.getResNo() + i); |
3830 | |
3831 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), |
3832 | DAG.getVTList(ValValueVTs), Values)); |
3833 | } |
3834 | |
3835 | void SelectionDAGBuilder::visitGetElementPtr(const User &I) { |
3836 | Value *Op0 = I.getOperand(0); |
3837 | // Note that the pointer operand may be a vector of pointers. Take the scalar |
3838 | // element which holds a pointer. |
3839 | unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace(); |
3840 | SDValue N = getValue(Op0); |
3841 | SDLoc dl = getCurSDLoc(); |
3842 | auto &TLI = DAG.getTargetLoweringInfo(); |
3843 | |
3844 | // Normalize Vector GEP - all scalar operands should be converted to the |
3845 | // splat vector. |
3846 | bool IsVectorGEP = I.getType()->isVectorTy(); |
3847 | ElementCount VectorElementCount = |
3848 | IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount() |
3849 | : ElementCount::getFixed(0); |
3850 | |
3851 | if (IsVectorGEP && !N.getValueType().isVector()) { |
3852 | LLVMContext &Context = *DAG.getContext(); |
3853 | EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount); |
3854 | if (VectorElementCount.isScalable()) |
3855 | N = DAG.getSplatVector(VT, dl, N); |
3856 | else |
3857 | N = DAG.getSplatBuildVector(VT, dl, N); |
3858 | } |
3859 | |
3860 | for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I); |
3861 | GTI != E; ++GTI) { |
3862 | const Value *Idx = GTI.getOperand(); |
3863 | if (StructType *StTy = GTI.getStructTypeOrNull()) { |
3864 | unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); |
3865 | if (Field) { |
3866 | // N = N + Offset |
3867 | uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field); |
3868 | |
3869 | // In an inbounds GEP with an offset that is nonnegative even when |
3870 | // interpreted as signed, assume there is no unsigned overflow. |
3871 | SDNodeFlags Flags; |
3872 | if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds()) |
3873 | Flags.setNoUnsignedWrap(true); |
3874 | |
3875 | N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, |
3876 | DAG.getConstant(Offset, dl, N.getValueType()), Flags); |
3877 | } |
3878 | } else { |
3879 | // IdxSize is the width of the arithmetic according to IR semantics. |
3880 | // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth |
3881 | // (and fix up the result later). |
3882 | unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS); |
3883 | MVT IdxTy = MVT::getIntegerVT(IdxSize); |
3884 | TypeSize ElementSize = DL->getTypeAllocSize(GTI.getIndexedType()); |
3885 | // We intentionally mask away the high bits here; ElementSize may not |
3886 | // fit in IdxTy. |
3887 | APInt ElementMul(IdxSize, ElementSize.getKnownMinSize()); |
3888 | bool ElementScalable = ElementSize.isScalable(); |
3889 | |
3890 | // If this is a scalar constant or a splat vector of constants, |
3891 | // handle it quickly. |
3892 | const auto *C = dyn_cast<Constant>(Idx); |
3893 | if (C && isa<VectorType>(C->getType())) |
3894 | C = C->getSplatValue(); |
3895 | |
3896 | const auto *CI = dyn_cast_or_null<ConstantInt>(C); |
3897 | if (CI && CI->isZero()) |
3898 | continue; |
3899 | if (CI && !ElementScalable) { |
3900 | APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize); |
3901 | LLVMContext &Context = *DAG.getContext(); |
3902 | SDValue OffsVal; |
3903 | if (IsVectorGEP) |
3904 | OffsVal = DAG.getConstant( |
3905 | Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount)); |
3906 | else |
3907 | OffsVal = DAG.getConstant(Offs, dl, IdxTy); |
3908 | |
3909 | // In an inbounds GEP with an offset that is nonnegative even when |
3910 | // interpreted as signed, assume there is no unsigned overflow. |
3911 | SDNodeFlags Flags; |
3912 | if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds()) |
3913 | Flags.setNoUnsignedWrap(true); |
3914 | |
3915 | OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType()); |
3916 | |
3917 | N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags); |
3918 | continue; |
3919 | } |
3920 | |
3921 | // N = N + Idx * ElementMul; |
3922 | SDValue IdxN = getValue(Idx); |
3923 | |
3924 | if (!IdxN.getValueType().isVector() && IsVectorGEP) { |
3925 | EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), |
3926 | VectorElementCount); |
3927 | if (VectorElementCount.isScalable()) |
3928 | IdxN = DAG.getSplatVector(VT, dl, IdxN); |
3929 | else |
3930 | IdxN = DAG.getSplatBuildVector(VT, dl, IdxN); |
3931 | } |
3932 | |
3933 | // If the index is smaller or larger than intptr_t, truncate or extend |
3934 | // it. |
3935 | IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType()); |
3936 | |
3937 | if (ElementScalable) { |
3938 | EVT VScaleTy = N.getValueType().getScalarType(); |
3939 | SDValue VScale = DAG.getNode( |
3940 | ISD::VSCALE, dl, VScaleTy, |
3941 | DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy)); |
3942 | if (IsVectorGEP) |
3943 | VScale = DAG.getSplatVector(N.getValueType(), dl, VScale); |
3944 | IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale); |
3945 | } else { |
3946 | // If this is a multiply by a power of two, turn it into a shl |
3947 | // immediately. This is a very common case. |
3948 | if (ElementMul != 1) { |
3949 | if (ElementMul.isPowerOf2()) { |
3950 | unsigned Amt = ElementMul.logBase2(); |
3951 | IdxN = DAG.getNode(ISD::SHL, dl, |
3952 | N.getValueType(), IdxN, |
3953 | DAG.getConstant(Amt, dl, IdxN.getValueType())); |
3954 | } else { |
3955 | SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl, |
3956 | IdxN.getValueType()); |
3957 | IdxN = DAG.getNode(ISD::MUL, dl, |
3958 | N.getValueType(), IdxN, Scale); |
3959 | } |
3960 | } |
3961 | } |
3962 | |
3963 | N = DAG.getNode(ISD::ADD, dl, |
3964 | N.getValueType(), N, IdxN); |
3965 | } |
3966 | } |
3967 | |
3968 | MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS); |
3969 | MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS); |
3970 | if (IsVectorGEP) { |
3971 | PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount); |
3972 | PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount); |
3973 | } |
3974 | |
3975 | if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds()) |
3976 | N = DAG.getPtrExtendInReg(N, dl, PtrMemTy); |
3977 | |
3978 | setValue(&I, N); |
3979 | } |
3980 | |
3981 | void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { |
3982 | // If this is a fixed sized alloca in the entry block of the function, |
3983 | // allocate it statically on the stack. |
3984 | if (FuncInfo.StaticAllocaMap.count(&I)) |
3985 | return; // getValue will auto-populate this. |
3986 | |
3987 | SDLoc dl = getCurSDLoc(); |
3988 | Type *Ty = I.getAllocatedType(); |
3989 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
3990 | auto &DL = DAG.getDataLayout(); |
3991 | uint64_t TySize = DL.getTypeAllocSize(Ty); |
3992 | MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign()); |
3993 | |
3994 | SDValue AllocSize = getValue(I.getArraySize()); |
3995 | |
3996 | EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace()); |
3997 | if (AllocSize.getValueType() != IntPtr) |
3998 | AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr); |
3999 | |
4000 | AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, |
4001 | AllocSize, |
4002 | DAG.getConstant(TySize, dl, IntPtr)); |
4003 | |
4004 | // Handle alignment. If the requested alignment is less than or equal to |
4005 | // the stack alignment, ignore it. If the size is greater than or equal to |
4006 | // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. |
4007 | Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign(); |
4008 | if (*Alignment <= StackAlign) |
4009 | Alignment = None; |
4010 | |
4011 | const uint64_t StackAlignMask = StackAlign.value() - 1U; |
4012 | // Round the size of the allocation up to the stack alignment size |
4013 | // by add SA-1 to the size. This doesn't overflow because we're computing |
4014 | // an address inside an alloca. |
4015 | SDNodeFlags Flags; |
4016 | Flags.setNoUnsignedWrap(true); |
4017 | AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize, |
4018 | DAG.getConstant(StackAlignMask, dl, IntPtr), Flags); |
4019 | |
4020 | // Mask out the low bits for alignment purposes. |
4021 | AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize, |
4022 | DAG.getConstant(~StackAlignMask, dl, IntPtr)); |
4023 | |
4024 | SDValue Ops[] = { |
4025 | getRoot(), AllocSize, |
4026 | DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)}; |
4027 | SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); |
4028 | SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops); |
4029 | setValue(&I, DSA); |
4030 | DAG.setRoot(DSA.getValue(1)); |
4031 | |
4032 | assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())((void)0); |
4033 | } |
4034 | |
4035 | void SelectionDAGBuilder::visitLoad(const LoadInst &I) { |
4036 | if (I.isAtomic()) |
4037 | return visitAtomicLoad(I); |
4038 | |
4039 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4040 | const Value *SV = I.getOperand(0); |
4041 | if (TLI.supportSwiftError()) { |
4042 | // Swifterror values can come from either a function parameter with |
4043 | // swifterror attribute or an alloca with swifterror attribute. |
4044 | if (const Argument *Arg = dyn_cast<Argument>(SV)) { |
4045 | if (Arg->hasSwiftErrorAttr()) |
4046 | return visitLoadFromSwiftError(I); |
4047 | } |
4048 | |
4049 | if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) { |
4050 | if (Alloca->isSwiftError()) |
4051 | return visitLoadFromSwiftError(I); |
4052 | } |
4053 | } |
4054 | |
4055 | SDValue Ptr = getValue(SV); |
4056 | |
4057 | Type *Ty = I.getType(); |
4058 | Align Alignment = I.getAlign(); |
4059 | |
4060 | AAMDNodes AAInfo; |
4061 | I.getAAMetadata(AAInfo); |
4062 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); |
4063 | |
4064 | SmallVector<EVT, 4> ValueVTs, MemVTs; |
4065 | SmallVector<uint64_t, 4> Offsets; |
4066 | ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets); |
4067 | unsigned NumValues = ValueVTs.size(); |
4068 | if (NumValues == 0) |
4069 | return; |
4070 | |
4071 | bool isVolatile = I.isVolatile(); |
4072 | |
4073 | SDValue Root; |
4074 | bool ConstantMemory = false; |
4075 | if (isVolatile) |
4076 | // Serialize volatile loads with other side effects. |
4077 | Root = getRoot(); |
4078 | else if (NumValues > MaxParallelChains) |
4079 | Root = getMemoryRoot(); |
4080 | else if (AA && |
4081 | AA->pointsToConstantMemory(MemoryLocation( |
4082 | SV, |
4083 | LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), |
4084 | AAInfo))) { |
4085 | // Do not serialize (non-volatile) loads of constant memory with anything. |
4086 | Root = DAG.getEntryNode(); |
4087 | ConstantMemory = true; |
4088 | } else { |
4089 | // Do not serialize non-volatile loads against each other. |
4090 | Root = DAG.getRoot(); |
4091 | } |
4092 | |
4093 | SDLoc dl = getCurSDLoc(); |
4094 | |
4095 | if (isVolatile) |
4096 | Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG); |
4097 | |
4098 | // An aggregate load cannot wrap around the address space, so offsets to its |
4099 | // parts don't wrap either. |
4100 | SDNodeFlags Flags; |
4101 | Flags.setNoUnsignedWrap(true); |
4102 | |
4103 | SmallVector<SDValue, 4> Values(NumValues); |
4104 | SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); |
4105 | EVT PtrVT = Ptr.getValueType(); |
4106 | |
4107 | MachineMemOperand::Flags MMOFlags |
4108 | = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout()); |
4109 | |
4110 | unsigned ChainI = 0; |
4111 | for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { |
4112 | // Serializing loads here may result in excessive register pressure, and |
4113 | // TokenFactor places arbitrary choke points on the scheduler. SD scheduling |
4114 | // could recover a bit by hoisting nodes upward in the chain by recognizing |
4115 | // they are side-effect free or do not alias. The optimizer should really |
4116 | // avoid this case by converting large object/array copies to llvm.memcpy |
4117 | // (MaxParallelChains should always remain as failsafe). |
4118 | if (ChainI == MaxParallelChains) { |
4119 | assert(PendingLoads.empty() && "PendingLoads must be serialized first")((void)0); |
4120 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
4121 | makeArrayRef(Chains.data(), ChainI)); |
4122 | Root = Chain; |
4123 | ChainI = 0; |
4124 | } |
4125 | SDValue A = DAG.getNode(ISD::ADD, dl, |
4126 | PtrVT, Ptr, |
4127 | DAG.getConstant(Offsets[i], dl, PtrVT), |
4128 | Flags); |
4129 | |
4130 | SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, |
4131 | MachinePointerInfo(SV, Offsets[i]), Alignment, |
4132 | MMOFlags, AAInfo, Ranges); |
4133 | Chains[ChainI] = L.getValue(1); |
4134 | |
4135 | if (MemVTs[i] != ValueVTs[i]) |
4136 | L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]); |
4137 | |
4138 | Values[i] = L; |
4139 | } |
4140 | |
4141 | if (!ConstantMemory) { |
4142 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
4143 | makeArrayRef(Chains.data(), ChainI)); |
4144 | if (isVolatile) |
4145 | DAG.setRoot(Chain); |
4146 | else |
4147 | PendingLoads.push_back(Chain); |
4148 | } |
4149 | |
4150 | setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl, |
4151 | DAG.getVTList(ValueVTs), Values)); |
4152 | } |
4153 | |
4154 | void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) { |
4155 | assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((void)0) |
4156 | "call visitStoreToSwiftError when backend supports swifterror")((void)0); |
4157 | |
4158 | SmallVector<EVT, 4> ValueVTs; |
4159 | SmallVector<uint64_t, 4> Offsets; |
4160 | const Value *SrcV = I.getOperand(0); |
4161 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), |
4162 | SrcV->getType(), ValueVTs, &Offsets); |
4163 | assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((void)0) |
4164 | "expect a single EVT for swifterror")((void)0); |
4165 | |
4166 | SDValue Src = getValue(SrcV); |
4167 | // Create a virtual register, then update the virtual register. |
4168 | Register VReg = |
4169 | SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand()); |
4170 | // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue |
4171 | // Chain can be getRoot or getControlRoot. |
4172 | SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg, |
4173 | SDValue(Src.getNode(), Src.getResNo())); |
4174 | DAG.setRoot(CopyNode); |
4175 | } |
4176 | |
4177 | void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { |
4178 | assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((void)0) |
4179 | "call visitLoadFromSwiftError when backend supports swifterror")((void)0); |
4180 | |
4181 | assert(!I.isVolatile() &&((void)0) |
4182 | !I.hasMetadata(LLVMContext::MD_nontemporal) &&((void)0) |
4183 | !I.hasMetadata(LLVMContext::MD_invariant_load) &&((void)0) |
4184 | "Support volatile, non temporal, invariant for load_from_swift_error")((void)0); |
4185 | |
4186 | const Value *SV = I.getOperand(0); |
4187 | Type *Ty = I.getType(); |
4188 | AAMDNodes AAInfo; |
4189 | I.getAAMetadata(AAInfo); |
4190 | assert(((void)0) |
4191 | (!AA ||((void)0) |
4192 | !AA->pointsToConstantMemory(MemoryLocation(((void)0) |
4193 | SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),((void)0) |
4194 | AAInfo))) &&((void)0) |
4195 | "load_from_swift_error should not be constant memory")((void)0); |
4196 | |
4197 | SmallVector<EVT, 4> ValueVTs; |
4198 | SmallVector<uint64_t, 4> Offsets; |
4199 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty, |
4200 | ValueVTs, &Offsets); |
4201 | assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((void)0) |
4202 | "expect a single EVT for swifterror")((void)0); |
4203 | |
4204 | // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT |
4205 | SDValue L = DAG.getCopyFromReg( |
4206 | getRoot(), getCurSDLoc(), |
4207 | SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]); |
4208 | |
4209 | setValue(&I, L); |
4210 | } |
4211 | |
4212 | void SelectionDAGBuilder::visitStore(const StoreInst &I) { |
4213 | if (I.isAtomic()) |
4214 | return visitAtomicStore(I); |
4215 | |
4216 | const Value *SrcV = I.getOperand(0); |
4217 | const Value *PtrV = I.getOperand(1); |
4218 | |
4219 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4220 | if (TLI.supportSwiftError()) { |
4221 | // Swifterror values can come from either a function parameter with |
4222 | // swifterror attribute or an alloca with swifterror attribute. |
4223 | if (const Argument *Arg = dyn_cast<Argument>(PtrV)) { |
4224 | if (Arg->hasSwiftErrorAttr()) |
4225 | return visitStoreToSwiftError(I); |
4226 | } |
4227 | |
4228 | if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) { |
4229 | if (Alloca->isSwiftError()) |
4230 | return visitStoreToSwiftError(I); |
4231 | } |
4232 | } |
4233 | |
4234 | SmallVector<EVT, 4> ValueVTs, MemVTs; |
4235 | SmallVector<uint64_t, 4> Offsets; |
4236 | ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), |
4237 | SrcV->getType(), ValueVTs, &MemVTs, &Offsets); |
4238 | unsigned NumValues = ValueVTs.size(); |
4239 | if (NumValues == 0) |
4240 | return; |
4241 | |
4242 | // Get the lowered operands. Note that we do this after |
4243 | // checking if NumResults is zero, because with zero results |
4244 | // the operands won't have values in the map. |
4245 | SDValue Src = getValue(SrcV); |
4246 | SDValue Ptr = getValue(PtrV); |
4247 | |
4248 | SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot(); |
4249 | SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); |
4250 | SDLoc dl = getCurSDLoc(); |
4251 | Align Alignment = I.getAlign(); |
4252 | AAMDNodes AAInfo; |
4253 | I.getAAMetadata(AAInfo); |
4254 | |
4255 | auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); |
4256 | |
4257 | // An aggregate load cannot wrap around the address space, so offsets to its |
4258 | // parts don't wrap either. |
4259 | SDNodeFlags Flags; |
4260 | Flags.setNoUnsignedWrap(true); |
4261 | |
4262 | unsigned ChainI = 0; |
4263 | for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { |
4264 | // See visitLoad comments. |
4265 | if (ChainI == MaxParallelChains) { |
4266 | SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
4267 | makeArrayRef(Chains.data(), ChainI)); |
4268 | Root = Chain; |
4269 | ChainI = 0; |
4270 | } |
4271 | SDValue Add = |
4272 | DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); |
4273 | SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); |
4274 | if (MemVTs[i] != ValueVTs[i]) |
4275 | Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); |
4276 | SDValue St = |
4277 | DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]), |
4278 | Alignment, MMOFlags, AAInfo); |
4279 | Chains[ChainI] = St; |
4280 | } |
4281 | |
4282 | SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
4283 | makeArrayRef(Chains.data(), ChainI)); |
4284 | DAG.setRoot(StoreNode); |
4285 | } |
4286 | |
4287 | void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, |
4288 | bool IsCompressing) { |
4289 | SDLoc sdl = getCurSDLoc(); |
4290 | |
4291 | auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, |
4292 | MaybeAlign &Alignment) { |
4293 | // llvm.masked.store.*(Src0, Ptr, alignment, Mask) |
4294 | Src0 = I.getArgOperand(0); |
4295 | Ptr = I.getArgOperand(1); |
4296 | Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue(); |
4297 | Mask = I.getArgOperand(3); |
4298 | }; |
4299 | auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, |
4300 | MaybeAlign &Alignment) { |
4301 | // llvm.masked.compressstore.*(Src0, Ptr, Mask) |
4302 | Src0 = I.getArgOperand(0); |
4303 | Ptr = I.getArgOperand(1); |
4304 | Mask = I.getArgOperand(2); |
4305 | Alignment = None; |
4306 | }; |
4307 | |
4308 | Value *PtrOperand, *MaskOperand, *Src0Operand; |
4309 | MaybeAlign Alignment; |
4310 | if (IsCompressing) |
4311 | getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); |
4312 | else |
4313 | getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); |
4314 | |
4315 | SDValue Ptr = getValue(PtrOperand); |
4316 | SDValue Src0 = getValue(Src0Operand); |
4317 | SDValue Mask = getValue(MaskOperand); |
4318 | SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); |
4319 | |
4320 | EVT VT = Src0.getValueType(); |
4321 | if (!Alignment) |
4322 | Alignment = DAG.getEVTAlign(VT); |
4323 | |
4324 | AAMDNodes AAInfo; |
4325 | I.getAAMetadata(AAInfo); |
4326 | |
4327 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
4328 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, |
4329 | // TODO: Make MachineMemOperands aware of scalable |
4330 | // vectors. |
4331 | VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo); |
4332 | SDValue StoreNode = |
4333 | DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO, |
4334 | ISD::UNINDEXED, false /* Truncating */, IsCompressing); |
4335 | DAG.setRoot(StoreNode); |
4336 | setValue(&I, StoreNode); |
4337 | } |
4338 | |
4339 | // Get a uniform base for the Gather/Scatter intrinsic. |
4340 | // The first argument of the Gather/Scatter intrinsic is a vector of pointers. |
4341 | // We try to represent it as a base pointer + vector of indices. |
4342 | // Usually, the vector of pointers comes from a 'getelementptr' instruction. |
4343 | // The first operand of the GEP may be a single pointer or a vector of pointers |
4344 | // Example: |
4345 | // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind |
4346 | // or |
4347 | // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind |
4348 | // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, .. |
4349 | // |
4350 | // When the first GEP operand is a single pointer - it is the uniform base we |
4351 | // are looking for. If first operand of the GEP is a splat vector - we |
4352 | // extract the splat value and use it as a uniform base. |
4353 | // In all other cases the function returns 'false'. |
4354 | static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, |
4355 | ISD::MemIndexType &IndexType, SDValue &Scale, |
4356 | SelectionDAGBuilder *SDB, const BasicBlock *CurBB) { |
4357 | SelectionDAG& DAG = SDB->DAG; |
4358 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4359 | const DataLayout &DL = DAG.getDataLayout(); |
4360 | |
4361 | assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")((void)0); |
4362 | |
4363 | // Handle splat constant pointer. |
4364 | if (auto *C = dyn_cast<Constant>(Ptr)) { |
4365 | C = C->getSplatValue(); |
4366 | if (!C) |
4367 | return false; |
4368 | |
4369 | Base = SDB->getValue(C); |
4370 | |
4371 | ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount(); |
4372 | EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts); |
4373 | Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT); |
4374 | IndexType = ISD::SIGNED_SCALED; |
4375 | Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); |
4376 | return true; |
4377 | } |
4378 | |
4379 | const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr); |
4380 | if (!GEP || GEP->getParent() != CurBB) |
4381 | return false; |
4382 | |
4383 | if (GEP->getNumOperands() != 2) |
4384 | return false; |
4385 | |
4386 | const Value *BasePtr = GEP->getPointerOperand(); |
4387 | const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1); |
4388 | |
4389 | // Make sure the base is scalar and the index is a vector. |
4390 | if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy()) |
4391 | return false; |
4392 | |
4393 | Base = SDB->getValue(BasePtr); |
4394 | Index = SDB->getValue(IndexVal); |
4395 | IndexType = ISD::SIGNED_SCALED; |
4396 | Scale = DAG.getTargetConstant( |
4397 | DL.getTypeAllocSize(GEP->getResultElementType()), |
4398 | SDB->getCurSDLoc(), TLI.getPointerTy(DL)); |
4399 | return true; |
4400 | } |
4401 | |
4402 | void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { |
4403 | SDLoc sdl = getCurSDLoc(); |
4404 | |
4405 | // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask) |
4406 | const Value *Ptr = I.getArgOperand(1); |
4407 | SDValue Src0 = getValue(I.getArgOperand(0)); |
4408 | SDValue Mask = getValue(I.getArgOperand(3)); |
4409 | EVT VT = Src0.getValueType(); |
4410 | Align Alignment = cast<ConstantInt>(I.getArgOperand(2)) |
4411 | ->getMaybeAlignValue() |
4412 | .getValueOr(DAG.getEVTAlign(VT.getScalarType())); |
4413 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4414 | |
4415 | AAMDNodes AAInfo; |
4416 | I.getAAMetadata(AAInfo); |
4417 | |
4418 | SDValue Base; |
4419 | SDValue Index; |
4420 | ISD::MemIndexType IndexType; |
4421 | SDValue Scale; |
4422 | bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, |
4423 | I.getParent()); |
4424 | |
4425 | unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); |
4426 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
4427 | MachinePointerInfo(AS), MachineMemOperand::MOStore, |
4428 | // TODO: Make MachineMemOperands aware of scalable |
4429 | // vectors. |
4430 | MemoryLocation::UnknownSize, Alignment, AAInfo); |
4431 | if (!UniformBase) { |
4432 | Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); |
4433 | Index = getValue(Ptr); |
4434 | IndexType = ISD::SIGNED_UNSCALED; |
4435 | Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); |
4436 | } |
4437 | |
4438 | EVT IdxVT = Index.getValueType(); |
4439 | EVT EltTy = IdxVT.getVectorElementType(); |
4440 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { |
4441 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); |
4442 | Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); |
4443 | } |
4444 | |
4445 | SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale }; |
4446 | SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl, |
4447 | Ops, MMO, IndexType, false); |
4448 | DAG.setRoot(Scatter); |
4449 | setValue(&I, Scatter); |
4450 | } |
4451 | |
4452 | void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { |
4453 | SDLoc sdl = getCurSDLoc(); |
4454 | |
4455 | auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, |
4456 | MaybeAlign &Alignment) { |
4457 | // @llvm.masked.load.*(Ptr, alignment, Mask, Src0) |
4458 | Ptr = I.getArgOperand(0); |
4459 | Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue(); |
4460 | Mask = I.getArgOperand(2); |
4461 | Src0 = I.getArgOperand(3); |
4462 | }; |
4463 | auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, |
4464 | MaybeAlign &Alignment) { |
4465 | // @llvm.masked.expandload.*(Ptr, Mask, Src0) |
4466 | Ptr = I.getArgOperand(0); |
4467 | Alignment = None; |
4468 | Mask = I.getArgOperand(1); |
4469 | Src0 = I.getArgOperand(2); |
4470 | }; |
4471 | |
4472 | Value *PtrOperand, *MaskOperand, *Src0Operand; |
4473 | MaybeAlign Alignment; |
4474 | if (IsExpanding) |
4475 | getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); |
4476 | else |
4477 | getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); |
4478 | |
4479 | SDValue Ptr = getValue(PtrOperand); |
4480 | SDValue Src0 = getValue(Src0Operand); |
4481 | SDValue Mask = getValue(MaskOperand); |
4482 | SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); |
4483 | |
4484 | EVT VT = Src0.getValueType(); |
4485 | if (!Alignment) |
4486 | Alignment = DAG.getEVTAlign(VT); |
4487 | |
4488 | AAMDNodes AAInfo; |
4489 | I.getAAMetadata(AAInfo); |
4490 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); |
4491 | |
4492 | // Do not serialize masked loads of constant memory with anything. |
4493 | MemoryLocation ML; |
4494 | if (VT.isScalableVector()) |
4495 | ML = MemoryLocation::getAfter(PtrOperand); |
4496 | else |
4497 | ML = MemoryLocation(PtrOperand, LocationSize::precise( |
4498 | DAG.getDataLayout().getTypeStoreSize(I.getType())), |
4499 | AAInfo); |
4500 | bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); |
4501 | |
4502 | SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); |
4503 | |
4504 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
4505 | MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, |
4506 | // TODO: Make MachineMemOperands aware of scalable |
4507 | // vectors. |
4508 | VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges); |
4509 | |
4510 | SDValue Load = |
4511 | DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO, |
4512 | ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding); |
4513 | if (AddToChain) |
4514 | PendingLoads.push_back(Load.getValue(1)); |
4515 | setValue(&I, Load); |
4516 | } |
4517 | |
4518 | void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { |
4519 | SDLoc sdl = getCurSDLoc(); |
4520 | |
4521 | // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) |
4522 | const Value *Ptr = I.getArgOperand(0); |
4523 | SDValue Src0 = getValue(I.getArgOperand(3)); |
4524 | SDValue Mask = getValue(I.getArgOperand(2)); |
4525 | |
4526 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4527 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
4528 | Align Alignment = cast<ConstantInt>(I.getArgOperand(1)) |
4529 | ->getMaybeAlignValue() |
4530 | .getValueOr(DAG.getEVTAlign(VT.getScalarType())); |
4531 | |
4532 | AAMDNodes AAInfo; |
4533 | I.getAAMetadata(AAInfo); |
4534 | const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); |
4535 | |
4536 | SDValue Root = DAG.getRoot(); |
4537 | SDValue Base; |
4538 | SDValue Index; |
4539 | ISD::MemIndexType IndexType; |
4540 | SDValue Scale; |
4541 | bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, |
4542 | I.getParent()); |
4543 | unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); |
4544 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
4545 | MachinePointerInfo(AS), MachineMemOperand::MOLoad, |
4546 | // TODO: Make MachineMemOperands aware of scalable |
4547 | // vectors. |
4548 | MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges); |
4549 | |
4550 | if (!UniformBase) { |
4551 | Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); |
4552 | Index = getValue(Ptr); |
4553 | IndexType = ISD::SIGNED_UNSCALED; |
4554 | Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); |
4555 | } |
4556 | |
4557 | EVT IdxVT = Index.getValueType(); |
4558 | EVT EltTy = IdxVT.getVectorElementType(); |
4559 | if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { |
4560 | EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); |
4561 | Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); |
4562 | } |
4563 | |
4564 | SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale }; |
4565 | SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, |
4566 | Ops, MMO, IndexType, ISD::NON_EXTLOAD); |
4567 | |
4568 | PendingLoads.push_back(Gather.getValue(1)); |
4569 | setValue(&I, Gather); |
4570 | } |
4571 | |
4572 | void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { |
4573 | SDLoc dl = getCurSDLoc(); |
4574 | AtomicOrdering SuccessOrdering = I.getSuccessOrdering(); |
4575 | AtomicOrdering FailureOrdering = I.getFailureOrdering(); |
4576 | SyncScope::ID SSID = I.getSyncScopeID(); |
4577 | |
4578 | SDValue InChain = getRoot(); |
4579 | |
4580 | MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType(); |
4581 | SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); |
4582 | |
4583 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4584 | auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); |
4585 | |
4586 | MachineFunction &MF = DAG.getMachineFunction(); |
4587 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
4588 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), |
4589 | DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering, |
4590 | FailureOrdering); |
4591 | |
4592 | SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, |
4593 | dl, MemVT, VTs, InChain, |
4594 | getValue(I.getPointerOperand()), |
4595 | getValue(I.getCompareOperand()), |
4596 | getValue(I.getNewValOperand()), MMO); |
4597 | |
4598 | SDValue OutChain = L.getValue(2); |
4599 | |
4600 | setValue(&I, L); |
4601 | DAG.setRoot(OutChain); |
4602 | } |
4603 | |
4604 | void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { |
4605 | SDLoc dl = getCurSDLoc(); |
4606 | ISD::NodeType NT; |
4607 | switch (I.getOperation()) { |
4608 | default: llvm_unreachable("Unknown atomicrmw operation")__builtin_unreachable(); |
4609 | case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; |
4610 | case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; |
4611 | case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; |
4612 | case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; |
4613 | case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; |
4614 | case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; |
4615 | case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; |
4616 | case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; |
4617 | case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; |
4618 | case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; |
4619 | case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; |
4620 | case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break; |
4621 | case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break; |
4622 | } |
4623 | AtomicOrdering Ordering = I.getOrdering(); |
4624 | SyncScope::ID SSID = I.getSyncScopeID(); |
4625 | |
4626 | SDValue InChain = getRoot(); |
4627 | |
4628 | auto MemVT = getValue(I.getValOperand()).getSimpleValueType(); |
4629 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4630 | auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); |
4631 | |
4632 | MachineFunction &MF = DAG.getMachineFunction(); |
4633 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
4634 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), |
4635 | DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering); |
4636 | |
4637 | SDValue L = |
4638 | DAG.getAtomic(NT, dl, MemVT, InChain, |
4639 | getValue(I.getPointerOperand()), getValue(I.getValOperand()), |
4640 | MMO); |
4641 | |
4642 | SDValue OutChain = L.getValue(1); |
4643 | |
4644 | setValue(&I, L); |
4645 | DAG.setRoot(OutChain); |
4646 | } |
4647 | |
4648 | void SelectionDAGBuilder::visitFence(const FenceInst &I) { |
4649 | SDLoc dl = getCurSDLoc(); |
4650 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4651 | SDValue Ops[3]; |
4652 | Ops[0] = getRoot(); |
4653 | Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl, |
4654 | TLI.getFenceOperandTy(DAG.getDataLayout())); |
4655 | Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl, |
4656 | TLI.getFenceOperandTy(DAG.getDataLayout())); |
4657 | DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops)); |
4658 | } |
4659 | |
4660 | void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { |
4661 | SDLoc dl = getCurSDLoc(); |
4662 | AtomicOrdering Order = I.getOrdering(); |
4663 | SyncScope::ID SSID = I.getSyncScopeID(); |
4664 | |
4665 | SDValue InChain = getRoot(); |
4666 | |
4667 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4668 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
4669 | EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); |
4670 | |
4671 | if (!TLI.supportsUnalignedAtomics() && |
4672 | I.getAlignment() < MemVT.getSizeInBits() / 8) |
4673 | report_fatal_error("Cannot generate unaligned atomic load"); |
4674 | |
4675 | auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout()); |
4676 | |
4677 | MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( |
4678 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), |
4679 | I.getAlign(), AAMDNodes(), nullptr, SSID, Order); |
4680 | |
4681 | InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); |
4682 | |
4683 | SDValue Ptr = getValue(I.getPointerOperand()); |
4684 | |
4685 | if (TLI.lowerAtomicLoadAsLoadSDNode(I)) { |
4686 | // TODO: Once this is better exercised by tests, it should be merged with |
4687 | // the normal path for loads to prevent future divergence. |
4688 | SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO); |
4689 | if (MemVT != VT) |
4690 | L = DAG.getPtrExtOrTrunc(L, dl, VT); |
4691 | |
4692 | setValue(&I, L); |
4693 | SDValue OutChain = L.getValue(1); |
4694 | if (!I.isUnordered()) |
4695 | DAG.setRoot(OutChain); |
4696 | else |
4697 | PendingLoads.push_back(OutChain); |
4698 | return; |
4699 | } |
4700 | |
4701 | SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain, |
4702 | Ptr, MMO); |
4703 | |
4704 | SDValue OutChain = L.getValue(1); |
4705 | if (MemVT != VT) |
4706 | L = DAG.getPtrExtOrTrunc(L, dl, VT); |
4707 | |
4708 | setValue(&I, L); |
4709 | DAG.setRoot(OutChain); |
4710 | } |
4711 | |
4712 | void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { |
4713 | SDLoc dl = getCurSDLoc(); |
4714 | |
4715 | AtomicOrdering Ordering = I.getOrdering(); |
4716 | SyncScope::ID SSID = I.getSyncScopeID(); |
4717 | |
4718 | SDValue InChain = getRoot(); |
4719 | |
4720 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4721 | EVT MemVT = |
4722 | TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); |
4723 | |
4724 | if (I.getAlignment() < MemVT.getSizeInBits() / 8) |
4725 | report_fatal_error("Cannot generate unaligned atomic store"); |
4726 | |
4727 | auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); |
4728 | |
4729 | MachineFunction &MF = DAG.getMachineFunction(); |
4730 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
4731 | MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), |
4732 | I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering); |
4733 | |
4734 | SDValue Val = getValue(I.getValueOperand()); |
4735 | if (Val.getValueType() != MemVT) |
4736 | Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT); |
4737 | SDValue Ptr = getValue(I.getPointerOperand()); |
4738 | |
4739 | if (TLI.lowerAtomicStoreAsStoreSDNode(I)) { |
4740 | // TODO: Once this is better exercised by tests, it should be merged with |
4741 | // the normal path for stores to prevent future divergence. |
4742 | SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO); |
4743 | DAG.setRoot(S); |
4744 | return; |
4745 | } |
4746 | SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, |
4747 | Ptr, Val, MMO); |
4748 | |
4749 | |
4750 | DAG.setRoot(OutChain); |
4751 | } |
4752 | |
4753 | /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC |
4754 | /// node. |
4755 | void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, |
4756 | unsigned Intrinsic) { |
4757 | // Ignore the callsite's attributes. A specific call site may be marked with |
4758 | // readnone, but the lowering code will expect the chain based on the |
4759 | // definition. |
4760 | const Function *F = I.getCalledFunction(); |
4761 | bool HasChain = !F->doesNotAccessMemory(); |
4762 | bool OnlyLoad = HasChain && F->onlyReadsMemory(); |
4763 | |
4764 | // Build the operand list. |
4765 | SmallVector<SDValue, 8> Ops; |
4766 | if (HasChain) { // If this intrinsic has side-effects, chainify it. |
4767 | if (OnlyLoad) { |
4768 | // We don't need to serialize loads against other loads. |
4769 | Ops.push_back(DAG.getRoot()); |
4770 | } else { |
4771 | Ops.push_back(getRoot()); |
4772 | } |
4773 | } |
4774 | |
4775 | // Info is set by getTgtMemInstrinsic |
4776 | TargetLowering::IntrinsicInfo Info; |
4777 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
4778 | bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, |
4779 | DAG.getMachineFunction(), |
4780 | Intrinsic); |
4781 | |
4782 | // Add the intrinsic ID as an integer operand if it's not a target intrinsic. |
4783 | if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || |
4784 | Info.opc == ISD::INTRINSIC_W_CHAIN) |
4785 | Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(), |
4786 | TLI.getPointerTy(DAG.getDataLayout()))); |
4787 | |
4788 | // Add all operands of the call to the operand list. |
4789 | for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { |
4790 | const Value *Arg = I.getArgOperand(i); |
4791 | if (!I.paramHasAttr(i, Attribute::ImmArg)) { |
4792 | Ops.push_back(getValue(Arg)); |
4793 | continue; |
4794 | } |
4795 | |
4796 | // Use TargetConstant instead of a regular constant for immarg. |
4797 | EVT VT = TLI.getValueType(*DL, Arg->getType(), true); |
4798 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) { |
4799 | assert(CI->getBitWidth() <= 64 &&((void)0) |
4800 | "large intrinsic immediates not handled")((void)0); |
4801 | Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT)); |
4802 | } else { |
4803 | Ops.push_back( |
4804 | DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT)); |
4805 | } |
4806 | } |
4807 | |
4808 | SmallVector<EVT, 4> ValueVTs; |
4809 | ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); |
4810 | |
4811 | if (HasChain) |
4812 | ValueVTs.push_back(MVT::Other); |
4813 | |
4814 | SDVTList VTs = DAG.getVTList(ValueVTs); |
4815 | |
4816 | // Propagate fast-math-flags from IR to node(s). |
4817 | SDNodeFlags Flags; |
4818 | if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) |
4819 | Flags.copyFMF(*FPMO); |
4820 | SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); |
4821 | |
4822 | // Create the node. |
4823 | SDValue Result; |
4824 | if (IsTgtIntrinsic) { |
4825 | // This is target intrinsic that touches memory |
4826 | AAMDNodes AAInfo; |
4827 | I.getAAMetadata(AAInfo); |
4828 | Result = |
4829 | DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT, |
4830 | MachinePointerInfo(Info.ptrVal, Info.offset), |
4831 | Info.align, Info.flags, Info.size, AAInfo); |
4832 | } else if (!HasChain) { |
4833 | Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); |
4834 | } else if (!I.getType()->isVoidTy()) { |
4835 | Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); |
4836 | } else { |
4837 | Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); |
4838 | } |
4839 | |
4840 | if (HasChain) { |
4841 | SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); |
4842 | if (OnlyLoad) |
4843 | PendingLoads.push_back(Chain); |
4844 | else |
4845 | DAG.setRoot(Chain); |
4846 | } |
4847 | |
4848 | if (!I.getType()->isVoidTy()) { |
4849 | if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) { |
4850 | EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy); |
4851 | Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result); |
4852 | } else |
4853 | Result = lowerRangeToAssertZExt(DAG, I, Result); |
4854 | |
4855 | MaybeAlign Alignment = I.getRetAlign(); |
4856 | if (!Alignment) |
4857 | Alignment = F->getAttributes().getRetAlignment(); |
4858 | // Insert `assertalign` node if there's an alignment. |
4859 | if (InsertAssertAlign && Alignment) { |
4860 | Result = |
4861 | DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); |
4862 | } |
4863 | |
4864 | setValue(&I, Result); |
4865 | } |
4866 | } |
4867 | |
4868 | /// GetSignificand - Get the significand and build it into a floating-point |
4869 | /// number with exponent of 1: |
4870 | /// |
4871 | /// Op = (Op & 0x007fffff) | 0x3f800000; |
4872 | /// |
4873 | /// where Op is the hexadecimal representation of floating point value. |
4874 | static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) { |
4875 | SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, |
4876 | DAG.getConstant(0x007fffff, dl, MVT::i32)); |
4877 | SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, |
4878 | DAG.getConstant(0x3f800000, dl, MVT::i32)); |
4879 | return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); |
4880 | } |
4881 | |
4882 | /// GetExponent - Get the exponent: |
4883 | /// |
4884 | /// (float)(int)(((Op & 0x7f800000) >> 23) - 127); |
4885 | /// |
4886 | /// where Op is the hexadecimal representation of floating point value. |
4887 | static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, |
4888 | const TargetLowering &TLI, const SDLoc &dl) { |
4889 | SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, |
4890 | DAG.getConstant(0x7f800000, dl, MVT::i32)); |
4891 | SDValue t1 = DAG.getNode( |
4892 | ISD::SRL, dl, MVT::i32, t0, |
4893 | DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
4894 | SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, |
4895 | DAG.getConstant(127, dl, MVT::i32)); |
4896 | return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); |
4897 | } |
4898 | |
4899 | /// getF32Constant - Get 32-bit floating point constant. |
4900 | static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, |
4901 | const SDLoc &dl) { |
4902 | return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl, |
4903 | MVT::f32); |
4904 | } |
4905 | |
4906 | static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, |
4907 | SelectionDAG &DAG) { |
4908 | // TODO: What fast-math-flags should be set on the floating-point nodes? |
4909 | |
4910 | // IntegerPartOfX = ((int32_t)(t0); |
4911 | SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); |
4912 | |
4913 | // FractionalPartOfX = t0 - (float)IntegerPartOfX; |
4914 | SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); |
4915 | SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); |
4916 | |
4917 | // IntegerPartOfX <<= 23; |
4918 | IntegerPartOfX = DAG.getNode( |
4919 | ISD::SHL, dl, MVT::i32, IntegerPartOfX, |
4920 | DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy( |
4921 | DAG.getDataLayout()))); |
4922 | |
4923 | SDValue TwoToFractionalPartOfX; |
4924 | if (LimitFloatPrecision <= 6) { |
4925 | // For floating-point precision of 6: |
4926 | // |
4927 | // TwoToFractionalPartOfX = |
4928 | // 0.997535578f + |
4929 | // (0.735607626f + 0.252464424f * x) * x; |
4930 | // |
4931 | // error 0.0144103317, which is 6 bits |
4932 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
4933 | getF32Constant(DAG, 0x3e814304, dl)); |
4934 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, |
4935 | getF32Constant(DAG, 0x3f3c50c8, dl)); |
4936 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
4937 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, |
4938 | getF32Constant(DAG, 0x3f7f5e7e, dl)); |
4939 | } else if (LimitFloatPrecision <= 12) { |
4940 | // For floating-point precision of 12: |
4941 | // |
4942 | // TwoToFractionalPartOfX = |
4943 | // 0.999892986f + |
4944 | // (0.696457318f + |
4945 | // (0.224338339f + 0.792043434e-1f * x) * x) * x; |
4946 | // |
4947 | // error 0.000107046256, which is 13 to 14 bits |
4948 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
4949 | getF32Constant(DAG, 0x3da235e3, dl)); |
4950 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, |
4951 | getF32Constant(DAG, 0x3e65b8f3, dl)); |
4952 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
4953 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, |
4954 | getF32Constant(DAG, 0x3f324b07, dl)); |
4955 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); |
4956 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, |
4957 | getF32Constant(DAG, 0x3f7ff8fd, dl)); |
4958 | } else { // LimitFloatPrecision <= 18 |
4959 | // For floating-point precision of 18: |
4960 | // |
4961 | // TwoToFractionalPartOfX = |
4962 | // 0.999999982f + |
4963 | // (0.693148872f + |
4964 | // (0.240227044f + |
4965 | // (0.554906021e-1f + |
4966 | // (0.961591928e-2f + |
4967 | // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; |
4968 | // error 2.47208000*10^(-7), which is better than 18 bits |
4969 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
4970 | getF32Constant(DAG, 0x3924b03e, dl)); |
4971 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, |
4972 | getF32Constant(DAG, 0x3ab24b87, dl)); |
4973 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
4974 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, |
4975 | getF32Constant(DAG, 0x3c1d8c17, dl)); |
4976 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); |
4977 | SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, |
4978 | getF32Constant(DAG, 0x3d634a1d, dl)); |
4979 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); |
4980 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, |
4981 | getF32Constant(DAG, 0x3e75fe14, dl)); |
4982 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); |
4983 | SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, |
4984 | getF32Constant(DAG, 0x3f317234, dl)); |
4985 | SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); |
4986 | TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, |
4987 | getF32Constant(DAG, 0x3f800000, dl)); |
4988 | } |
4989 | |
4990 | // Add the exponent into the result in integer domain. |
4991 | SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX); |
4992 | return DAG.getNode(ISD::BITCAST, dl, MVT::f32, |
4993 | DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX)); |
4994 | } |
4995 | |
4996 | /// expandExp - Lower an exp intrinsic. Handles the special sequences for |
4997 | /// limited-precision mode. |
4998 | static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, |
4999 | const TargetLowering &TLI, SDNodeFlags Flags) { |
5000 | if (Op.getValueType() == MVT::f32 && |
5001 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { |
5002 | |
5003 | // Put the exponent in the right bit position for later addition to the |
5004 | // final result: |
5005 | // |
5006 | // t0 = Op * log2(e) |
5007 | |
5008 | // TODO: What fast-math-flags should be set here? |
5009 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, |
5010 | DAG.getConstantFP(numbers::log2ef, dl, MVT::f32)); |
5011 | return getLimitedPrecisionExp2(t0, dl, DAG); |
5012 | } |
5013 | |
5014 | // No special expansion. |
5015 | return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags); |
5016 | } |
5017 | |
5018 | /// expandLog - Lower a log intrinsic. Handles the special sequences for |
5019 | /// limited-precision mode. |
5020 | static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, |
5021 | const TargetLowering &TLI, SDNodeFlags Flags) { |
5022 | // TODO: What fast-math-flags should be set on the floating-point nodes? |
5023 | |
5024 | if (Op.getValueType() == MVT::f32 && |
5025 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { |
5026 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); |
5027 | |
5028 | // Scale the exponent by log(2). |
5029 | SDValue Exp = GetExponent(DAG, Op1, TLI, dl); |
5030 | SDValue LogOfExponent = |
5031 | DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, |
5032 | DAG.getConstantFP(numbers::ln2f, dl, MVT::f32)); |
5033 | |
5034 | // Get the significand and build it into a floating-point number with |
5035 | // exponent of 1. |
5036 | SDValue X = GetSignificand(DAG, Op1, dl); |
5037 | |
5038 | SDValue LogOfMantissa; |
5039 | if (LimitFloatPrecision <= 6) { |
5040 | // For floating-point precision of 6: |
5041 | // |
5042 | // LogofMantissa = |
5043 | // -1.1609546f + |
5044 | // (1.4034025f - 0.23903021f * x) * x; |
5045 | // |
5046 | // error 0.0034276066, which is better than 8 bits |
5047 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5048 | getF32Constant(DAG, 0xbe74c456, dl)); |
5049 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, |
5050 | getF32Constant(DAG, 0x3fb3a2b1, dl)); |
5051 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5052 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, |
5053 | getF32Constant(DAG, 0x3f949a29, dl)); |
5054 | } else if (LimitFloatPrecision <= 12) { |
5055 | // For floating-point precision of 12: |
5056 | // |
5057 | // LogOfMantissa = |
5058 | // -1.7417939f + |
5059 | // (2.8212026f + |
5060 | // (-1.4699568f + |
5061 | // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; |
5062 | // |
5063 | // error 0.000061011436, which is 14 bits |
5064 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5065 | getF32Constant(DAG, 0xbd67b6d6, dl)); |
5066 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, |
5067 | getF32Constant(DAG, 0x3ee4f4b8, dl)); |
5068 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5069 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, |
5070 | getF32Constant(DAG, 0x3fbc278b, dl)); |
5071 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
5072 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, |
5073 | getF32Constant(DAG, 0x40348e95, dl)); |
5074 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); |
5075 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, |
5076 | getF32Constant(DAG, 0x3fdef31a, dl)); |
5077 | } else { // LimitFloatPrecision <= 18 |
5078 | // For floating-point precision of 18: |
5079 | // |
5080 | // LogOfMantissa = |
5081 | // -2.1072184f + |
5082 | // (4.2372794f + |
5083 | // (-3.7029485f + |
5084 | // (2.2781945f + |
5085 | // (-0.87823314f + |
5086 | // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; |
5087 | // |
5088 | // error 0.0000023660568, which is better than 18 bits |
5089 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5090 | getF32Constant(DAG, 0xbc91e5ac, dl)); |
5091 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, |
5092 | getF32Constant(DAG, 0x3e4350aa, dl)); |
5093 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5094 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, |
5095 | getF32Constant(DAG, 0x3f60d3e3, dl)); |
5096 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
5097 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, |
5098 | getF32Constant(DAG, 0x4011cdf0, dl)); |
5099 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); |
5100 | SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, |
5101 | getF32Constant(DAG, 0x406cfd1c, dl)); |
5102 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); |
5103 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, |
5104 | getF32Constant(DAG, 0x408797cb, dl)); |
5105 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); |
5106 | LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, |
5107 | getF32Constant(DAG, 0x4006dcab, dl)); |
5108 | } |
5109 | |
5110 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); |
5111 | } |
5112 | |
5113 | // No special expansion. |
5114 | return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags); |
5115 | } |
5116 | |
5117 | /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for |
5118 | /// limited-precision mode. |
5119 | static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, |
5120 | const TargetLowering &TLI, SDNodeFlags Flags) { |
5121 | // TODO: What fast-math-flags should be set on the floating-point nodes? |
5122 | |
5123 | if (Op.getValueType() == MVT::f32 && |
5124 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { |
5125 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); |
5126 | |
5127 | // Get the exponent. |
5128 | SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); |
5129 | |
5130 | // Get the significand and build it into a floating-point number with |
5131 | // exponent of 1. |
5132 | SDValue X = GetSignificand(DAG, Op1, dl); |
5133 | |
5134 | // Different possible minimax approximations of significand in |
5135 | // floating-point for various degrees of accuracy over [1,2]. |
5136 | SDValue Log2ofMantissa; |
5137 | if (LimitFloatPrecision <= 6) { |
5138 | // For floating-point precision of 6: |
5139 | // |
5140 | // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; |
5141 | // |
5142 | // error 0.0049451742, which is more than 7 bits |
5143 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5144 | getF32Constant(DAG, 0xbeb08fe0, dl)); |
5145 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, |
5146 | getF32Constant(DAG, 0x40019463, dl)); |
5147 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5148 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, |
5149 | getF32Constant(DAG, 0x3fd6633d, dl)); |
5150 | } else if (LimitFloatPrecision <= 12) { |
5151 | // For floating-point precision of 12: |
5152 | // |
5153 | // Log2ofMantissa = |
5154 | // -2.51285454f + |
5155 | // (4.07009056f + |
5156 | // (-2.12067489f + |
5157 | // (.645142248f - 0.816157886e-1f * x) * x) * x) * x; |
5158 | // |
5159 | // error 0.0000876136000, which is better than 13 bits |
5160 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5161 | getF32Constant(DAG, 0xbda7262e, dl)); |
5162 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, |
5163 | getF32Constant(DAG, 0x3f25280b, dl)); |
5164 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5165 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, |
5166 | getF32Constant(DAG, 0x4007b923, dl)); |
5167 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
5168 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, |
5169 | getF32Constant(DAG, 0x40823e2f, dl)); |
5170 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); |
5171 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, |
5172 | getF32Constant(DAG, 0x4020d29c, dl)); |
5173 | } else { // LimitFloatPrecision <= 18 |
5174 | // For floating-point precision of 18: |
5175 | // |
5176 | // Log2ofMantissa = |
5177 | // -3.0400495f + |
5178 | // (6.1129976f + |
5179 | // (-5.3420409f + |
5180 | // (3.2865683f + |
5181 | // (-1.2669343f + |
5182 | // (0.27515199f - |
5183 | // 0.25691327e-1f * x) * x) * x) * x) * x) * x; |
5184 | // |
5185 | // error 0.0000018516, which is better than 18 bits |
5186 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5187 | getF32Constant(DAG, 0xbcd2769e, dl)); |
5188 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, |
5189 | getF32Constant(DAG, 0x3e8ce0b9, dl)); |
5190 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5191 | SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, |
5192 | getF32Constant(DAG, 0x3fa22ae7, dl)); |
5193 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
5194 | SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, |
5195 | getF32Constant(DAG, 0x40525723, dl)); |
5196 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); |
5197 | SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, |
5198 | getF32Constant(DAG, 0x40aaf200, dl)); |
5199 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); |
5200 | SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, |
5201 | getF32Constant(DAG, 0x40c39dad, dl)); |
5202 | SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); |
5203 | Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, |
5204 | getF32Constant(DAG, 0x4042902c, dl)); |
5205 | } |
5206 | |
5207 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); |
5208 | } |
5209 | |
5210 | // No special expansion. |
5211 | return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags); |
5212 | } |
5213 | |
5214 | /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for |
5215 | /// limited-precision mode. |
5216 | static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, |
5217 | const TargetLowering &TLI, SDNodeFlags Flags) { |
5218 | // TODO: What fast-math-flags should be set on the floating-point nodes? |
5219 | |
5220 | if (Op.getValueType() == MVT::f32 && |
5221 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { |
5222 | SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); |
5223 | |
5224 | // Scale the exponent by log10(2) [0.30102999f]. |
5225 | SDValue Exp = GetExponent(DAG, Op1, TLI, dl); |
5226 | SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, |
5227 | getF32Constant(DAG, 0x3e9a209a, dl)); |
5228 | |
5229 | // Get the significand and build it into a floating-point number with |
5230 | // exponent of 1. |
5231 | SDValue X = GetSignificand(DAG, Op1, dl); |
5232 | |
5233 | SDValue Log10ofMantissa; |
5234 | if (LimitFloatPrecision <= 6) { |
5235 | // For floating-point precision of 6: |
5236 | // |
5237 | // Log10ofMantissa = |
5238 | // -0.50419619f + |
5239 | // (0.60948995f - 0.10380950f * x) * x; |
5240 | // |
5241 | // error 0.0014886165, which is 6 bits |
5242 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5243 | getF32Constant(DAG, 0xbdd49a13, dl)); |
5244 | SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, |
5245 | getF32Constant(DAG, 0x3f1c0789, dl)); |
5246 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5247 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, |
5248 | getF32Constant(DAG, 0x3f011300, dl)); |
5249 | } else if (LimitFloatPrecision <= 12) { |
5250 | // For floating-point precision of 12: |
5251 | // |
5252 | // Log10ofMantissa = |
5253 | // -0.64831180f + |
5254 | // (0.91751397f + |
5255 | // (-0.31664806f + 0.47637168e-1f * x) * x) * x; |
5256 | // |
5257 | // error 0.00019228036, which is better than 12 bits |
5258 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5259 | getF32Constant(DAG, 0x3d431f31, dl)); |
5260 | SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, |
5261 | getF32Constant(DAG, 0x3ea21fb2, dl)); |
5262 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5263 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, |
5264 | getF32Constant(DAG, 0x3f6ae232, dl)); |
5265 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
5266 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, |
5267 | getF32Constant(DAG, 0x3f25f7c3, dl)); |
5268 | } else { // LimitFloatPrecision <= 18 |
5269 | // For floating-point precision of 18: |
5270 | // |
5271 | // Log10ofMantissa = |
5272 | // -0.84299375f + |
5273 | // (1.5327582f + |
5274 | // (-1.0688956f + |
5275 | // (0.49102474f + |
5276 | // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; |
5277 | // |
5278 | // error 0.0000037995730, which is better than 18 bits |
5279 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, |
5280 | getF32Constant(DAG, 0x3c5d51ce, dl)); |
5281 | SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, |
5282 | getF32Constant(DAG, 0x3e00685a, dl)); |
5283 | SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); |
5284 | SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, |
5285 | getF32Constant(DAG, 0x3efb6798, dl)); |
5286 | SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); |
5287 | SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, |
5288 | getF32Constant(DAG, 0x3f88d192, dl)); |
5289 | SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); |
5290 | SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, |
5291 | getF32Constant(DAG, 0x3fc4316c, dl)); |
5292 | SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); |
5293 | Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, |
5294 | getF32Constant(DAG, 0x3f57ce70, dl)); |
5295 | } |
5296 | |
5297 | return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); |
5298 | } |
5299 | |
5300 | // No special expansion. |
5301 | return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags); |
5302 | } |
5303 | |
5304 | /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for |
5305 | /// limited-precision mode. |
5306 | static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, |
5307 | const TargetLowering &TLI, SDNodeFlags Flags) { |
5308 | if (Op.getValueType() == MVT::f32 && |
5309 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) |
5310 | return getLimitedPrecisionExp2(Op, dl, DAG); |
5311 | |
5312 | // No special expansion. |
5313 | return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags); |
5314 | } |
5315 | |
5316 | /// visitPow - Lower a pow intrinsic. Handles the special sequences for |
5317 | /// limited-precision mode with x == 10.0f. |
5318 | static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, |
5319 | SelectionDAG &DAG, const TargetLowering &TLI, |
5320 | SDNodeFlags Flags) { |
5321 | bool IsExp10 = false; |
5322 | if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 && |
5323 | LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { |
5324 | if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { |
5325 | APFloat Ten(10.0f); |
5326 | IsExp10 = LHSC->isExactlyValue(Ten); |
5327 | } |
5328 | } |
5329 | |
5330 | // TODO: What fast-math-flags should be set on the FMUL node? |
5331 | if (IsExp10) { |
5332 | // Put the exponent in the right bit position for later addition to the |
5333 | // final result: |
5334 | // |
5335 | // #define LOG2OF10 3.3219281f |
5336 | // t0 = Op * LOG2OF10; |
5337 | SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, |
5338 | getF32Constant(DAG, 0x40549a78, dl)); |
5339 | return getLimitedPrecisionExp2(t0, dl, DAG); |
5340 | } |
5341 | |
5342 | // No special expansion. |
5343 | return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags); |
5344 | } |
5345 | |
5346 | /// ExpandPowI - Expand a llvm.powi intrinsic. |
5347 | static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, |
5348 | SelectionDAG &DAG) { |
5349 | // If RHS is a constant, we can expand this out to a multiplication tree, |
5350 | // otherwise we end up lowering to a call to __powidf2 (for example). When |
5351 | // optimizing for size, we only want to do this if the expansion would produce |
5352 | // a small number of multiplies, otherwise we do the full expansion. |
5353 | if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { |
5354 | // Get the exponent as a positive value. |
5355 | unsigned Val = RHSC->getSExtValue(); |
5356 | if ((int)Val < 0) Val = -Val; |
5357 | |
5358 | // powi(x, 0) -> 1.0 |
5359 | if (Val == 0) |
5360 | return DAG.getConstantFP(1.0, DL, LHS.getValueType()); |
5361 | |
5362 | bool OptForSize = DAG.shouldOptForSize(); |
5363 | if (!OptForSize || |
5364 | // If optimizing for size, don't insert too many multiplies. |
5365 | // This inserts up to 5 multiplies. |
5366 | countPopulation(Val) + Log2_32(Val) < 7) { |
5367 | // We use the simple binary decomposition method to generate the multiply |
5368 | // sequence. There are more optimal ways to do this (for example, |
5369 | // powi(x,15) generates one more multiply than it should), but this has |
5370 | // the benefit of being both really simple and much better than a libcall. |
5371 | SDValue Res; // Logically starts equal to 1.0 |
5372 | SDValue CurSquare = LHS; |
5373 | // TODO: Intrinsics should have fast-math-flags that propagate to these |
5374 | // nodes. |
5375 | while (Val) { |
5376 | if (Val & 1) { |
5377 | if (Res.getNode()) |
5378 | Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare); |
5379 | else |
5380 | Res = CurSquare; // 1.0*CurSquare. |
5381 | } |
5382 | |
5383 | CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), |
5384 | CurSquare, CurSquare); |
5385 | Val >>= 1; |
5386 | } |
5387 | |
5388 | // If the original was negative, invert the result, producing 1/(x*x*x). |
5389 | if (RHSC->getSExtValue() < 0) |
5390 | Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), |
5391 | DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res); |
5392 | return Res; |
5393 | } |
5394 | } |
5395 | |
5396 | // Otherwise, expand to a libcall. |
5397 | return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); |
5398 | } |
5399 | |
5400 | static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, |
5401 | SDValue LHS, SDValue RHS, SDValue Scale, |
5402 | SelectionDAG &DAG, const TargetLowering &TLI) { |
5403 | EVT VT = LHS.getValueType(); |
5404 | bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; |
5405 | bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; |
5406 | LLVMContext &Ctx = *DAG.getContext(); |
5407 | |
5408 | // If the type is legal but the operation isn't, this node might survive all |
5409 | // the way to operation legalization. If we end up there and we do not have |
5410 | // the ability to widen the type (if VT*2 is not legal), we cannot expand the |
5411 | // node. |
5412 | |
5413 | // Coax the legalizer into expanding the node during type legalization instead |
5414 | // by bumping the size by one bit. This will force it to Promote, enabling the |
5415 | // early expansion and avoiding the need to expand later. |
5416 | |
5417 | // We don't have to do this if Scale is 0; that can always be expanded, unless |
5418 | // it's a saturating signed operation. Those can experience true integer |
5419 | // division overflow, a case which we must avoid. |
5420 | |
5421 | // FIXME: We wouldn't have to do this (or any of the early |
5422 | // expansion/promotion) if it was possible to expand a libcall of an |
5423 | // illegal type during operation legalization. But it's not, so things |
5424 | // get a bit hacky. |
5425 | unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue(); |
5426 | if ((ScaleInt > 0 || (Saturating && Signed)) && |
5427 | (TLI.isTypeLegal(VT) || |
5428 | (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) { |
5429 | TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction( |
5430 | Opcode, VT, ScaleInt); |
5431 | if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) { |
5432 | EVT PromVT; |
5433 | if (VT.isScalarInteger()) |
5434 | PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1); |
5435 | else if (VT.isVector()) { |
5436 | PromVT = VT.getVectorElementType(); |
5437 | PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1); |
5438 | PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount()); |
5439 | } else |
5440 | llvm_unreachable("Wrong VT for DIVFIX?")__builtin_unreachable(); |
5441 | if (Signed) { |
5442 | LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT); |
5443 | RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT); |
5444 | } else { |
5445 | LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT); |
5446 | RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT); |
5447 | } |
5448 | EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout()); |
5449 | // For saturating operations, we need to shift up the LHS to get the |
5450 | // proper saturation width, and then shift down again afterwards. |
5451 | if (Saturating) |
5452 | LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS, |
5453 | DAG.getConstant(1, DL, ShiftTy)); |
5454 | SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale); |
5455 | if (Saturating) |
5456 | Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res, |
5457 | DAG.getConstant(1, DL, ShiftTy)); |
5458 | return DAG.getZExtOrTrunc(Res, DL, VT); |
5459 | } |
5460 | } |
5461 | |
5462 | return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale); |
5463 | } |
5464 | |
5465 | // getUnderlyingArgRegs - Find underlying registers used for a truncated, |
5466 | // bitcasted, or split argument. Returns a list of <Register, size in bits> |
5467 | static void |
5468 | getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs, |
5469 | const SDValue &N) { |
5470 | switch (N.getOpcode()) { |
5471 | case ISD::CopyFromReg: { |
5472 | SDValue Op = N.getOperand(1); |
5473 | Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(), |
5474 | Op.getValueType().getSizeInBits()); |
5475 | return; |
5476 | } |
5477 | case ISD::BITCAST: |
5478 | case ISD::AssertZext: |
5479 | case ISD::AssertSext: |
5480 | case ISD::TRUNCATE: |
5481 | getUnderlyingArgRegs(Regs, N.getOperand(0)); |
5482 | return; |
5483 | case ISD::BUILD_PAIR: |
5484 | case ISD::BUILD_VECTOR: |
5485 | case ISD::CONCAT_VECTORS: |
5486 | for (SDValue Op : N->op_values()) |
5487 | getUnderlyingArgRegs(Regs, Op); |
5488 | return; |
5489 | default: |
5490 | return; |
5491 | } |
5492 | } |
5493 | |
5494 | /// If the DbgValueInst is a dbg_value of a function argument, create the |
5495 | /// corresponding DBG_VALUE machine instruction for it now. At the end of |
5496 | /// instruction selection, they will be inserted to the entry BB. |
5497 | /// We don't currently support this for variadic dbg_values, as they shouldn't |
5498 | /// appear for function arguments or in the prologue. |
5499 | bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( |
5500 | const Value *V, DILocalVariable *Variable, DIExpression *Expr, |
5501 | DILocation *DL, bool IsDbgDeclare, const SDValue &N) { |
5502 | const Argument *Arg = dyn_cast<Argument>(V); |
5503 | if (!Arg) |
5504 | return false; |
5505 | |
5506 | MachineFunction &MF = DAG.getMachineFunction(); |
5507 | const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); |
5508 | |
5509 | // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind |
5510 | // we've been asked to pursue. |
5511 | auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr, |
5512 | bool Indirect) { |
5513 | if (Reg.isVirtual() && TM.Options.ValueTrackingVariableLocations) { |
5514 | // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF |
5515 | // pointing at the VReg, which will be patched up later. |
5516 | auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF); |
5517 | auto MIB = BuildMI(MF, DL, Inst); |
5518 | MIB.addReg(Reg, RegState::Debug); |
5519 | MIB.addImm(0); |
5520 | MIB.addMetadata(Variable); |
5521 | auto *NewDIExpr = FragExpr; |
5522 | // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into |
5523 | // the DIExpression. |
5524 | if (Indirect) |
5525 | NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore); |
5526 | MIB.addMetadata(NewDIExpr); |
5527 | return MIB; |
5528 | } else { |
5529 | // Create a completely standard DBG_VALUE. |
5530 | auto &Inst = TII->get(TargetOpcode::DBG_VALUE); |
5531 | return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr); |
5532 | } |
5533 | }; |
5534 | |
5535 | if (!IsDbgDeclare) { |
5536 | // ArgDbgValues are hoisted to the beginning of the entry block. So we |
5537 | // should only emit as ArgDbgValue if the dbg.value intrinsic is found in |
5538 | // the entry block. |
5539 | bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front(); |
5540 | if (!IsInEntryBlock) |
5541 | return false; |
5542 | |
5543 | // ArgDbgValues are hoisted to the beginning of the entry block. So we |
5544 | // should only emit as ArgDbgValue if the dbg.value intrinsic describes a |
5545 | // variable that also is a param. |
5546 | // |
5547 | // Although, if we are at the top of the entry block already, we can still |
5548 | // emit using ArgDbgValue. This might catch some situations when the |
5549 | // dbg.value refers to an argument that isn't used in the entry block, so |
5550 | // any CopyToReg node would be optimized out and the only way to express |
5551 | // this DBG_VALUE is by using the physical reg (or FI) as done in this |
5552 | // method. ArgDbgValues are hoisted to the beginning of the entry block. So |
5553 | // we should only emit as ArgDbgValue if the Variable is an argument to the |
5554 | // current function, and the dbg.value intrinsic is found in the entry |
5555 | // block. |
5556 | bool VariableIsFunctionInputArg = Variable->isParameter() && |
5557 | !DL->getInlinedAt(); |
5558 | bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder; |
5559 | if (!IsInPrologue && !VariableIsFunctionInputArg) |
5560 | return false; |
5561 | |
5562 | // Here we assume that a function argument on IR level only can be used to |
5563 | // describe one input parameter on source level. If we for example have |
5564 | // source code like this |
5565 | // |
5566 | // struct A { long x, y; }; |
5567 | // void foo(struct A a, long b) { |
5568 | // ... |
5569 | // b = a.x; |
5570 | // ... |
5571 | // } |
5572 | // |
5573 | // and IR like this |
5574 | // |
5575 | // define void @foo(i32 %a1, i32 %a2, i32 %b) { |
5576 | // entry: |
5577 | // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment |
5578 | // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment |
5579 | // call void @llvm.dbg.value(metadata i32 %b, "b", |
5580 | // ... |
5581 | // call void @llvm.dbg.value(metadata i32 %a1, "b" |
5582 | // ... |
5583 | // |
5584 | // then the last dbg.value is describing a parameter "b" using a value that |
5585 | // is an argument. But since we already has used %a1 to describe a parameter |
5586 | // we should not handle that last dbg.value here (that would result in an |
5587 | // incorrect hoisting of the DBG_VALUE to the function entry). |
5588 | // Notice that we allow one dbg.value per IR level argument, to accommodate |
5589 | // for the situation with fragments above. |
5590 | if (VariableIsFunctionInputArg) { |
5591 | unsigned ArgNo = Arg->getArgNo(); |
5592 | if (ArgNo >= FuncInfo.DescribedArgs.size()) |
5593 | FuncInfo.DescribedArgs.resize(ArgNo + 1, false); |
5594 | else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo)) |
5595 | return false; |
5596 | FuncInfo.DescribedArgs.set(ArgNo); |
5597 | } |
5598 | } |
5599 | |
5600 | bool IsIndirect = false; |
5601 | Optional<MachineOperand> Op; |
5602 | // Some arguments' frame index is recorded during argument lowering. |
5603 | int FI = FuncInfo.getArgumentFrameIndex(Arg); |
5604 | if (FI != std::numeric_limits<int>::max()) |
5605 | Op = MachineOperand::CreateFI(FI); |
5606 | |
5607 | SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes; |
5608 | if (!Op && N.getNode()) { |
5609 | getUnderlyingArgRegs(ArgRegsAndSizes, N); |
5610 | Register Reg; |
5611 | if (ArgRegsAndSizes.size() == 1) |
5612 | Reg = ArgRegsAndSizes.front().first; |
5613 | |
5614 | if (Reg && Reg.isVirtual()) { |
5615 | MachineRegisterInfo &RegInfo = MF.getRegInfo(); |
5616 | Register PR = RegInfo.getLiveInPhysReg(Reg); |
5617 | if (PR) |
5618 | Reg = PR; |
5619 | } |
5620 | if (Reg) { |
5621 | Op = MachineOperand::CreateReg(Reg, false); |
5622 | IsIndirect = IsDbgDeclare; |
5623 | } |
5624 | } |
5625 | |
5626 | if (!Op && N.getNode()) { |
5627 | // Check if frame index is available. |
5628 | SDValue LCandidate = peekThroughBitcasts(N); |
5629 | if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode())) |
5630 | if (FrameIndexSDNode *FINode = |
5631 | dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) |
5632 | Op = MachineOperand::CreateFI(FINode->getIndex()); |
5633 | } |
5634 | |
5635 | if (!Op) { |
5636 | // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg |
5637 | auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>> |
5638 | SplitRegs) { |
5639 | unsigned Offset = 0; |
5640 | for (auto RegAndSize : SplitRegs) { |
5641 | // If the expression is already a fragment, the current register |
5642 | // offset+size might extend beyond the fragment. In this case, only |
5643 | // the register bits that are inside the fragment are relevant. |
5644 | int RegFragmentSizeInBits = RegAndSize.second; |
5645 | if (auto ExprFragmentInfo = Expr->getFragmentInfo()) { |
5646 | uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits; |
5647 | // The register is entirely outside the expression fragment, |
5648 | // so is irrelevant for debug info. |
5649 | if (Offset >= ExprFragmentSizeInBits) |
5650 | break; |
5651 | // The register is partially outside the expression fragment, only |
5652 | // the low bits within the fragment are relevant for debug info. |
5653 | if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) { |
5654 | RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset; |
5655 | } |
5656 | } |
5657 | |
5658 | auto FragmentExpr = DIExpression::createFragmentExpression( |
5659 | Expr, Offset, RegFragmentSizeInBits); |
5660 | Offset += RegAndSize.second; |
5661 | // If a valid fragment expression cannot be created, the variable's |
5662 | // correct value cannot be determined and so it is set as Undef. |
5663 | if (!FragmentExpr) { |
5664 | SDDbgValue *SDV = DAG.getConstantDbgValue( |
5665 | Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder); |
5666 | DAG.AddDbgValue(SDV, false); |
5667 | continue; |
5668 | } |
5669 | MachineInstr *NewMI = |
5670 | MakeVRegDbgValue(RegAndSize.first, *FragmentExpr, IsDbgDeclare); |
5671 | FuncInfo.ArgDbgValues.push_back(NewMI); |
5672 | } |
5673 | }; |
5674 | |
5675 | // Check if ValueMap has reg number. |
5676 | DenseMap<const Value *, Register>::const_iterator |
5677 | VMI = FuncInfo.ValueMap.find(V); |
5678 | if (VMI != FuncInfo.ValueMap.end()) { |
5679 | const auto &TLI = DAG.getTargetLoweringInfo(); |
5680 | RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second, |
5681 | V->getType(), None); |
5682 | if (RFV.occupiesMultipleRegs()) { |
5683 | splitMultiRegDbgValue(RFV.getRegsAndSizes()); |
5684 | return true; |
5685 | } |
5686 | |
5687 | Op = MachineOperand::CreateReg(VMI->second, false); |
5688 | IsIndirect = IsDbgDeclare; |
5689 | } else if (ArgRegsAndSizes.size() > 1) { |
5690 | // This was split due to the calling convention, and no virtual register |
5691 | // mapping exists for the value. |
5692 | splitMultiRegDbgValue(ArgRegsAndSizes); |
5693 | return true; |
5694 | } |
5695 | } |
5696 | |
5697 | if (!Op) |
5698 | return false; |
5699 | |
5700 | assert(Variable->isValidLocationForIntrinsic(DL) &&((void)0) |
5701 | "Expected inlined-at fields to agree")((void)0); |
5702 | MachineInstr *NewMI = nullptr; |
5703 | |
5704 | if (Op->isReg()) |
5705 | NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect); |
5706 | else |
5707 | NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op, |
5708 | Variable, Expr); |
5709 | |
5710 | FuncInfo.ArgDbgValues.push_back(NewMI); |
5711 | return true; |
5712 | } |
5713 | |
5714 | /// Return the appropriate SDDbgValue based on N. |
5715 | SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N, |
5716 | DILocalVariable *Variable, |
5717 | DIExpression *Expr, |
5718 | const DebugLoc &dl, |
5719 | unsigned DbgSDNodeOrder) { |
5720 | if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { |
5721 | // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe |
5722 | // stack slot locations. |
5723 | // |
5724 | // Consider "int x = 0; int *px = &x;". There are two kinds of interesting |
5725 | // debug values here after optimization: |
5726 | // |
5727 | // dbg.value(i32* %px, !"int *px", !DIExpression()), and |
5728 | // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) |
5729 | // |
5730 | // Both describe the direct values of their associated variables. |
5731 | return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), |
5732 | /*IsIndirect*/ false, dl, DbgSDNodeOrder); |
5733 | } |
5734 | return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), |
5735 | /*IsIndirect*/ false, dl, DbgSDNodeOrder); |
5736 | } |
5737 | |
5738 | static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) { |
5739 | switch (Intrinsic) { |
5740 | case Intrinsic::smul_fix: |
5741 | return ISD::SMULFIX; |
5742 | case Intrinsic::umul_fix: |
5743 | return ISD::UMULFIX; |
5744 | case Intrinsic::smul_fix_sat: |
5745 | return ISD::SMULFIXSAT; |
5746 | case Intrinsic::umul_fix_sat: |
5747 | return ISD::UMULFIXSAT; |
5748 | case Intrinsic::sdiv_fix: |
5749 | return ISD::SDIVFIX; |
5750 | case Intrinsic::udiv_fix: |
5751 | return ISD::UDIVFIX; |
5752 | case Intrinsic::sdiv_fix_sat: |
5753 | return ISD::SDIVFIXSAT; |
5754 | case Intrinsic::udiv_fix_sat: |
5755 | return ISD::UDIVFIXSAT; |
5756 | default: |
5757 | llvm_unreachable("Unhandled fixed point intrinsic")__builtin_unreachable(); |
5758 | } |
5759 | } |
5760 | |
5761 | void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I, |
5762 | const char *FunctionName) { |
5763 | assert(FunctionName && "FunctionName must not be nullptr")((void)0); |
5764 | SDValue Callee = DAG.getExternalSymbol( |
5765 | FunctionName, |
5766 | DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); |
5767 | LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall()); |
5768 | } |
5769 | |
5770 | /// Given a @llvm.call.preallocated.setup, return the corresponding |
5771 | /// preallocated call. |
5772 | static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) { |
5773 | assert(cast<CallBase>(PreallocatedSetup)((void)0) |
5774 | ->getCalledFunction()((void)0) |
5775 | ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&((void)0) |
5776 | "expected call_preallocated_setup Value")((void)0); |
5777 | for (auto *U : PreallocatedSetup->users()) { |
5778 | auto *UseCall = cast<CallBase>(U); |
5779 | const Function *Fn = UseCall->getCalledFunction(); |
5780 | if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) { |
5781 | return UseCall; |
5782 | } |
5783 | } |
5784 | llvm_unreachable("expected corresponding call to preallocated setup/arg")__builtin_unreachable(); |
5785 | } |
5786 | |
5787 | /// Lower the call to the specified intrinsic function. |
5788 | void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, |
5789 | unsigned Intrinsic) { |
5790 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
5791 | SDLoc sdl = getCurSDLoc(); |
5792 | DebugLoc dl = getCurDebugLoc(); |
5793 | SDValue Res; |
5794 | |
5795 | SDNodeFlags Flags; |
5796 | if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) |
5797 | Flags.copyFMF(*FPOp); |
5798 | |
5799 | switch (Intrinsic) { |
5800 | default: |
5801 | // By default, turn this into a target intrinsic node. |
5802 | visitTargetIntrinsic(I, Intrinsic); |
5803 | return; |
5804 | case Intrinsic::vscale: { |
5805 | match(&I, m_VScale(DAG.getDataLayout())); |
5806 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
5807 | setValue(&I, |
5808 | DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1))); |
5809 | return; |
5810 | } |
5811 | case Intrinsic::vastart: visitVAStart(I); return; |
5812 | case Intrinsic::vaend: visitVAEnd(I); return; |
5813 | case Intrinsic::vacopy: visitVACopy(I); return; |
5814 | case Intrinsic::returnaddress: |
5815 | setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, |
5816 | TLI.getPointerTy(DAG.getDataLayout()), |
5817 | getValue(I.getArgOperand(0)))); |
5818 | return; |
5819 | case Intrinsic::addressofreturnaddress: |
5820 | setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl, |
5821 | TLI.getPointerTy(DAG.getDataLayout()))); |
5822 | return; |
5823 | case Intrinsic::sponentry: |
5824 | setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl, |
5825 | TLI.getFrameIndexTy(DAG.getDataLayout()))); |
5826 | return; |
5827 | case Intrinsic::frameaddress: |
5828 | setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, |
5829 | TLI.getFrameIndexTy(DAG.getDataLayout()), |
5830 | getValue(I.getArgOperand(0)))); |
5831 | return; |
5832 | case Intrinsic::read_volatile_register: |
5833 | case Intrinsic::read_register: { |
5834 | Value *Reg = I.getArgOperand(0); |
5835 | SDValue Chain = getRoot(); |
5836 | SDValue RegName = |
5837 | DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); |
5838 | EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); |
5839 | Res = DAG.getNode(ISD::READ_REGISTER, sdl, |
5840 | DAG.getVTList(VT, MVT::Other), Chain, RegName); |
5841 | setValue(&I, Res); |
5842 | DAG.setRoot(Res.getValue(1)); |
5843 | return; |
5844 | } |
5845 | case Intrinsic::write_register: { |
5846 | Value *Reg = I.getArgOperand(0); |
5847 | Value *RegValue = I.getArgOperand(1); |
5848 | SDValue Chain = getRoot(); |
5849 | SDValue RegName = |
5850 | DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); |
5851 | DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, |
5852 | RegName, getValue(RegValue))); |
5853 | return; |
5854 | } |
5855 | case Intrinsic::memcpy: { |
5856 | const auto &MCI = cast<MemCpyInst>(I); |
5857 | SDValue Op1 = getValue(I.getArgOperand(0)); |
5858 | SDValue Op2 = getValue(I.getArgOperand(1)); |
5859 | SDValue Op3 = getValue(I.getArgOperand(2)); |
5860 | // @llvm.memcpy defines 0 and 1 to both mean no alignment. |
5861 | Align DstAlign = MCI.getDestAlign().valueOrOne(); |
5862 | Align SrcAlign = MCI.getSourceAlign().valueOrOne(); |
5863 | Align Alignment = commonAlignment(DstAlign, SrcAlign); |
5864 | bool isVol = MCI.isVolatile(); |
5865 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); |
5866 | // FIXME: Support passing different dest/src alignments to the memcpy DAG |
5867 | // node. |
5868 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); |
5869 | AAMDNodes AAInfo; |
5870 | I.getAAMetadata(AAInfo); |
5871 | SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol, |
5872 | /* AlwaysInline */ false, isTC, |
5873 | MachinePointerInfo(I.getArgOperand(0)), |
5874 | MachinePointerInfo(I.getArgOperand(1)), AAInfo); |
5875 | updateDAGForMaybeTailCall(MC); |
5876 | return; |
5877 | } |
5878 | case Intrinsic::memcpy_inline: { |
5879 | const auto &MCI = cast<MemCpyInlineInst>(I); |
5880 | SDValue Dst = getValue(I.getArgOperand(0)); |
5881 | SDValue Src = getValue(I.getArgOperand(1)); |
5882 | SDValue Size = getValue(I.getArgOperand(2)); |
5883 | assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size")((void)0); |
5884 | // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment. |
5885 | Align DstAlign = MCI.getDestAlign().valueOrOne(); |
5886 | Align SrcAlign = MCI.getSourceAlign().valueOrOne(); |
5887 | Align Alignment = commonAlignment(DstAlign, SrcAlign); |
5888 | bool isVol = MCI.isVolatile(); |
5889 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); |
5890 | // FIXME: Support passing different dest/src alignments to the memcpy DAG |
5891 | // node. |
5892 | AAMDNodes AAInfo; |
5893 | I.getAAMetadata(AAInfo); |
5894 | SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol, |
5895 | /* AlwaysInline */ true, isTC, |
5896 | MachinePointerInfo(I.getArgOperand(0)), |
5897 | MachinePointerInfo(I.getArgOperand(1)), AAInfo); |
5898 | updateDAGForMaybeTailCall(MC); |
5899 | return; |
5900 | } |
5901 | case Intrinsic::memset: { |
5902 | const auto &MSI = cast<MemSetInst>(I); |
5903 | SDValue Op1 = getValue(I.getArgOperand(0)); |
5904 | SDValue Op2 = getValue(I.getArgOperand(1)); |
5905 | SDValue Op3 = getValue(I.getArgOperand(2)); |
5906 | // @llvm.memset defines 0 and 1 to both mean no alignment. |
5907 | Align Alignment = MSI.getDestAlign().valueOrOne(); |
5908 | bool isVol = MSI.isVolatile(); |
5909 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); |
5910 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); |
5911 | AAMDNodes AAInfo; |
5912 | I.getAAMetadata(AAInfo); |
5913 | SDValue MS = DAG.getMemset(Root, sdl, Op1, Op2, Op3, Alignment, isVol, isTC, |
5914 | MachinePointerInfo(I.getArgOperand(0)), AAInfo); |
5915 | updateDAGForMaybeTailCall(MS); |
5916 | return; |
5917 | } |
5918 | case Intrinsic::memmove: { |
5919 | const auto &MMI = cast<MemMoveInst>(I); |
5920 | SDValue Op1 = getValue(I.getArgOperand(0)); |
5921 | SDValue Op2 = getValue(I.getArgOperand(1)); |
5922 | SDValue Op3 = getValue(I.getArgOperand(2)); |
5923 | // @llvm.memmove defines 0 and 1 to both mean no alignment. |
5924 | Align DstAlign = MMI.getDestAlign().valueOrOne(); |
5925 | Align SrcAlign = MMI.getSourceAlign().valueOrOne(); |
5926 | Align Alignment = commonAlignment(DstAlign, SrcAlign); |
5927 | bool isVol = MMI.isVolatile(); |
5928 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); |
5929 | // FIXME: Support passing different dest/src alignments to the memmove DAG |
5930 | // node. |
5931 | SDValue Root = isVol ? getRoot() : getMemoryRoot(); |
5932 | AAMDNodes AAInfo; |
5933 | I.getAAMetadata(AAInfo); |
5934 | SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, |
5935 | isTC, MachinePointerInfo(I.getArgOperand(0)), |
5936 | MachinePointerInfo(I.getArgOperand(1)), AAInfo); |
5937 | updateDAGForMaybeTailCall(MM); |
5938 | return; |
5939 | } |
5940 | case Intrinsic::memcpy_element_unordered_atomic: { |
5941 | const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I); |
5942 | SDValue Dst = getValue(MI.getRawDest()); |
5943 | SDValue Src = getValue(MI.getRawSource()); |
5944 | SDValue Length = getValue(MI.getLength()); |
5945 | |
5946 | unsigned DstAlign = MI.getDestAlignment(); |
5947 | unsigned SrcAlign = MI.getSourceAlignment(); |
5948 | Type *LengthTy = MI.getLength()->getType(); |
5949 | unsigned ElemSz = MI.getElementSizeInBytes(); |
5950 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); |
5951 | SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src, |
5952 | SrcAlign, Length, LengthTy, ElemSz, isTC, |
5953 | MachinePointerInfo(MI.getRawDest()), |
5954 | MachinePointerInfo(MI.getRawSource())); |
5955 | updateDAGForMaybeTailCall(MC); |
5956 | return; |
5957 | } |
5958 | case Intrinsic::memmove_element_unordered_atomic: { |
5959 | auto &MI = cast<AtomicMemMoveInst>(I); |
5960 | SDValue Dst = getValue(MI.getRawDest()); |
5961 | SDValue Src = getValue(MI.getRawSource()); |
5962 | SDValue Length = getValue(MI.getLength()); |
5963 | |
5964 | unsigned DstAlign = MI.getDestAlignment(); |
5965 | unsigned SrcAlign = MI.getSourceAlignment(); |
5966 | Type *LengthTy = MI.getLength()->getType(); |
5967 | unsigned ElemSz = MI.getElementSizeInBytes(); |
5968 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); |
5969 | SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src, |
5970 | SrcAlign, Length, LengthTy, ElemSz, isTC, |
5971 | MachinePointerInfo(MI.getRawDest()), |
5972 | MachinePointerInfo(MI.getRawSource())); |
5973 | updateDAGForMaybeTailCall(MC); |
5974 | return; |
5975 | } |
5976 | case Intrinsic::memset_element_unordered_atomic: { |
5977 | auto &MI = cast<AtomicMemSetInst>(I); |
5978 | SDValue Dst = getValue(MI.getRawDest()); |
5979 | SDValue Val = getValue(MI.getValue()); |
5980 | SDValue Length = getValue(MI.getLength()); |
5981 | |
5982 | unsigned DstAlign = MI.getDestAlignment(); |
5983 | Type *LengthTy = MI.getLength()->getType(); |
5984 | unsigned ElemSz = MI.getElementSizeInBytes(); |
5985 | bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); |
5986 | SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length, |
5987 | LengthTy, ElemSz, isTC, |
5988 | MachinePointerInfo(MI.getRawDest())); |
5989 | updateDAGForMaybeTailCall(MC); |
5990 | return; |
5991 | } |
5992 | case Intrinsic::call_preallocated_setup: { |
5993 | const CallBase *PreallocatedCall = FindPreallocatedCall(&I); |
5994 | SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); |
5995 | SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other, |
5996 | getRoot(), SrcValue); |
5997 | setValue(&I, Res); |
5998 | DAG.setRoot(Res); |
5999 | return; |
6000 | } |
6001 | case Intrinsic::call_preallocated_arg: { |
6002 | const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0)); |
6003 | SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); |
6004 | SDValue Ops[3]; |
6005 | Ops[0] = getRoot(); |
6006 | Ops[1] = SrcValue; |
6007 | Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl, |
6008 | MVT::i32); // arg index |
6009 | SDValue Res = DAG.getNode( |
6010 | ISD::PREALLOCATED_ARG, sdl, |
6011 | DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops); |
6012 | setValue(&I, Res); |
6013 | DAG.setRoot(Res.getValue(1)); |
6014 | return; |
6015 | } |
6016 | case Intrinsic::dbg_addr: |
6017 | case Intrinsic::dbg_declare: { |
6018 | // Assume dbg.addr and dbg.declare can not currently use DIArgList, i.e. |
6019 | // they are non-variadic. |
6020 | const auto &DI = cast<DbgVariableIntrinsic>(I); |
6021 | assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList")((void)0); |
6022 | DILocalVariable *Variable = DI.getVariable(); |
6023 | DIExpression *Expression = DI.getExpression(); |
6024 | dropDanglingDebugInfo(Variable, Expression); |
6025 | assert(Variable && "Missing variable")((void)0); |
6026 | LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DIdo { } while (false) |
6027 | << "\n")do { } while (false); |
6028 | // Check if address has undef value. |
6029 | const Value *Address = DI.getVariableLocationOp(0); |
6030 | if (!Address || isa<UndefValue>(Address) || |
6031 | (Address->use_empty() && !isa<Argument>(Address))) { |
6032 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { } while (false) |
6033 | << " (bad/undef/unused-arg address)\n")do { } while (false); |
6034 | return; |
6035 | } |
6036 | |
6037 | bool isParameter = Variable->isParameter() || isa<Argument>(Address); |
6038 | |
6039 | // Check if this variable can be described by a frame index, typically |
6040 | // either as a static alloca or a byval parameter. |
6041 | int FI = std::numeric_limits<int>::max(); |
6042 | if (const auto *AI = |
6043 | dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) { |
6044 | if (AI->isStaticAlloca()) { |
6045 | auto I = FuncInfo.StaticAllocaMap.find(AI); |
6046 | if (I != FuncInfo.StaticAllocaMap.end()) |
6047 | FI = I->second; |
6048 | } |
6049 | } else if (const auto *Arg = dyn_cast<Argument>( |
6050 | Address->stripInBoundsConstantOffsets())) { |
6051 | FI = FuncInfo.getArgumentFrameIndex(Arg); |
6052 | } |
6053 | |
6054 | // llvm.dbg.addr is control dependent and always generates indirect |
6055 | // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in |
6056 | // the MachineFunction variable table. |
6057 | if (FI != std::numeric_limits<int>::max()) { |
6058 | if (Intrinsic == Intrinsic::dbg_addr) { |
6059 | SDDbgValue *SDV = DAG.getFrameIndexDbgValue( |
6060 | Variable, Expression, FI, getRoot().getNode(), /*IsIndirect*/ true, |
6061 | dl, SDNodeOrder); |
6062 | DAG.AddDbgValue(SDV, isParameter); |
6063 | } else { |
6064 | LLVM_DEBUG(dbgs() << "Skipping " << DIdo { } while (false) |
6065 | << " (variable info stashed in MF side table)\n")do { } while (false); |
6066 | } |
6067 | return; |
6068 | } |
6069 | |
6070 | SDValue &N = NodeMap[Address]; |
6071 | if (!N.getNode() && isa<Argument>(Address)) |
6072 | // Check unused arguments map. |
6073 | N = UnusedArgNodeMap[Address]; |
6074 | SDDbgValue *SDV; |
6075 | if (N.getNode()) { |
6076 | if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) |
6077 | Address = BCI->getOperand(0); |
6078 | // Parameters are handled specially. |
6079 | auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); |
6080 | if (isParameter && FINode) { |
6081 | // Byval parameter. We have a frame index at this point. |
6082 | SDV = |
6083 | DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(), |
6084 | /*IsIndirect*/ true, dl, SDNodeOrder); |
6085 | } else if (isa<Argument>(Address)) { |
6086 | // Address is an argument, so try to emit its dbg value using |
6087 | // virtual register info from the FuncInfo.ValueMap. |
6088 | EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N); |
6089 | return; |
6090 | } else { |
6091 | SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), |
6092 | true, dl, SDNodeOrder); |
6093 | } |
6094 | DAG.AddDbgValue(SDV, isParameter); |
6095 | } else { |
6096 | // If Address is an argument then try to emit its dbg value using |
6097 | // virtual register info from the FuncInfo.ValueMap. |
6098 | if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, |
6099 | N)) { |
6100 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { } while (false) |
6101 | << " (could not emit func-arg dbg_value)\n")do { } while (false); |
6102 | } |
6103 | } |
6104 | return; |
6105 | } |
6106 | case Intrinsic::dbg_label: { |
6107 | const DbgLabelInst &DI = cast<DbgLabelInst>(I); |
6108 | DILabel *Label = DI.getLabel(); |
6109 | assert(Label && "Missing label")((void)0); |
6110 | |
6111 | SDDbgLabel *SDV; |
6112 | SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder); |
6113 | DAG.AddDbgLabel(SDV); |
6114 | return; |
6115 | } |
6116 | case Intrinsic::dbg_value: { |
6117 | const DbgValueInst &DI = cast<DbgValueInst>(I); |
6118 | assert(DI.getVariable() && "Missing variable")((void)0); |
6119 | |
6120 | DILocalVariable *Variable = DI.getVariable(); |
6121 | DIExpression *Expression = DI.getExpression(); |
6122 | dropDanglingDebugInfo(Variable, Expression); |
6123 | SmallVector<Value *, 4> Values(DI.getValues()); |
6124 | if (Values.empty()) |
6125 | return; |
6126 | |
6127 | if (std::count(Values.begin(), Values.end(), nullptr)) |
6128 | return; |
6129 | |
6130 | bool IsVariadic = DI.hasArgList(); |
6131 | if (!handleDebugValue(Values, Variable, Expression, dl, DI.getDebugLoc(), |
6132 | SDNodeOrder, IsVariadic)) |
6133 | addDanglingDebugInfo(&DI, dl, SDNodeOrder); |
6134 | return; |
6135 | } |
6136 | |
6137 | case Intrinsic::eh_typeid_for: { |
6138 | // Find the type id for the given typeinfo. |
6139 | GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0)); |
6140 | unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV); |
6141 | Res = DAG.getConstant(TypeID, sdl, MVT::i32); |
6142 | setValue(&I, Res); |
6143 | return; |
6144 | } |
6145 | |
6146 | case Intrinsic::eh_return_i32: |
6147 | case Intrinsic::eh_return_i64: |
6148 | DAG.getMachineFunction().setCallsEHReturn(true); |
6149 | DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, |
6150 | MVT::Other, |
6151 | getControlRoot(), |
6152 | getValue(I.getArgOperand(0)), |
6153 | getValue(I.getArgOperand(1)))); |
6154 | return; |
6155 | case Intrinsic::eh_unwind_init: |
6156 | DAG.getMachineFunction().setCallsUnwindInit(true); |
6157 | return; |
6158 | case Intrinsic::eh_dwarf_cfa: |
6159 | setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl, |
6160 | TLI.getPointerTy(DAG.g |