Bug Summary

File:src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1110, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 1 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -D PIC -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -D_RET_PROTECTOR -ret-protector -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

→
1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/FoldingSet.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Triple.h"
26#include "llvm/ADT/Twine.h"
27#include "llvm/Analysis/BlockFrequencyInfo.h"
28#include "llvm/Analysis/MemoryLocation.h"
29#include "llvm/Analysis/ProfileSummaryInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/CodeGen/FunctionLoweringInfo.h"
32#include "llvm/CodeGen/ISDOpcodes.h"
33#include "llvm/CodeGen/MachineBasicBlock.h"
34#include "llvm/CodeGen/MachineConstantPool.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineMemOperand.h"
38#include "llvm/CodeGen/RuntimeLibcalls.h"
39#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
40#include "llvm/CodeGen/SelectionDAGNodes.h"
41#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
42#include "llvm/CodeGen/TargetFrameLowering.h"
43#include "llvm/CodeGen/TargetLowering.h"
44#include "llvm/CodeGen/TargetRegisterInfo.h"
45#include "llvm/CodeGen/TargetSubtargetInfo.h"
46#include "llvm/CodeGen/ValueTypes.h"
47#include "llvm/IR/Constant.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/DebugLoc.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/GlobalValue.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Type.h"
57#include "llvm/IR/Value.h"
58#include "llvm/Support/Casting.h"
59#include "llvm/Support/CodeGen.h"
60#include "llvm/Support/Compiler.h"
61#include "llvm/Support/Debug.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/KnownBits.h"
64#include "llvm/Support/MachineValueType.h"
65#include "llvm/Support/ManagedStatic.h"
66#include "llvm/Support/MathExtras.h"
67#include "llvm/Support/Mutex.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/Transforms/Utils/SizeOpts.h"
72#include <algorithm>
73#include <cassert>
74#include <cstdint>
75#include <cstdlib>
76#include <limits>
77#include <set>
78#include <string>
79#include <utility>
80#include <vector>
81
82using namespace llvm;
83
84/// makeVTList - Return an instance of the SDVTList struct initialized with the
85/// specified members.
86static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
87 SDVTList Res = {VTs, NumVTs};
88 return Res;
89}
90
91// Default null implementations of the callbacks.
92void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
93void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
94void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
95
96void SelectionDAG::DAGNodeDeletedListener::anchor() {}
97
98#define DEBUG_TYPE"selectiondag" "selectiondag"
99
100static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
101 cl::Hidden, cl::init(true),
102 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
103
104static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
105 cl::desc("Number limit for gluing ld/st of memcpy."),
106 cl::Hidden, cl::init(0));
107
108static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
109 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);)do { } while (false);
110}
111
112//===----------------------------------------------------------------------===//
113// ConstantFPSDNode Class
114//===----------------------------------------------------------------------===//
115
116/// isExactlyValue - We don't rely on operator== working on double values, as
117/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
118/// As such, this method can be used to do an exact bit-for-bit comparison of
119/// two floating point values.
120bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
121 return getValueAPF().bitwiseIsEqual(V);
122}
123
124bool ConstantFPSDNode::isValueValidForType(EVT VT,
125 const APFloat& Val) {
126 assert(VT.isFloatingPoint() && "Can only convert between FP types")((void)0);
127
128 // convert modifies in place, so make a copy.
129 APFloat Val2 = APFloat(Val);
130 bool losesInfo;
131 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
132 APFloat::rmNearestTiesToEven,
133 &losesInfo);
134 return !losesInfo;
135}
136
137//===----------------------------------------------------------------------===//
138// ISD Namespace
139//===----------------------------------------------------------------------===//
140
141bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
142 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
143 unsigned EltSize =
144 N->getValueType(0).getVectorElementType().getSizeInBits();
145 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
146 SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize);
147 return true;
148 }
149 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
150 SplatVal = Op0->getValueAPF().bitcastToAPInt().truncOrSelf(EltSize);
151 return true;
152 }
153 }
154
155 auto *BV = dyn_cast<BuildVectorSDNode>(N);
156 if (!BV)
157 return false;
158
159 APInt SplatUndef;
160 unsigned SplatBitSize;
161 bool HasUndefs;
162 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
163 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
164 EltSize) &&
165 EltSize == SplatBitSize;
166}
167
168// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
169// specializations of the more general isConstantSplatVector()?
170
171bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
172 // Look through a bit convert.
173 while (N->getOpcode() == ISD::BITCAST)
174 N = N->getOperand(0).getNode();
175
176 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
177 APInt SplatVal;
178 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue();
179 }
180
181 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
182
183 unsigned i = 0, e = N->getNumOperands();
184
185 // Skip over all of the undef values.
186 while (i != e && N->getOperand(i).isUndef())
187 ++i;
188
189 // Do not accept an all-undef vector.
190 if (i == e) return false;
191
192 // Do not accept build_vectors that aren't all constants or which have non-~0
193 // elements. We have to be a bit careful here, as the type of the constant
194 // may not be the same as the type of the vector elements due to type
195 // legalization (the elements are promoted to a legal type for the target and
196 // a vector of a type may be legal when the base element type is not).
197 // We only want to check enough bits to cover the vector elements, because
198 // we care if the resultant vector is all ones, not whether the individual
199 // constants are.
200 SDValue NotZero = N->getOperand(i);
201 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
202 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
203 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
204 return false;
205 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
206 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
207 return false;
208 } else
209 return false;
210
211 // Okay, we have at least one ~0 value, check to see if the rest match or are
212 // undefs. Even with the above element type twiddling, this should be OK, as
213 // the same type legalization should have applied to all the elements.
214 for (++i; i != e; ++i)
215 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
216 return false;
217 return true;
218}
219
220bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
221 // Look through a bit convert.
222 while (N->getOpcode() == ISD::BITCAST)
223 N = N->getOperand(0).getNode();
224
225 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
226 APInt SplatVal;
227 return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue();
228 }
229
230 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
231
232 bool IsAllUndef = true;
233 for (const SDValue &Op : N->op_values()) {
234 if (Op.isUndef())
235 continue;
236 IsAllUndef = false;
237 // Do not accept build_vectors that aren't all constants or which have non-0
238 // elements. We have to be a bit careful here, as the type of the constant
239 // may not be the same as the type of the vector elements due to type
240 // legalization (the elements are promoted to a legal type for the target
241 // and a vector of a type may be legal when the base element type is not).
242 // We only want to check enough bits to cover the vector elements, because
243 // we care if the resultant vector is all zeros, not whether the individual
244 // constants are.
245 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
246 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
247 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
248 return false;
249 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
250 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
251 return false;
252 } else
253 return false;
254 }
255
256 // Do not accept an all-undef vector.
257 if (IsAllUndef)
258 return false;
259 return true;
260}
261
262bool ISD::isBuildVectorAllOnes(const SDNode *N) {
263 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
264}
265
266bool ISD::isBuildVectorAllZeros(const SDNode *N) {
267 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
268}
269
270bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
271 if (N->getOpcode() != ISD::BUILD_VECTOR)
272 return false;
273
274 for (const SDValue &Op : N->op_values()) {
275 if (Op.isUndef())
276 continue;
277 if (!isa<ConstantSDNode>(Op))
278 return false;
279 }
280 return true;
281}
282
283bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
284 if (N->getOpcode() != ISD::BUILD_VECTOR)
285 return false;
286
287 for (const SDValue &Op : N->op_values()) {
288 if (Op.isUndef())
289 continue;
290 if (!isa<ConstantFPSDNode>(Op))
291 return false;
292 }
293 return true;
294}
295
296bool ISD::allOperandsUndef(const SDNode *N) {
297 // Return false if the node has no operands.
298 // This is "logically inconsistent" with the definition of "all" but
299 // is probably the desired behavior.
300 if (N->getNumOperands() == 0)
301 return false;
302 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
303}
304
305bool ISD::matchUnaryPredicate(SDValue Op,
306 std::function<bool(ConstantSDNode *)> Match,
307 bool AllowUndefs) {
308 // FIXME: Add support for scalar UNDEF cases?
309 if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
3
←
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
16
←
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
17
←
Assuming 'Cst' is null
→
18
←
Taking false branch
→
310 return Match(Cst);
311
312 // FIXME: Add support for vector UNDEF cases?
313 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
19
←
Calling 'SDValue::getOpcode'
→
314 ISD::SPLAT_VECTOR != Op.getOpcode())
315 return false;
316
317 EVT SVT = Op.getValueType().getScalarType();
318 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
319 if (AllowUndefs && Op.getOperand(i).isUndef()) {
320 if (!Match(nullptr))
321 return false;
322 continue;
323 }
324
325 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
326 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
327 return false;
328 }
329 return true;
330}
331
332bool ISD::matchBinaryPredicate(
333 SDValue LHS, SDValue RHS,
334 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
335 bool AllowUndefs, bool AllowTypeMismatch) {
336 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
337 return false;
338
339 // TODO: Add support for scalar UNDEF cases?
340 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
341 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
342 return Match(LHSCst, RHSCst);
343
344 // TODO: Add support for vector UNDEF cases?
345 if (LHS.getOpcode() != RHS.getOpcode() ||
346 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
347 LHS.getOpcode() != ISD::SPLAT_VECTOR))
348 return false;
349
350 EVT SVT = LHS.getValueType().getScalarType();
351 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
352 SDValue LHSOp = LHS.getOperand(i);
353 SDValue RHSOp = RHS.getOperand(i);
354 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
355 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
356 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
357 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
358 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
359 return false;
360 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
361 LHSOp.getValueType() != RHSOp.getValueType()))
362 return false;
363 if (!Match(LHSCst, RHSCst))
364 return false;
365 }
366 return true;
367}
368
369ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
370 switch (VecReduceOpcode) {
371 default:
372 llvm_unreachable("Expected VECREDUCE opcode")__builtin_unreachable();
373 case ISD::VECREDUCE_FADD:
374 case ISD::VECREDUCE_SEQ_FADD:
375 return ISD::FADD;
376 case ISD::VECREDUCE_FMUL:
377 case ISD::VECREDUCE_SEQ_FMUL:
378 return ISD::FMUL;
379 case ISD::VECREDUCE_ADD:
380 return ISD::ADD;
381 case ISD::VECREDUCE_MUL:
382 return ISD::MUL;
383 case ISD::VECREDUCE_AND:
384 return ISD::AND;
385 case ISD::VECREDUCE_OR:
386 return ISD::OR;
387 case ISD::VECREDUCE_XOR:
388 return ISD::XOR;
389 case ISD::VECREDUCE_SMAX:
390 return ISD::SMAX;
391 case ISD::VECREDUCE_SMIN:
392 return ISD::SMIN;
393 case ISD::VECREDUCE_UMAX:
394 return ISD::UMAX;
395 case ISD::VECREDUCE_UMIN:
396 return ISD::UMIN;
397 case ISD::VECREDUCE_FMAX:
398 return ISD::FMAXNUM;
399 case ISD::VECREDUCE_FMIN:
400 return ISD::FMINNUM;
401 }
402}
403
404bool ISD::isVPOpcode(unsigned Opcode) {
405 switch (Opcode) {
406 default:
407 return false;
408#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
409 case ISD::SDOPC: \
410 return true;
411#include "llvm/IR/VPIntrinsics.def"
412 }
413}
414
415/// The operand position of the vector mask.
416Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
417 switch (Opcode) {
418 default:
419 return None;
420#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, ...) \
421 case ISD::SDOPC: \
422 return MASKPOS;
423#include "llvm/IR/VPIntrinsics.def"
424 }
425}
426
427/// The operand position of the explicit vector length parameter.
428Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
429 switch (Opcode) {
430 default:
431 return None;
432#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
433 case ISD::SDOPC: \
434 return EVLPOS;
435#include "llvm/IR/VPIntrinsics.def"
436 }
437}
438
439ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
440 switch (ExtType) {
441 case ISD::EXTLOAD:
442 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
443 case ISD::SEXTLOAD:
444 return ISD::SIGN_EXTEND;
445 case ISD::ZEXTLOAD:
446 return ISD::ZERO_EXTEND;
447 default:
448 break;
449 }
450
451 llvm_unreachable("Invalid LoadExtType")__builtin_unreachable();
452}
453
454ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
455 // To perform this operation, we just need to swap the L and G bits of the
456 // operation.
457 unsigned OldL = (Operation >> 2) & 1;
458 unsigned OldG = (Operation >> 1) & 1;
459 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
460 (OldL << 1) | // New G bit
461 (OldG << 2)); // New L bit.
462}
463
464static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
465 unsigned Operation = Op;
466 if (isIntegerLike)
467 Operation ^= 7; // Flip L, G, E bits, but not U.
468 else
469 Operation ^= 15; // Flip all of the condition bits.
470
471 if (Operation > ISD::SETTRUE2)
472 Operation &= ~8; // Don't let N and U bits get set.
473
474 return ISD::CondCode(Operation);
475}
476
477ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
478 return getSetCCInverseImpl(Op, Type.isInteger());
479}
480
481ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
482 bool isIntegerLike) {
483 return getSetCCInverseImpl(Op, isIntegerLike);
484}
485
486/// For an integer comparison, return 1 if the comparison is a signed operation
487/// and 2 if the result is an unsigned comparison. Return zero if the operation
488/// does not depend on the sign of the input (setne and seteq).
489static int isSignedOp(ISD::CondCode Opcode) {
490 switch (Opcode) {
491 default: llvm_unreachable("Illegal integer setcc operation!")__builtin_unreachable();
492 case ISD::SETEQ:
493 case ISD::SETNE: return 0;
494 case ISD::SETLT:
495 case ISD::SETLE:
496 case ISD::SETGT:
497 case ISD::SETGE: return 1;
498 case ISD::SETULT:
499 case ISD::SETULE:
500 case ISD::SETUGT:
501 case ISD::SETUGE: return 2;
502 }
503}
504
505ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
506 EVT Type) {
507 bool IsInteger = Type.isInteger();
508 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
509 // Cannot fold a signed integer setcc with an unsigned integer setcc.
510 return ISD::SETCC_INVALID;
511
512 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
513
514 // If the N and U bits get set, then the resultant comparison DOES suddenly
515 // care about orderedness, and it is true when ordered.
516 if (Op > ISD::SETTRUE2)
517 Op &= ~16; // Clear the U bit if the N bit is set.
518
519 // Canonicalize illegal integer setcc's.
520 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
521 Op = ISD::SETNE;
522
523 return ISD::CondCode(Op);
524}
525
526ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
527 EVT Type) {
528 bool IsInteger = Type.isInteger();
529 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
530 // Cannot fold a signed setcc with an unsigned setcc.
531 return ISD::SETCC_INVALID;
532
533 // Combine all of the condition bits.
534 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
535
536 // Canonicalize illegal integer setcc's.
537 if (IsInteger) {
538 switch (Result) {
539 default: break;
540 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
541 case ISD::SETOEQ: // SETEQ & SETU[LG]E
542 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
543 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
544 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
545 }
546 }
547
548 return Result;
549}
550
551//===----------------------------------------------------------------------===//
552// SDNode Profile Support
553//===----------------------------------------------------------------------===//
554
555/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
556static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
557 ID.AddInteger(OpC);
558}
559
560/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
561/// solely with their pointer.
562static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
563 ID.AddPointer(VTList.VTs);
564}
565
566/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
567static void AddNodeIDOperands(FoldingSetNodeID &ID,
568 ArrayRef<SDValue> Ops) {
569 for (auto& Op : Ops) {
570 ID.AddPointer(Op.getNode());
571 ID.AddInteger(Op.getResNo());
572 }
573}
574
575/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
576static void AddNodeIDOperands(FoldingSetNodeID &ID,
577 ArrayRef<SDUse> Ops) {
578 for (auto& Op : Ops) {
579 ID.AddPointer(Op.getNode());
580 ID.AddInteger(Op.getResNo());
581 }
582}
583
584static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
585 SDVTList VTList, ArrayRef<SDValue> OpList) {
586 AddNodeIDOpcode(ID, OpC);
587 AddNodeIDValueTypes(ID, VTList);
588 AddNodeIDOperands(ID, OpList);
589}
590
591/// If this is an SDNode with special info, add this info to the NodeID data.
592static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
593 switch (N->getOpcode()) {
594 case ISD::TargetExternalSymbol:
595 case ISD::ExternalSymbol:
596 case ISD::MCSymbol:
597 llvm_unreachable("Should only be used on nodes with operands")__builtin_unreachable();
598 default: break; // Normal nodes don't need extra info.
599 case ISD::TargetConstant:
600 case ISD::Constant: {
601 const ConstantSDNode *C = cast<ConstantSDNode>(N);
602 ID.AddPointer(C->getConstantIntValue());
603 ID.AddBoolean(C->isOpaque());
604 break;
605 }
606 case ISD::TargetConstantFP:
607 case ISD::ConstantFP:
608 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
609 break;
610 case ISD::TargetGlobalAddress:
611 case ISD::GlobalAddress:
612 case ISD::TargetGlobalTLSAddress:
613 case ISD::GlobalTLSAddress: {
614 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
615 ID.AddPointer(GA->getGlobal());
616 ID.AddInteger(GA->getOffset());
617 ID.AddInteger(GA->getTargetFlags());
618 break;
619 }
620 case ISD::BasicBlock:
621 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
622 break;
623 case ISD::Register:
624 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
625 break;
626 case ISD::RegisterMask:
627 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
628 break;
629 case ISD::SRCVALUE:
630 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
631 break;
632 case ISD::FrameIndex:
633 case ISD::TargetFrameIndex:
634 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
635 break;
636 case ISD::LIFETIME_START:
637 case ISD::LIFETIME_END:
638 if (cast<LifetimeSDNode>(N)->hasOffset()) {
639 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
640 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
641 }
642 break;
643 case ISD::PSEUDO_PROBE:
644 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
645 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
646 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
647 break;
648 case ISD::JumpTable:
649 case ISD::TargetJumpTable:
650 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
651 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
652 break;
653 case ISD::ConstantPool:
654 case ISD::TargetConstantPool: {
655 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
656 ID.AddInteger(CP->getAlign().value());
657 ID.AddInteger(CP->getOffset());
658 if (CP->isMachineConstantPoolEntry())
659 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
660 else
661 ID.AddPointer(CP->getConstVal());
662 ID.AddInteger(CP->getTargetFlags());
663 break;
664 }
665 case ISD::TargetIndex: {
666 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
667 ID.AddInteger(TI->getIndex());
668 ID.AddInteger(TI->getOffset());
669 ID.AddInteger(TI->getTargetFlags());
670 break;
671 }
672 case ISD::LOAD: {
673 const LoadSDNode *LD = cast<LoadSDNode>(N);
674 ID.AddInteger(LD->getMemoryVT().getRawBits());
675 ID.AddInteger(LD->getRawSubclassData());
676 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
677 break;
678 }
679 case ISD::STORE: {
680 const StoreSDNode *ST = cast<StoreSDNode>(N);
681 ID.AddInteger(ST->getMemoryVT().getRawBits());
682 ID.AddInteger(ST->getRawSubclassData());
683 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
684 break;
685 }
686 case ISD::MLOAD: {
687 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
688 ID.AddInteger(MLD->getMemoryVT().getRawBits());
689 ID.AddInteger(MLD->getRawSubclassData());
690 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
691 break;
692 }
693 case ISD::MSTORE: {
694 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
695 ID.AddInteger(MST->getMemoryVT().getRawBits());
696 ID.AddInteger(MST->getRawSubclassData());
697 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
698 break;
699 }
700 case ISD::MGATHER: {
701 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
702 ID.AddInteger(MG->getMemoryVT().getRawBits());
703 ID.AddInteger(MG->getRawSubclassData());
704 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
705 break;
706 }
707 case ISD::MSCATTER: {
708 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
709 ID.AddInteger(MS->getMemoryVT().getRawBits());
710 ID.AddInteger(MS->getRawSubclassData());
711 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
712 break;
713 }
714 case ISD::ATOMIC_CMP_SWAP:
715 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
716 case ISD::ATOMIC_SWAP:
717 case ISD::ATOMIC_LOAD_ADD:
718 case ISD::ATOMIC_LOAD_SUB:
719 case ISD::ATOMIC_LOAD_AND:
720 case ISD::ATOMIC_LOAD_CLR:
721 case ISD::ATOMIC_LOAD_OR:
722 case ISD::ATOMIC_LOAD_XOR:
723 case ISD::ATOMIC_LOAD_NAND:
724 case ISD::ATOMIC_LOAD_MIN:
725 case ISD::ATOMIC_LOAD_MAX:
726 case ISD::ATOMIC_LOAD_UMIN:
727 case ISD::ATOMIC_LOAD_UMAX:
728 case ISD::ATOMIC_LOAD:
729 case ISD::ATOMIC_STORE: {
730 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
731 ID.AddInteger(AT->getMemoryVT().getRawBits());
732 ID.AddInteger(AT->getRawSubclassData());
733 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
734 break;
735 }
736 case ISD::PREFETCH: {
737 const MemSDNode *PF = cast<MemSDNode>(N);
738 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
739 break;
740 }
741 case ISD::VECTOR_SHUFFLE: {
742 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
743 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
744 i != e; ++i)
745 ID.AddInteger(SVN->getMaskElt(i));
746 break;
747 }
748 case ISD::TargetBlockAddress:
749 case ISD::BlockAddress: {
750 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
751 ID.AddPointer(BA->getBlockAddress());
752 ID.AddInteger(BA->getOffset());
753 ID.AddInteger(BA->getTargetFlags());
754 break;
755 }
756 } // end switch (N->getOpcode())
757
758 // Target specific memory nodes could also have address spaces to check.
759 if (N->isTargetMemoryOpcode())
760 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
761}
762
763/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
764/// data.
765static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
766 AddNodeIDOpcode(ID, N->getOpcode());
767 // Add the return value info.
768 AddNodeIDValueTypes(ID, N->getVTList());
769 // Add the operand info.
770 AddNodeIDOperands(ID, N->ops());
771
772 // Handle SDNode leafs with special info.
773 AddNodeIDCustom(ID, N);
774}
775
776//===----------------------------------------------------------------------===//
777// SelectionDAG Class
778//===----------------------------------------------------------------------===//
779
780/// doNotCSE - Return true if CSE should not be performed for this node.
781static bool doNotCSE(SDNode *N) {
782 if (N->getValueType(0) == MVT::Glue)
783 return true; // Never CSE anything that produces a flag.
784
785 switch (N->getOpcode()) {
786 default: break;
787 case ISD::HANDLENODE:
788 case ISD::EH_LABEL:
789 return true; // Never CSE these nodes.
790 }
791
792 // Check that remaining values produced are not flags.
793 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
794 if (N->getValueType(i) == MVT::Glue)
795 return true; // Never CSE anything that produces a flag.
796
797 return false;
798}
799
800/// RemoveDeadNodes - This method deletes all unreachable nodes in the
801/// SelectionDAG.
802void SelectionDAG::RemoveDeadNodes() {
803 // Create a dummy node (which is not added to allnodes), that adds a reference
804 // to the root node, preventing it from being deleted.
805 HandleSDNode Dummy(getRoot());
806
807 SmallVector<SDNode*, 128> DeadNodes;
808
809 // Add all obviously-dead nodes to the DeadNodes worklist.
810 for (SDNode &Node : allnodes())
811 if (Node.use_empty())
812 DeadNodes.push_back(&Node);
813
814 RemoveDeadNodes(DeadNodes);
815
816 // If the root changed (e.g. it was a dead load, update the root).
817 setRoot(Dummy.getValue());
818}
819
820/// RemoveDeadNodes - This method deletes the unreachable nodes in the
821/// given list, and any nodes that become unreachable as a result.
822void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
823
824 // Process the worklist, deleting the nodes and adding their uses to the
825 // worklist.
826 while (!DeadNodes.empty()) {
827 SDNode *N = DeadNodes.pop_back_val();
828 // Skip to next node if we've already managed to delete the node. This could
829 // happen if replacing a node causes a node previously added to the node to
830 // be deleted.
831 if (N->getOpcode() == ISD::DELETED_NODE)
832 continue;
833
834 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
835 DUL->NodeDeleted(N, nullptr);
836
837 // Take the node out of the appropriate CSE map.
838 RemoveNodeFromCSEMaps(N);
839
840 // Next, brutally remove the operand list. This is safe to do, as there are
841 // no cycles in the graph.
842 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
843 SDUse &Use = *I++;
844 SDNode *Operand = Use.getNode();
845 Use.set(SDValue());
846
847 // Now that we removed this operand, see if there are no uses of it left.
848 if (Operand->use_empty())
849 DeadNodes.push_back(Operand);
850 }
851
852 DeallocateNode(N);
853 }
854}
855
856void SelectionDAG::RemoveDeadNode(SDNode *N){
857 SmallVector<SDNode*, 16> DeadNodes(1, N);
858
859 // Create a dummy node that adds a reference to the root node, preventing
860 // it from being deleted. (This matters if the root is an operand of the
861 // dead node.)
862 HandleSDNode Dummy(getRoot());
863
864 RemoveDeadNodes(DeadNodes);
865}
866
867void SelectionDAG::DeleteNode(SDNode *N) {
868 // First take this out of the appropriate CSE map.
869 RemoveNodeFromCSEMaps(N);
870
871 // Finally, remove uses due to operands of this node, remove from the
872 // AllNodes list, and delete the node.
873 DeleteNodeNotInCSEMaps(N);
874}
875
876void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
877 assert(N->getIterator() != AllNodes.begin() &&((void)0)
878 "Cannot delete the entry node!")((void)0);
879 assert(N->use_empty() && "Cannot delete a node that is not dead!")((void)0);
880
881 // Drop all of the operands and decrement used node's use counts.
882 N->DropOperands();
883
884 DeallocateNode(N);
885}
886
887void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
888 assert(!(V->isVariadic() && isParameter))((void)0);
889 if (isParameter)
890 ByvalParmDbgValues.push_back(V);
891 else
892 DbgValues.push_back(V);
893 for (const SDNode *Node : V->getSDNodes())
894 if (Node)
895 DbgValMap[Node].push_back(V);
896}
897
898void SDDbgInfo::erase(const SDNode *Node) {
899 DbgValMapType::iterator I = DbgValMap.find(Node);
900 if (I == DbgValMap.end())
901 return;
902 for (auto &Val: I->second)
903 Val->setIsInvalidated();
904 DbgValMap.erase(I);
905}
906
907void SelectionDAG::DeallocateNode(SDNode *N) {
908 // If we have operands, deallocate them.
909 removeOperands(N);
910
911 NodeAllocator.Deallocate(AllNodes.remove(N));
912
913 // Set the opcode to DELETED_NODE to help catch bugs when node
914 // memory is reallocated.
915 // FIXME: There are places in SDag that have grown a dependency on the opcode
916 // value in the released node.
917 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
918 N->NodeType = ISD::DELETED_NODE;
919
920 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
921 // them and forget about that node.
922 DbgInfo->erase(N);
923}
924
925#ifndef NDEBUG1
926/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
927static void VerifySDNode(SDNode *N) {
928 switch (N->getOpcode()) {
929 default:
930 break;
931 case ISD::BUILD_PAIR: {
932 EVT VT = N->getValueType(0);
933 assert(N->getNumValues() == 1 && "Too many results!")((void)0);
934 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&((void)0)
935 "Wrong return type!")((void)0);
936 assert(N->getNumOperands() == 2 && "Wrong number of operands!")((void)0);
937 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&((void)0)
938 "Mismatched operand types!")((void)0);
939 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&((void)0)
940 "Wrong operand type!")((void)0);
941 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&((void)0)
942 "Wrong return type size")((void)0);
943 break;
944 }
945 case ISD::BUILD_VECTOR: {
946 assert(N->getNumValues() == 1 && "Too many results!")((void)0);
947 assert(N->getValueType(0).isVector() && "Wrong return type!")((void)0);
948 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&((void)0)
949 "Wrong number of operands!")((void)0);
950 EVT EltVT = N->getValueType(0).getVectorElementType();
951 for (const SDUse &Op : N->ops()) {
952 assert((Op.getValueType() == EltVT ||((void)0)
953 (EltVT.isInteger() && Op.getValueType().isInteger() &&((void)0)
954 EltVT.bitsLE(Op.getValueType()))) &&((void)0)
955 "Wrong operand type!")((void)0);
956 assert(Op.getValueType() == N->getOperand(0).getValueType() &&((void)0)
957 "Operands must all have the same type")((void)0);
958 }
959 break;
960 }
961 }
962}
963#endif // NDEBUG
964
965/// Insert a newly allocated node into the DAG.
966///
967/// Handles insertion into the all nodes list and CSE map, as well as
968/// verification and other common operations when a new node is allocated.
969void SelectionDAG::InsertNode(SDNode *N) {
970 AllNodes.push_back(N);
971#ifndef NDEBUG1
972 N->PersistentId = NextPersistentId++;
973 VerifySDNode(N);
974#endif
975 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
976 DUL->NodeInserted(N);
977}
978
979/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
980/// correspond to it. This is useful when we're about to delete or repurpose
981/// the node. We don't want future request for structurally identical nodes
982/// to return N anymore.
983bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
984 bool Erased = false;
985 switch (N->getOpcode()) {
986 case ISD::HANDLENODE: return false; // noop.
987 case ISD::CONDCODE:
988 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&((void)0)
989 "Cond code doesn't exist!")((void)0);
990 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
991 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
992 break;
993 case ISD::ExternalSymbol:
994 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
995 break;
996 case ISD::TargetExternalSymbol: {
997 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
998 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
999 ESN->getSymbol(), ESN->getTargetFlags()));
1000 break;
1001 }
1002 case ISD::MCSymbol: {
1003 auto *MCSN = cast<MCSymbolSDNode>(N);
1004 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1005 break;
1006 }
1007 case ISD::VALUETYPE: {
1008 EVT VT = cast<VTSDNode>(N)->getVT();
1009 if (VT.isExtended()) {
1010 Erased = ExtendedValueTypeNodes.erase(VT);
1011 } else {
1012 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1013 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1014 }
1015 break;
1016 }
1017 default:
1018 // Remove it from the CSE Map.
1019 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!")((void)0);
1020 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!")((void)0);
1021 Erased = CSEMap.RemoveNode(N);
1022 break;
1023 }
1024#ifndef NDEBUG1
1025 // Verify that the node was actually in one of the CSE maps, unless it has a
1026 // flag result (which cannot be CSE'd) or is one of the special cases that are
1027 // not subject to CSE.
1028 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1029 !N->isMachineOpcode() && !doNotCSE(N)) {
1030 N->dump(this);
1031 dbgs() << "\n";
1032 llvm_unreachable("Node is not in map!")__builtin_unreachable();
1033 }
1034#endif
1035 return Erased;
1036}
1037
1038/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1039/// maps and modified in place. Add it back to the CSE maps, unless an identical
1040/// node already exists, in which case transfer all its users to the existing
1041/// node. This transfer can potentially trigger recursive merging.
1042void
1043SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1044 // For node types that aren't CSE'd, just act as if no identical node
1045 // already exists.
1046 if (!doNotCSE(N)) {
1047 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1048 if (Existing != N) {
1049 // If there was already an existing matching node, use ReplaceAllUsesWith
1050 // to replace the dead one with the existing one. This can cause
1051 // recursive merging of other unrelated nodes down the line.
1052 ReplaceAllUsesWith(N, Existing);
1053
1054 // N is now dead. Inform the listeners and delete it.
1055 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1056 DUL->NodeDeleted(N, Existing);
1057 DeleteNodeNotInCSEMaps(N);
1058 return;
1059 }
1060 }
1061
1062 // If the node doesn't already exist, we updated it. Inform listeners.
1063 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1064 DUL->NodeUpdated(N);
1065}
1066
1067/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1068/// were replaced with those specified. If this node is never memoized,
1069/// return null, otherwise return a pointer to the slot it would take. If a
1070/// node already exists with these operands, the slot will be non-null.
1071SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1072 void *&InsertPos) {
1073 if (doNotCSE(N))
1074 return nullptr;
1075
1076 SDValue Ops[] = { Op };
1077 FoldingSetNodeID ID;
1078 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1079 AddNodeIDCustom(ID, N);
1080 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1081 if (Node)
1082 Node->intersectFlagsWith(N->getFlags());
1083 return Node;
1084}
1085
1086/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1087/// were replaced with those specified. If this node is never memoized,
1088/// return null, otherwise return a pointer to the slot it would take. If a
1089/// node already exists with these operands, the slot will be non-null.
1090SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1091 SDValue Op1, SDValue Op2,
1092 void *&InsertPos) {
1093 if (doNotCSE(N))
1094 return nullptr;
1095
1096 SDValue Ops[] = { Op1, Op2 };
1097 FoldingSetNodeID ID;
1098 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1099 AddNodeIDCustom(ID, N);
1100 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1101 if (Node)
1102 Node->intersectFlagsWith(N->getFlags());
1103 return Node;
1104}
1105
1106/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1107/// were replaced with those specified. If this node is never memoized,
1108/// return null, otherwise return a pointer to the slot it would take. If a
1109/// node already exists with these operands, the slot will be non-null.
1110SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1111 void *&InsertPos) {
1112 if (doNotCSE(N))
1113 return nullptr;
1114
1115 FoldingSetNodeID ID;
1116 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1117 AddNodeIDCustom(ID, N);
1118 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1119 if (Node)
1120 Node->intersectFlagsWith(N->getFlags());
1121 return Node;
1122}
1123
1124Align SelectionDAG::getEVTAlign(EVT VT) const {
1125 Type *Ty = VT == MVT::iPTR ?
1126 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
1127 VT.getTypeForEVT(*getContext());
1128
1129 return getDataLayout().getABITypeAlign(Ty);
1130}
1131
1132// EntryNode could meaningfully have debug info if we can find it...
1133SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
1134 : TM(tm), OptLevel(OL),
1135 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
1136 Root(getEntryNode()) {
1137 InsertNode(&EntryNode);
1138 DbgInfo = new SDDbgInfo();
1139}
1140
1141void SelectionDAG::init(MachineFunction &NewMF,
1142 OptimizationRemarkEmitter &NewORE,
1143 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
1144 LegacyDivergenceAnalysis * Divergence,
1145 ProfileSummaryInfo *PSIin,
1146 BlockFrequencyInfo *BFIin) {
1147 MF = &NewMF;
1148 SDAGISelPass = PassPtr;
1149 ORE = &NewORE;
1150 TLI = getSubtarget().getTargetLowering();
1151 TSI = getSubtarget().getSelectionDAGInfo();
1152 LibInfo = LibraryInfo;
1153 Context = &MF->getFunction().getContext();
1154 DA = Divergence;
1155 PSI = PSIin;
1156 BFI = BFIin;
1157}
1158
1159SelectionDAG::~SelectionDAG() {
1160 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners")((void)0);
1161 allnodes_clear();
1162 OperandRecycler.clear(OperandAllocator);
1163 delete DbgInfo;
1164}
1165
1166bool SelectionDAG::shouldOptForSize() const {
1167 return MF->getFunction().hasOptSize() ||
1168 llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1169}
1170
1171void SelectionDAG::allnodes_clear() {
1172 assert(&*AllNodes.begin() == &EntryNode)((void)0);
1173 AllNodes.remove(AllNodes.begin());
1174 while (!AllNodes.empty())
1175 DeallocateNode(&AllNodes.front());
1176#ifndef NDEBUG1
1177 NextPersistentId = 0;
1178#endif
1179}
1180
1181SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1182 void *&InsertPos) {
1183 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1184 if (N) {
1185 switch (N->getOpcode()) {
1186 default: break;
1187 case ISD::Constant:
1188 case ISD::ConstantFP:
1189 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "__builtin_unreachable()
1190 "debug location. Use another overload.")__builtin_unreachable();
1191 }
1192 }
1193 return N;
1194}
1195
1196SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1197 const SDLoc &DL, void *&InsertPos) {
1198 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1199 if (N) {
1200 switch (N->getOpcode()) {
1201 case ISD::Constant:
1202 case ISD::ConstantFP:
1203 // Erase debug location from the node if the node is used at several
1204 // different places. Do not propagate one location to all uses as it
1205 // will cause a worse single stepping debugging experience.
1206 if (N->getDebugLoc() != DL.getDebugLoc())
1207 N->setDebugLoc(DebugLoc());
1208 break;
1209 default:
1210 // When the node's point of use is located earlier in the instruction
1211 // sequence than its prior point of use, update its debug info to the
1212 // earlier location.
1213 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1214 N->setDebugLoc(DL.getDebugLoc());
1215 break;
1216 }
1217 }
1218 return N;
1219}
1220
1221void SelectionDAG::clear() {
1222 allnodes_clear();
1223 OperandRecycler.clear(OperandAllocator);
1224 OperandAllocator.Reset();
1225 CSEMap.clear();
1226
1227 ExtendedValueTypeNodes.clear();
1228 ExternalSymbols.clear();
1229 TargetExternalSymbols.clear();
1230 MCSymbols.clear();
1231 SDCallSiteDbgInfo.clear();
1232 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1233 static_cast<CondCodeSDNode*>(nullptr));
1234 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1235 static_cast<SDNode*>(nullptr));
1236
1237 EntryNode.UseList = nullptr;
1238 InsertNode(&EntryNode);
1239 Root = getEntryNode();
1240 DbgInfo->clear();
1241}
1242
1243SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1244 return VT.bitsGT(Op.getValueType())
1245 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1246 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
1247}
1248
1249std::pair<SDValue, SDValue>
1250SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1251 const SDLoc &DL, EVT VT) {
1252 assert(!VT.bitsEq(Op.getValueType()) &&((void)0)
1253 "Strict no-op FP extend/round not allowed.")((void)0);
1254 SDValue Res =
1255 VT.bitsGT(Op.getValueType())
1256 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1257 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1258 {Chain, Op, getIntPtrConstant(0, DL)});
1259
1260 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1261}
1262
1263SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1264 return VT.bitsGT(Op.getValueType()) ?
1265 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1266 getNode(ISD::TRUNCATE, DL, VT, Op);
1267}
1268
1269SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1270 return VT.bitsGT(Op.getValueType()) ?
1271 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1272 getNode(ISD::TRUNCATE, DL, VT, Op);
1273}
1274
1275SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1276 return VT.bitsGT(Op.getValueType()) ?
1277 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1278 getNode(ISD::TRUNCATE, DL, VT, Op);
1279}
1280
1281SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1282 EVT OpVT) {
1283 if (VT.bitsLE(Op.getValueType()))
1284 return getNode(ISD::TRUNCATE, SL, VT, Op);
1285
1286 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1287 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1288}
1289
1290SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1291 EVT OpVT = Op.getValueType();
1292 assert(VT.isInteger() && OpVT.isInteger() &&((void)0)
1293 "Cannot getZeroExtendInReg FP types")((void)0);
1294 assert(VT.isVector() == OpVT.isVector() &&((void)0)
1295 "getZeroExtendInReg type should be vector iff the operand "((void)0)
1296 "type is vector!")((void)0);
1297 assert((!VT.isVector() ||((void)0)
1298 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&((void)0)
1299 "Vector element counts must match in getZeroExtendInReg")((void)0);
1300 assert(VT.bitsLE(OpVT) && "Not extending!")((void)0);
1301 if (OpVT == VT)
1302 return Op;
1303 APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),
1304 VT.getScalarSizeInBits());
1305 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1306}
1307
1308SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1309 // Only unsigned pointer semantics are supported right now. In the future this
1310 // might delegate to TLI to check pointer signedness.
1311 return getZExtOrTrunc(Op, DL, VT);
1312}
1313
1314SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1315 // Only unsigned pointer semantics are supported right now. In the future this
1316 // might delegate to TLI to check pointer signedness.
1317 return getZeroExtendInReg(Op, DL, VT);
1318}
1319
1320/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1321SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1322 EVT EltVT = VT.getScalarType();
1323 SDValue NegOne =
1324 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1325 return getNode(ISD::XOR, DL, VT, Val, NegOne);
1326}
1327
1328SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1329 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1330 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1331}
1332
1333SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1334 EVT OpVT) {
1335 if (!V)
1336 return getConstant(0, DL, VT);
1337
1338 switch (TLI->getBooleanContents(OpVT)) {
1339 case TargetLowering::ZeroOrOneBooleanContent:
1340 case TargetLowering::UndefinedBooleanContent:
1341 return getConstant(1, DL, VT);
1342 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1343 return getAllOnesConstant(DL, VT);
1344 }
1345 llvm_unreachable("Unexpected boolean content enum!")__builtin_unreachable();
1346}
1347
1348SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1349 bool isT, bool isO) {
1350 EVT EltVT = VT.getScalarType();
1351 assert((EltVT.getSizeInBits() >= 64 ||((void)0)
1352 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&((void)0)
1353 "getConstant with a uint64_t value that doesn't fit in the type!")((void)0);
1354 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1355}
1356
1357SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1358 bool isT, bool isO) {
1359 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1360}
1361
1362SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1363 EVT VT, bool isT, bool isO) {
1364 assert(VT.isInteger() && "Cannot create FP integer constant!")((void)0);
1365
1366 EVT EltVT = VT.getScalarType();
1367 const ConstantInt *Elt = &Val;
1368
1369 // In some cases the vector type is legal but the element type is illegal and
1370 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1371 // inserted value (the type does not need to match the vector element type).
1372 // Any extra bits introduced will be truncated away.
1373 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1374 TargetLowering::TypePromoteInteger) {
1375 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1376 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1377 Elt = ConstantInt::get(*getContext(), NewVal);
1378 }
1379 // In other cases the element type is illegal and needs to be expanded, for
1380 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1381 // the value into n parts and use a vector type with n-times the elements.
1382 // Then bitcast to the type requested.
1383 // Legalizing constants too early makes the DAGCombiner's job harder so we
1384 // only legalize if the DAG tells us we must produce legal types.
1385 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1386 TLI->getTypeAction(*getContext(), EltVT) ==
1387 TargetLowering::TypeExpandInteger) {
1388 const APInt &NewVal = Elt->getValue();
1389 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1390 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1391
1392 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1393 if (VT.isScalableVector()) {
1394 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&((void)0)
1395 "Can only handle an even split!")((void)0);
1396 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1397
1398 SmallVector<SDValue, 2> ScalarParts;
1399 for (unsigned i = 0; i != Parts; ++i)
1400 ScalarParts.push_back(getConstant(
1401 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1402 ViaEltVT, isT, isO));
1403
1404 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1405 }
1406
1407 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1408 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1409
1410 // Check the temporary vector is the correct size. If this fails then
1411 // getTypeToTransformTo() probably returned a type whose size (in bits)
1412 // isn't a power-of-2 factor of the requested type size.
1413 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits())((void)0);
1414
1415 SmallVector<SDValue, 2> EltParts;
1416 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1417 EltParts.push_back(getConstant(
1418 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1419 ViaEltVT, isT, isO));
1420
1421 // EltParts is currently in little endian order. If we actually want
1422 // big-endian order then reverse it now.
1423 if (getDataLayout().isBigEndian())
1424 std::reverse(EltParts.begin(), EltParts.end());
1425
1426 // The elements must be reversed when the element order is different
1427 // to the endianness of the elements (because the BITCAST is itself a
1428 // vector shuffle in this situation). However, we do not need any code to
1429 // perform this reversal because getConstant() is producing a vector
1430 // splat.
1431 // This situation occurs in MIPS MSA.
1432
1433 SmallVector<SDValue, 8> Ops;
1434 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1435 llvm::append_range(Ops, EltParts);
1436
1437 SDValue V =
1438 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1439 return V;
1440 }
1441
1442 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&((void)0)
1443 "APInt size does not match type size!")((void)0);
1444 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1445 FoldingSetNodeID ID;
1446 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1447 ID.AddPointer(Elt);
1448 ID.AddBoolean(isO);
1449 void *IP = nullptr;
1450 SDNode *N = nullptr;
1451 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1452 if (!VT.isVector())
1453 return SDValue(N, 0);
1454
1455 if (!N) {
1456 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1457 CSEMap.InsertNode(N, IP);
1458 InsertNode(N);
1459 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1460 }
1461
1462 SDValue Result(N, 0);
1463 if (VT.isScalableVector())
1464 Result = getSplatVector(VT, DL, Result);
1465 else if (VT.isVector())
1466 Result = getSplatBuildVector(VT, DL, Result);
1467
1468 return Result;
1469}
1470
1471SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1472 bool isTarget) {
1473 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1474}
1475
1476SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1477 const SDLoc &DL, bool LegalTypes) {
1478 assert(VT.isInteger() && "Shift amount is not an integer type!")((void)0);
1479 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1480 return getConstant(Val, DL, ShiftVT);
1481}
1482
1483SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1484 bool isTarget) {
1485 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1486}
1487
1488SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1489 bool isTarget) {
1490 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1491}
1492
1493SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1494 EVT VT, bool isTarget) {
1495 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!")((void)0);
1496
1497 EVT EltVT = VT.getScalarType();
1498
1499 // Do the map lookup using the actual bit pattern for the floating point
1500 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1501 // we don't have issues with SNANs.
1502 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1503 FoldingSetNodeID ID;
1504 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1505 ID.AddPointer(&V);
1506 void *IP = nullptr;
1507 SDNode *N = nullptr;
1508 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1509 if (!VT.isVector())
1510 return SDValue(N, 0);
1511
1512 if (!N) {
1513 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1514 CSEMap.InsertNode(N, IP);
1515 InsertNode(N);
1516 }
1517
1518 SDValue Result(N, 0);
1519 if (VT.isScalableVector())
1520 Result = getSplatVector(VT, DL, Result);
1521 else if (VT.isVector())
1522 Result = getSplatBuildVector(VT, DL, Result);
1523 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1524 return Result;
1525}
1526
1527SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1528 bool isTarget) {
1529 EVT EltVT = VT.getScalarType();
1530 if (EltVT == MVT::f32)
1531 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1532 if (EltVT == MVT::f64)
1533 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1534 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1535 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1536 bool Ignored;
1537 APFloat APF = APFloat(Val);
1538 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1539 &Ignored);
1540 return getConstantFP(APF, DL, VT, isTarget);
1541 }
1542 llvm_unreachable("Unsupported type in getConstantFP")__builtin_unreachable();
1543}
1544
1545SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1546 EVT VT, int64_t Offset, bool isTargetGA,
1547 unsigned TargetFlags) {
1548 assert((TargetFlags == 0 || isTargetGA) &&((void)0)
1549 "Cannot set target flags on target-independent globals")((void)0);
1550
1551 // Truncate (with sign-extension) the offset value to the pointer size.
1552 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1553 if (BitWidth < 64)
1554 Offset = SignExtend64(Offset, BitWidth);
1555
1556 unsigned Opc;
1557 if (GV->isThreadLocal())
1558 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1559 else
1560 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1561
1562 FoldingSetNodeID ID;
1563 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1564 ID.AddPointer(GV);
1565 ID.AddInteger(Offset);
1566 ID.AddInteger(TargetFlags);
1567 void *IP = nullptr;
1568 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1569 return SDValue(E, 0);
1570
1571 auto *N = newSDNode<GlobalAddressSDNode>(
1572 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1573 CSEMap.InsertNode(N, IP);
1574 InsertNode(N);
1575 return SDValue(N, 0);
1576}
1577
1578SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1579 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1580 FoldingSetNodeID ID;
1581 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1582 ID.AddInteger(FI);
1583 void *IP = nullptr;
1584 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1585 return SDValue(E, 0);
1586
1587 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1588 CSEMap.InsertNode(N, IP);
1589 InsertNode(N);
1590 return SDValue(N, 0);
1591}
1592
1593SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1594 unsigned TargetFlags) {
1595 assert((TargetFlags == 0 || isTarget) &&((void)0)
1596 "Cannot set target flags on target-independent jump tables")((void)0);
1597 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1598 FoldingSetNodeID ID;
1599 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1600 ID.AddInteger(JTI);
1601 ID.AddInteger(TargetFlags);
1602 void *IP = nullptr;
1603 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1604 return SDValue(E, 0);
1605
1606 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1607 CSEMap.InsertNode(N, IP);
1608 InsertNode(N);
1609 return SDValue(N, 0);
1610}
1611
1612SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1613 MaybeAlign Alignment, int Offset,
1614 bool isTarget, unsigned TargetFlags) {
1615 assert((TargetFlags == 0 || isTarget) &&((void)0)
1616 "Cannot set target flags on target-independent globals")((void)0);
1617 if (!Alignment)
1618 Alignment = shouldOptForSize()
1619 ? getDataLayout().getABITypeAlign(C->getType())
1620 : getDataLayout().getPrefTypeAlign(C->getType());
1621 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1622 FoldingSetNodeID ID;
1623 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1624 ID.AddInteger(Alignment->value());
1625 ID.AddInteger(Offset);
1626 ID.AddPointer(C);
1627 ID.AddInteger(TargetFlags);
1628 void *IP = nullptr;
1629 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1630 return SDValue(E, 0);
1631
1632 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1633 TargetFlags);
1634 CSEMap.InsertNode(N, IP);
1635 InsertNode(N);
1636 SDValue V = SDValue(N, 0);
1637 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1638 return V;
1639}
1640
1641SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1642 MaybeAlign Alignment, int Offset,
1643 bool isTarget, unsigned TargetFlags) {
1644 assert((TargetFlags == 0 || isTarget) &&((void)0)
1645 "Cannot set target flags on target-independent globals")((void)0);
1646 if (!Alignment)
1647 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1648 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1649 FoldingSetNodeID ID;
1650 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1651 ID.AddInteger(Alignment->value());
1652 ID.AddInteger(Offset);
1653 C->addSelectionDAGCSEId(ID);
1654 ID.AddInteger(TargetFlags);
1655 void *IP = nullptr;
1656 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1657 return SDValue(E, 0);
1658
1659 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1660 TargetFlags);
1661 CSEMap.InsertNode(N, IP);
1662 InsertNode(N);
1663 return SDValue(N, 0);
1664}
1665
1666SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1667 unsigned TargetFlags) {
1668 FoldingSetNodeID ID;
1669 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1670 ID.AddInteger(Index);
1671 ID.AddInteger(Offset);
1672 ID.AddInteger(TargetFlags);
1673 void *IP = nullptr;
1674 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1675 return SDValue(E, 0);
1676
1677 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1678 CSEMap.InsertNode(N, IP);
1679 InsertNode(N);
1680 return SDValue(N, 0);
1681}
1682
1683SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1684 FoldingSetNodeID ID;
1685 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1686 ID.AddPointer(MBB);
1687 void *IP = nullptr;
1688 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1689 return SDValue(E, 0);
1690
1691 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1692 CSEMap.InsertNode(N, IP);
1693 InsertNode(N);
1694 return SDValue(N, 0);
1695}
1696
1697SDValue SelectionDAG::getValueType(EVT VT) {
1698 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1699 ValueTypeNodes.size())
1700 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1701
1702 SDNode *&N = VT.isExtended() ?
1703 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1704
1705 if (N) return SDValue(N, 0);
1706 N = newSDNode<VTSDNode>(VT);
1707 InsertNode(N);
1708 return SDValue(N, 0);
1709}
1710
1711SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1712 SDNode *&N = ExternalSymbols[Sym];
1713 if (N) return SDValue(N, 0);
1714 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1715 InsertNode(N);
1716 return SDValue(N, 0);
1717}
1718
1719SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1720 SDNode *&N = MCSymbols[Sym];
1721 if (N)
1722 return SDValue(N, 0);
1723 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1724 InsertNode(N);
1725 return SDValue(N, 0);
1726}
1727
1728SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1729 unsigned TargetFlags) {
1730 SDNode *&N =
1731 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1732 if (N) return SDValue(N, 0);
1733 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1734 InsertNode(N);
1735 return SDValue(N, 0);
1736}
1737
1738SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1739 if ((unsigned)Cond >= CondCodeNodes.size())
1740 CondCodeNodes.resize(Cond+1);
1741
1742 if (!CondCodeNodes[Cond]) {
1743 auto *N = newSDNode<CondCodeSDNode>(Cond);
1744 CondCodeNodes[Cond] = N;
1745 InsertNode(N);
1746 }
1747
1748 return SDValue(CondCodeNodes[Cond], 0);
1749}
1750
1751SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
1752 APInt One(ResVT.getScalarSizeInBits(), 1);
1753 return getStepVector(DL, ResVT, One);
1754}
1755
1756SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
1757 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth())((void)0);
1758 if (ResVT.isScalableVector())
1759 return getNode(
1760 ISD::STEP_VECTOR, DL, ResVT,
1761 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
1762
1763 SmallVector<SDValue, 16> OpsStepConstants;
1764 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
1765 OpsStepConstants.push_back(
1766 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
1767 return getBuildVector(ResVT, DL, OpsStepConstants);
1768}
1769
1770/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1771/// point at N1 to point at N2 and indices that point at N2 to point at N1.
1772static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1773 std::swap(N1, N2);
1774 ShuffleVectorSDNode::commuteMask(M);
1775}
1776
1777SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1778 SDValue N2, ArrayRef<int> Mask) {
1779 assert(VT.getVectorNumElements() == Mask.size() &&((void)0)
1780 "Must have the same number of vector elements as mask elements!")((void)0);
1781 assert(VT == N1.getValueType() && VT == N2.getValueType() &&((void)0)
1782 "Invalid VECTOR_SHUFFLE")((void)0);
1783
1784 // Canonicalize shuffle undef, undef -> undef
1785 if (N1.isUndef() && N2.isUndef())
1786 return getUNDEF(VT);
1787
1788 // Validate that all indices in Mask are within the range of the elements
1789 // input to the shuffle.
1790 int NElts = Mask.size();
1791 assert(llvm::all_of(Mask,((void)0)
1792 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&((void)0)
1793 "Index out of range")((void)0);
1794
1795 // Copy the mask so we can do any needed cleanup.
1796 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1797
1798 // Canonicalize shuffle v, v -> v, undef
1799 if (N1 == N2) {
1800 N2 = getUNDEF(VT);
1801 for (int i = 0; i != NElts; ++i)
1802 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1803 }
1804
1805 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1806 if (N1.isUndef())
1807 commuteShuffle(N1, N2, MaskVec);
1808
1809 if (TLI->hasVectorBlend()) {
1810 // If shuffling a splat, try to blend the splat instead. We do this here so
1811 // that even when this arises during lowering we don't have to re-handle it.
1812 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1813 BitVector UndefElements;
1814 SDValue Splat = BV->getSplatValue(&UndefElements);
1815 if (!Splat)
1816 return;
1817
1818 for (int i = 0; i < NElts; ++i) {
1819 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
1820 continue;
1821
1822 // If this input comes from undef, mark it as such.
1823 if (UndefElements[MaskVec[i] - Offset]) {
1824 MaskVec[i] = -1;
1825 continue;
1826 }
1827
1828 // If we can blend a non-undef lane, use that instead.
1829 if (!UndefElements[i])
1830 MaskVec[i] = i + Offset;
1831 }
1832 };
1833 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1834 BlendSplat(N1BV, 0);
1835 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1836 BlendSplat(N2BV, NElts);
1837 }
1838
1839 // Canonicalize all index into lhs, -> shuffle lhs, undef
1840 // Canonicalize all index into rhs, -> shuffle rhs, undef
1841 bool AllLHS = true, AllRHS = true;
1842 bool N2Undef = N2.isUndef();
1843 for (int i = 0; i != NElts; ++i) {
1844 if (MaskVec[i] >= NElts) {
1845 if (N2Undef)
1846 MaskVec[i] = -1;
1847 else
1848 AllLHS = false;
1849 } else if (MaskVec[i] >= 0) {
1850 AllRHS = false;
1851 }
1852 }
1853 if (AllLHS && AllRHS)
1854 return getUNDEF(VT);
1855 if (AllLHS && !N2Undef)
1856 N2 = getUNDEF(VT);
1857 if (AllRHS) {
1858 N1 = getUNDEF(VT);
1859 commuteShuffle(N1, N2, MaskVec);
1860 }
1861 // Reset our undef status after accounting for the mask.
1862 N2Undef = N2.isUndef();
1863 // Re-check whether both sides ended up undef.
1864 if (N1.isUndef() && N2Undef)
1865 return getUNDEF(VT);
1866
1867 // If Identity shuffle return that node.
1868 bool Identity = true, AllSame = true;
1869 for (int i = 0; i != NElts; ++i) {
1870 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1871 if (MaskVec[i] != MaskVec[0]) AllSame = false;
1872 }
1873 if (Identity && NElts)
1874 return N1;
1875
1876 // Shuffling a constant splat doesn't change the result.
1877 if (N2Undef) {
1878 SDValue V = N1;
1879
1880 // Look through any bitcasts. We check that these don't change the number
1881 // (and size) of elements and just changes their types.
1882 while (V.getOpcode() == ISD::BITCAST)
1883 V = V->getOperand(0);
1884
1885 // A splat should always show up as a build vector node.
1886 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1887 BitVector UndefElements;
1888 SDValue Splat = BV->getSplatValue(&UndefElements);
1889 // If this is a splat of an undef, shuffling it is also undef.
1890 if (Splat && Splat.isUndef())
1891 return getUNDEF(VT);
1892
1893 bool SameNumElts =
1894 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1895
1896 // We only have a splat which can skip shuffles if there is a splatted
1897 // value and no undef lanes rearranged by the shuffle.
1898 if (Splat && UndefElements.none()) {
1899 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1900 // number of elements match or the value splatted is a zero constant.
1901 if (SameNumElts)
1902 return N1;
1903 if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1904 if (C->isNullValue())
1905 return N1;
1906 }
1907
1908 // If the shuffle itself creates a splat, build the vector directly.
1909 if (AllSame && SameNumElts) {
1910 EVT BuildVT = BV->getValueType(0);
1911 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1912 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1913
1914 // We may have jumped through bitcasts, so the type of the
1915 // BUILD_VECTOR may not match the type of the shuffle.
1916 if (BuildVT != VT)
1917 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1918 return NewBV;
1919 }
1920 }
1921 }
1922
1923 FoldingSetNodeID ID;
1924 SDValue Ops[2] = { N1, N2 };
1925 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1926 for (int i = 0; i != NElts; ++i)
1927 ID.AddInteger(MaskVec[i]);
1928
1929 void* IP = nullptr;
1930 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1931 return SDValue(E, 0);
1932
1933 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1934 // SDNode doesn't have access to it. This memory will be "leaked" when
1935 // the node is deallocated, but recovered when the NodeAllocator is released.
1936 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1937 llvm::copy(MaskVec, MaskAlloc);
1938
1939 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1940 dl.getDebugLoc(), MaskAlloc);
1941 createOperands(N, Ops);
1942
1943 CSEMap.InsertNode(N, IP);
1944 InsertNode(N);
1945 SDValue V = SDValue(N, 0);
1946 NewSDValueDbgMsg(V, "Creating new node: ", this);
1947 return V;
1948}
1949
1950SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1951 EVT VT = SV.getValueType(0);
1952 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1953 ShuffleVectorSDNode::commuteMask(MaskVec);
1954
1955 SDValue Op0 = SV.getOperand(0);
1956 SDValue Op1 = SV.getOperand(1);
1957 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1958}
1959
1960SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1961 FoldingSetNodeID ID;
1962 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1963 ID.AddInteger(RegNo);
1964 void *IP = nullptr;
1965 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1966 return SDValue(E, 0);
1967
1968 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1969 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1970 CSEMap.InsertNode(N, IP);
1971 InsertNode(N);
1972 return SDValue(N, 0);
1973}
1974
1975SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1976 FoldingSetNodeID ID;
1977 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1978 ID.AddPointer(RegMask);
1979 void *IP = nullptr;
1980 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1981 return SDValue(E, 0);
1982
1983 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1984 CSEMap.InsertNode(N, IP);
1985 InsertNode(N);
1986 return SDValue(N, 0);
1987}
1988
1989SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1990 MCSymbol *Label) {
1991 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1992}
1993
1994SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1995 SDValue Root, MCSymbol *Label) {
1996 FoldingSetNodeID ID;
1997 SDValue Ops[] = { Root };
1998 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1999 ID.AddPointer(Label);
2000 void *IP = nullptr;
2001 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2002 return SDValue(E, 0);
2003
2004 auto *N =
2005 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2006 createOperands(N, Ops);
2007
2008 CSEMap.InsertNode(N, IP);
2009 InsertNode(N);
2010 return SDValue(N, 0);
2011}
2012
2013SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2014 int64_t Offset, bool isTarget,
2015 unsigned TargetFlags) {
2016 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2017
2018 FoldingSetNodeID ID;
2019 AddNodeIDNode(ID, Opc, getVTList(VT), None);
2020 ID.AddPointer(BA);
2021 ID.AddInteger(Offset);
2022 ID.AddInteger(TargetFlags);
2023 void *IP = nullptr;
2024 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2025 return SDValue(E, 0);
2026
2027 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
2028 CSEMap.InsertNode(N, IP);
2029 InsertNode(N);
2030 return SDValue(N, 0);
2031}
2032
2033SDValue SelectionDAG::getSrcValue(const Value *V) {
2034 FoldingSetNodeID ID;
2035 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
2036 ID.AddPointer(V);
2037
2038 void *IP = nullptr;
2039 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2040 return SDValue(E, 0);
2041
2042 auto *N = newSDNode<SrcValueSDNode>(V);
2043 CSEMap.InsertNode(N, IP);
2044 InsertNode(N);
2045 return SDValue(N, 0);
2046}
2047
2048SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2049 FoldingSetNodeID ID;
2050 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
2051 ID.AddPointer(MD);
2052
2053 void *IP = nullptr;
2054 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2055 return SDValue(E, 0);
2056
2057 auto *N = newSDNode<MDNodeSDNode>(MD);
2058 CSEMap.InsertNode(N, IP);
2059 InsertNode(N);
2060 return SDValue(N, 0);
2061}
2062
2063SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2064 if (VT == V.getValueType())
2065 return V;
2066
2067 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2068}
2069
2070SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2071 unsigned SrcAS, unsigned DestAS) {
2072 SDValue Ops[] = {Ptr};
2073 FoldingSetNodeID ID;
2074 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
2075 ID.AddInteger(SrcAS);
2076 ID.AddInteger(DestAS);
2077
2078 void *IP = nullptr;
2079 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2080 return SDValue(E, 0);
2081
2082 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2083 VT, SrcAS, DestAS);
2084 createOperands(N, Ops);
2085
2086 CSEMap.InsertNode(N, IP);
2087 InsertNode(N);
2088 return SDValue(N, 0);
2089}
2090
2091SDValue SelectionDAG::getFreeze(SDValue V) {
2092 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2093}
2094
2095/// getShiftAmountOperand - Return the specified value casted to
2096/// the target's desired shift amount type.
2097SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2098 EVT OpTy = Op.getValueType();
2099 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2100 if (OpTy == ShTy || OpTy.isVector()) return Op;
2101
2102 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2103}
2104
2105SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2106 SDLoc dl(Node);
2107 const TargetLowering &TLI = getTargetLoweringInfo();
2108 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2109 EVT VT = Node->getValueType(0);
2110 SDValue Tmp1 = Node->getOperand(0);
2111 SDValue Tmp2 = Node->getOperand(1);
2112 const MaybeAlign MA(Node->getConstantOperandVal(3));
2113
2114 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2115 Tmp2, MachinePointerInfo(V));
2116 SDValue VAList = VAListLoad;
2117
2118 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2119 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2120 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2121
2122 VAList =
2123 getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2124 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2125 }
2126
2127 // Increment the pointer, VAList, to the next vaarg
2128 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2129 getConstant(getDataLayout().getTypeAllocSize(
2130 VT.getTypeForEVT(*getContext())),
2131 dl, VAList.getValueType()));
2132 // Store the incremented VAList to the legalized pointer
2133 Tmp1 =
2134 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2135 // Load the actual argument out of the pointer VAList
2136 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2137}
2138
2139SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2140 SDLoc dl(Node);
2141 const TargetLowering &TLI = getTargetLoweringInfo();
2142 // This defaults to loading a pointer from the input and storing it to the
2143 // output, returning the chain.
2144 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2145 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2146 SDValue Tmp1 =
2147 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2148 Node->getOperand(2), MachinePointerInfo(VS));
2149 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2150 MachinePointerInfo(VD));
2151}
2152
2153Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2154 const DataLayout &DL = getDataLayout();
2155 Type *Ty = VT.getTypeForEVT(*getContext());
2156 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2157
2158 if (TLI->isTypeLegal(VT) || !VT.isVector())
2159 return RedAlign;
2160
2161 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2162 const Align StackAlign = TFI->getStackAlign();
2163
2164 // See if we can choose a smaller ABI alignment in cases where it's an
2165 // illegal vector type that will get broken down.
2166 if (RedAlign > StackAlign) {
2167 EVT IntermediateVT;
2168 MVT RegisterVT;
2169 unsigned NumIntermediates;
2170 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2171 NumIntermediates, RegisterVT);
2172 Ty = IntermediateVT.getTypeForEVT(*getContext());
2173 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2174 if (RedAlign2 < RedAlign)
2175 RedAlign = RedAlign2;
2176 }
2177
2178 return RedAlign;
2179}
2180
2181SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2182 MachineFrameInfo &MFI = MF->getFrameInfo();
2183 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2184 int StackID = 0;
2185 if (Bytes.isScalable())
2186 StackID = TFI->getStackIDForScalableVectors();
2187 // The stack id gives an indication of whether the object is scalable or
2188 // not, so it's safe to pass in the minimum size here.
2189 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment,
2190 false, nullptr, StackID);
2191 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2192}
2193
2194SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2195 Type *Ty = VT.getTypeForEVT(*getContext());
2196 Align StackAlign =
2197 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2198 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2199}
2200
2201SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2202 TypeSize VT1Size = VT1.getStoreSize();
2203 TypeSize VT2Size = VT2.getStoreSize();
2204 assert(VT1Size.isScalable() == VT2Size.isScalable() &&((void)0)
2205 "Don't know how to choose the maximum size when creating a stack "((void)0)
2206 "temporary")((void)0);
2207 TypeSize Bytes =
2208 VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size;
2209
2210 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2211 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2212 const DataLayout &DL = getDataLayout();
2213 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2214 return CreateStackTemporary(Bytes, Align);
2215}
2216
2217SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2218 ISD::CondCode Cond, const SDLoc &dl) {
2219 EVT OpVT = N1.getValueType();
2220
2221 // These setcc operations always fold.
2222 switch (Cond) {
2223 default: break;
2224 case ISD::SETFALSE:
2225 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2226 case ISD::SETTRUE:
2227 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2228
2229 case ISD::SETOEQ:
2230 case ISD::SETOGT:
2231 case ISD::SETOGE:
2232 case ISD::SETOLT:
2233 case ISD::SETOLE:
2234 case ISD::SETONE:
2235 case ISD::SETO:
2236 case ISD::SETUO:
2237 case ISD::SETUEQ:
2238 case ISD::SETUNE:
2239 assert(!OpVT.isInteger() && "Illegal setcc for integer!")((void)0);
2240 break;
2241 }
2242
2243 if (OpVT.isInteger()) {
2244 // For EQ and NE, we can always pick a value for the undef to make the
2245 // predicate pass or fail, so we can return undef.
2246 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2247 // icmp eq/ne X, undef -> undef.
2248 if ((N1.isUndef() || N2.isUndef()) &&
2249 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2250 return getUNDEF(VT);
2251
2252 // If both operands are undef, we can return undef for int comparison.
2253 // icmp undef, undef -> undef.
2254 if (N1.isUndef() && N2.isUndef())
2255 return getUNDEF(VT);
2256
2257 // icmp X, X -> true/false
2258 // icmp X, undef -> true/false because undef could be X.
2259 if (N1 == N2)
2260 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2261 }
2262
2263 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2264 const APInt &C2 = N2C->getAPIntValue();
2265 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2266 const APInt &C1 = N1C->getAPIntValue();
2267
2268 switch (Cond) {
2269 default: llvm_unreachable("Unknown integer setcc!")__builtin_unreachable();
2270 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT);
2271 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT);
2272 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
2273 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
2274 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
2275 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
2276 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
2277 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
2278 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
2279 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
2280 }
2281 }
2282 }
2283
2284 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2285 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2286
2287 if (N1CFP && N2CFP) {
2288 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2289 switch (Cond) {
2290 default: break;
2291 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2292 return getUNDEF(VT);
2293 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2294 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2295 OpVT);
2296 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2297 return getUNDEF(VT);
2298 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2299 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2300 R==APFloat::cmpLessThan, dl, VT,
2301 OpVT);
2302 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2303 return getUNDEF(VT);
2304 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2305 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2306 OpVT);
2307 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2308 return getUNDEF(VT);
2309 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2310 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2311 VT, OpVT);
2312 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2313 return getUNDEF(VT);
2314 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2315 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2316 R==APFloat::cmpEqual, dl, VT,
2317 OpVT);
2318 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2319 return getUNDEF(VT);
2320 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2321 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2322 R==APFloat::cmpEqual, dl, VT, OpVT);
2323 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2324 OpVT);
2325 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2326 OpVT);
2327 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2328 R==APFloat::cmpEqual, dl, VT,
2329 OpVT);
2330 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2331 OpVT);
2332 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2333 R==APFloat::cmpLessThan, dl, VT,
2334 OpVT);
2335 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2336 R==APFloat::cmpUnordered, dl, VT,
2337 OpVT);
2338 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2339 VT, OpVT);
2340 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2341 OpVT);
2342 }
2343 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2344 // Ensure that the constant occurs on the RHS.
2345 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2346 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2347 return SDValue();
2348 return getSetCC(dl, VT, N2, N1, SwappedCond);
2349 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2350 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2351 // If an operand is known to be a nan (or undef that could be a nan), we can
2352 // fold it.
2353 // Choosing NaN for the undef will always make unordered comparison succeed
2354 // and ordered comparison fails.
2355 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2356 switch (ISD::getUnorderedFlavor(Cond)) {
2357 default:
2358 llvm_unreachable("Unknown flavor!")__builtin_unreachable();
2359 case 0: // Known false.
2360 return getBoolConstant(false, dl, VT, OpVT);
2361 case 1: // Known true.
2362 return getBoolConstant(true, dl, VT, OpVT);
2363 case 2: // Undefined.
2364 return getUNDEF(VT);
2365 }
2366 }
2367
2368 // Could not fold it.
2369 return SDValue();
2370}
2371
2372/// See if the specified operand can be simplified with the knowledge that only
2373/// the bits specified by DemandedBits are used.
2374/// TODO: really we should be making this into the DAG equivalent of
2375/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2376SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2377 EVT VT = V.getValueType();
2378
2379 if (VT.isScalableVector())
2380 return SDValue();
2381
2382 APInt DemandedElts = VT.isVector()
2383 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2384 : APInt(1, 1);
2385 return GetDemandedBits(V, DemandedBits, DemandedElts);
2386}
2387
2388/// See if the specified operand can be simplified with the knowledge that only
2389/// the bits specified by DemandedBits are used in the elements specified by
2390/// DemandedElts.
2391/// TODO: really we should be making this into the DAG equivalent of
2392/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2393SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2394 const APInt &DemandedElts) {
2395 switch (V.getOpcode()) {
2396 default:
2397 return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts,
2398 *this, 0);
2399 case ISD::Constant: {
2400 const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue();
2401 APInt NewVal = CVal & DemandedBits;
2402 if (NewVal != CVal)
2403 return getConstant(NewVal, SDLoc(V), V.getValueType());
2404 break;
2405 }
2406 case ISD::SRL:
2407 // Only look at single-use SRLs.
2408 if (!V.getNode()->hasOneUse())
2409 break;
2410 if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2411 // See if we can recursively simplify the LHS.
2412 unsigned Amt = RHSC->getZExtValue();
2413
2414 // Watch out for shift count overflow though.
2415 if (Amt >= DemandedBits.getBitWidth())
2416 break;
2417 APInt SrcDemandedBits = DemandedBits << Amt;
2418 if (SDValue SimplifyLHS =
2419 GetDemandedBits(V.getOperand(0), SrcDemandedBits))
2420 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2421 V.getOperand(1));
2422 }
2423 break;
2424 }
2425 return SDValue();
2426}
2427
2428/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2429/// use this predicate to simplify operations downstream.
2430bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2431 unsigned BitWidth = Op.getScalarValueSizeInBits();
2432 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2433}
2434
2435/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2436/// this predicate to simplify operations downstream. Mask is known to be zero
2437/// for bits that V cannot have.
2438bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2439 unsigned Depth) const {
2440 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2441}
2442
2443/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2444/// DemandedElts. We use this predicate to simplify operations downstream.
2445/// Mask is known to be zero for bits that V cannot have.
2446bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2447 const APInt &DemandedElts,
2448 unsigned Depth) const {
2449 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2450}
2451
2452/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2453bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2454 unsigned Depth) const {
2455 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2456}
2457
2458/// isSplatValue - Return true if the vector V has the same value
2459/// across all DemandedElts. For scalable vectors it does not make
2460/// sense to specify which elements are demanded or undefined, therefore
2461/// they are simply ignored.
2462bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2463 APInt &UndefElts, unsigned Depth) {
2464 EVT VT = V.getValueType();
2465 assert(VT.isVector() && "Vector type expected")((void)0);
2466
2467 if (!VT.isScalableVector() && !DemandedElts)
2468 return false; // No demanded elts, better to assume we don't know anything.
2469
2470 if (Depth >= MaxRecursionDepth)
2471 return false; // Limit search depth.
2472
2473 // Deal with some common cases here that work for both fixed and scalable
2474 // vector types.
2475 switch (V.getOpcode()) {
2476 case ISD::SPLAT_VECTOR:
2477 UndefElts = V.getOperand(0).isUndef()
2478 ? APInt::getAllOnesValue(DemandedElts.getBitWidth())
2479 : APInt(DemandedElts.getBitWidth(), 0);
2480 return true;
2481 case ISD::ADD:
2482 case ISD::SUB:
2483 case ISD::AND:
2484 case ISD::XOR:
2485 case ISD::OR: {
2486 APInt UndefLHS, UndefRHS;
2487 SDValue LHS = V.getOperand(0);
2488 SDValue RHS = V.getOperand(1);
2489 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2490 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
2491 UndefElts = UndefLHS | UndefRHS;
2492 return true;
2493 }
2494 return false;
2495 }
2496 case ISD::ABS:
2497 case ISD::TRUNCATE:
2498 case ISD::SIGN_EXTEND:
2499 case ISD::ZERO_EXTEND:
2500 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2501 }
2502
2503 // We don't support other cases than those above for scalable vectors at
2504 // the moment.
2505 if (VT.isScalableVector())
2506 return false;
2507
2508 unsigned NumElts = VT.getVectorNumElements();
2509 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch")((void)0);
2510 UndefElts = APInt::getNullValue(NumElts);
2511
2512 switch (V.getOpcode()) {
2513 case ISD::BUILD_VECTOR: {
2514 SDValue Scl;
2515 for (unsigned i = 0; i != NumElts; ++i) {
2516 SDValue Op = V.getOperand(i);
2517 if (Op.isUndef()) {
2518 UndefElts.setBit(i);
2519 continue;
2520 }
2521 if (!DemandedElts[i])
2522 continue;
2523 if (Scl && Scl != Op)
2524 return false;
2525 Scl = Op;
2526 }
2527 return true;
2528 }
2529 case ISD::VECTOR_SHUFFLE: {
2530 // Check if this is a shuffle node doing a splat.
2531 // TODO: Do we need to handle shuffle(splat, undef, mask)?
2532 int SplatIndex = -1;
2533 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2534 for (int i = 0; i != (int)NumElts; ++i) {
2535 int M = Mask[i];
2536 if (M < 0) {
2537 UndefElts.setBit(i);
2538 continue;
2539 }
2540 if (!DemandedElts[i])
2541 continue;
2542 if (0 <= SplatIndex && SplatIndex != M)
2543 return false;
2544 SplatIndex = M;
2545 }
2546 return true;
2547 }
2548 case ISD::EXTRACT_SUBVECTOR: {
2549 // Offset the demanded elts by the subvector index.
2550 SDValue Src = V.getOperand(0);
2551 // We don't support scalable vectors at the moment.
2552 if (Src.getValueType().isScalableVector())
2553 return false;
2554 uint64_t Idx = V.getConstantOperandVal(1);
2555 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2556 APInt UndefSrcElts;
2557 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2558 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2559 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2560 return true;
2561 }
2562 break;
2563 }
2564 }
2565
2566 return false;
2567}
2568
2569/// Helper wrapper to main isSplatValue function.
2570bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2571 EVT VT = V.getValueType();
2572 assert(VT.isVector() && "Vector type expected")((void)0);
2573
2574 APInt UndefElts;
2575 APInt DemandedElts;
2576
2577 // For now we don't support this with scalable vectors.
2578 if (!VT.isScalableVector())
2579 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2580 return isSplatValue(V, DemandedElts, UndefElts) &&
2581 (AllowUndefs || !UndefElts);
2582}
2583
2584SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2585 V = peekThroughExtractSubvectors(V);
2586
2587 EVT VT = V.getValueType();
2588 unsigned Opcode = V.getOpcode();
2589 switch (Opcode) {
2590 default: {
2591 APInt UndefElts;
2592 APInt DemandedElts;
2593
2594 if (!VT.isScalableVector())
2595 DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2596
2597 if (isSplatValue(V, DemandedElts, UndefElts)) {
2598 if (VT.isScalableVector()) {
2599 // DemandedElts and UndefElts are ignored for scalable vectors, since
2600 // the only supported cases are SPLAT_VECTOR nodes.
2601 SplatIdx = 0;
2602 } else {
2603 // Handle case where all demanded elements are UNDEF.
2604 if (DemandedElts.isSubsetOf(UndefElts)) {
2605 SplatIdx = 0;
2606 return getUNDEF(VT);
2607 }
2608 SplatIdx = (UndefElts & DemandedElts).countTrailingOnes();
2609 }
2610 return V;
2611 }
2612 break;
2613 }
2614 case ISD::SPLAT_VECTOR:
2615 SplatIdx = 0;
2616 return V;
2617 case ISD::VECTOR_SHUFFLE: {
2618 if (VT.isScalableVector())
2619 return SDValue();
2620
2621 // Check if this is a shuffle node doing a splat.
2622 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2623 // getTargetVShiftNode currently struggles without the splat source.
2624 auto *SVN = cast<ShuffleVectorSDNode>(V);
2625 if (!SVN->isSplat())
2626 break;
2627 int Idx = SVN->getSplatIndex();
2628 int NumElts = V.getValueType().getVectorNumElements();
2629 SplatIdx = Idx % NumElts;
2630 return V.getOperand(Idx / NumElts);
2631 }
2632 }
2633
2634 return SDValue();
2635}
2636
2637SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
2638 int SplatIdx;
2639 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2640 EVT SVT = SrcVector.getValueType().getScalarType();
2641 EVT LegalSVT = SVT;
2642 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
2643 if (!SVT.isInteger())
2644 return SDValue();
2645 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
2646 if (LegalSVT.bitsLT(SVT))
2647 return SDValue();
2648 }
2649 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
2650 getVectorIdxConstant(SplatIdx, SDLoc(V)));
2651 }
2652 return SDValue();
2653}
2654
2655const APInt *
2656SelectionDAG::getValidShiftAmountConstant(SDValue V,
2657 const APInt &DemandedElts) const {
2658 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||((void)0)
2659 V.getOpcode() == ISD::SRA) &&((void)0)
2660 "Unknown shift node")((void)0);
2661 unsigned BitWidth = V.getScalarValueSizeInBits();
2662 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2663 // Shifting more than the bitwidth is not valid.
2664 const APInt &ShAmt = SA->getAPIntValue();
2665 if (ShAmt.ult(BitWidth))
2666 return &ShAmt;
2667 }
2668 return nullptr;
2669}
2670
2671const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
2672 SDValue V, const APInt &DemandedElts) const {
2673 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||((void)0)
2674 V.getOpcode() == ISD::SRA) &&((void)0)
2675 "Unknown shift node")((void)0);
2676 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2677 return ValidAmt;
2678 unsigned BitWidth = V.getScalarValueSizeInBits();
2679 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2680 if (!BV)
2681 return nullptr;
2682 const APInt *MinShAmt = nullptr;
2683 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2684 if (!DemandedElts[i])
2685 continue;
2686 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2687 if (!SA)
2688 return nullptr;
2689 // Shifting more than the bitwidth is not valid.
2690 const APInt &ShAmt = SA->getAPIntValue();
2691 if (ShAmt.uge(BitWidth))
2692 return nullptr;
2693 if (MinShAmt && MinShAmt->ule(ShAmt))
2694 continue;
2695 MinShAmt = &ShAmt;
2696 }
2697 return MinShAmt;
2698}
2699
2700const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
2701 SDValue V, const APInt &DemandedElts) const {
2702 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||((void)0)
2703 V.getOpcode() == ISD::SRA) &&((void)0)
2704 "Unknown shift node")((void)0);
2705 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2706 return ValidAmt;
2707 unsigned BitWidth = V.getScalarValueSizeInBits();
2708 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2709 if (!BV)
2710 return nullptr;
2711 const APInt *MaxShAmt = nullptr;
2712 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
2713 if (!DemandedElts[i])
2714 continue;
2715 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
2716 if (!SA)
2717 return nullptr;
2718 // Shifting more than the bitwidth is not valid.
2719 const APInt &ShAmt = SA->getAPIntValue();
2720 if (ShAmt.uge(BitWidth))
2721 return nullptr;
2722 if (MaxShAmt && MaxShAmt->uge(ShAmt))
2723 continue;
2724 MaxShAmt = &ShAmt;
2725 }
2726 return MaxShAmt;
2727}
2728
2729/// Determine which bits of Op are known to be either zero or one and return
2730/// them in Known. For vectors, the known bits are those that are shared by
2731/// every vector element.
2732KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2733 EVT VT = Op.getValueType();
2734
2735 // TOOD: Until we have a plan for how to represent demanded elements for
2736 // scalable vectors, we can just bail out for now.
2737 if (Op.getValueType().isScalableVector()) {
2738 unsigned BitWidth = Op.getScalarValueSizeInBits();
2739 return KnownBits(BitWidth);
2740 }
2741
2742 APInt DemandedElts = VT.isVector()
2743 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2744 : APInt(1, 1);
2745 return computeKnownBits(Op, DemandedElts, Depth);
2746}
2747
2748/// Determine which bits of Op are known to be either zero or one and return
2749/// them in Known. The DemandedElts argument allows us to only collect the known
2750/// bits that are shared by the requested vector elements.
2751KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2752 unsigned Depth) const {
2753 unsigned BitWidth = Op.getScalarValueSizeInBits();
2754
2755 KnownBits Known(BitWidth); // Don't know anything.
2756
2757 // TOOD: Until we have a plan for how to represent demanded elements for
2758 // scalable vectors, we can just bail out for now.
2759 if (Op.getValueType().isScalableVector())
2760 return Known;
2761
2762 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2763 // We know all of the bits for a constant!
2764 return KnownBits::makeConstant(C->getAPIntValue());
2765 }
2766 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2767 // We know all of the bits for a constant fp!
2768 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
2769 }
2770
2771 if (Depth >= MaxRecursionDepth)
2772 return Known; // Limit search depth.
2773
2774 KnownBits Known2;
2775 unsigned NumElts = DemandedElts.getBitWidth();
2776 assert((!Op.getValueType().isVector() ||((void)0)
2777 NumElts == Op.getValueType().getVectorNumElements()) &&((void)0)
2778 "Unexpected vector size")((void)0);
2779
2780 if (!DemandedElts)
2781 return Known; // No demanded elts, better to assume we don't know anything.
2782
2783 unsigned Opcode = Op.getOpcode();
2784 switch (Opcode) {
2785 case ISD::BUILD_VECTOR:
2786 // Collect the known bits that are shared by every demanded vector element.
2787 Known.Zero.setAllBits(); Known.One.setAllBits();
2788 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
2789 if (!DemandedElts[i])
2790 continue;
2791
2792 SDValue SrcOp = Op.getOperand(i);
2793 Known2 = computeKnownBits(SrcOp, Depth + 1);
2794
2795 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2796 if (SrcOp.getValueSizeInBits() != BitWidth) {
2797 assert(SrcOp.getValueSizeInBits() > BitWidth &&((void)0)
2798 "Expected BUILD_VECTOR implicit truncation")((void)0);
2799 Known2 = Known2.trunc(BitWidth);
2800 }
2801
2802 // Known bits are the values that are shared by every demanded element.
2803 Known = KnownBits::commonBits(Known, Known2);
2804
2805 // If we don't know any bits, early out.
2806 if (Known.isUnknown())
2807 break;
2808 }
2809 break;
2810 case ISD::VECTOR_SHUFFLE: {
2811 // Collect the known bits that are shared by every vector element referenced
2812 // by the shuffle.
2813 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2814 Known.Zero.setAllBits(); Known.One.setAllBits();
2815 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2816 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")((void)0);
2817 for (unsigned i = 0; i != NumElts; ++i) {
2818 if (!DemandedElts[i])
2819 continue;
2820
2821 int M = SVN->getMaskElt(i);
2822 if (M < 0) {
2823 // For UNDEF elements, we don't know anything about the common state of
2824 // the shuffle result.
2825 Known.resetAll();
2826 DemandedLHS.clearAllBits();
2827 DemandedRHS.clearAllBits();
2828 break;
2829 }
2830
2831 if ((unsigned)M < NumElts)
2832 DemandedLHS.setBit((unsigned)M % NumElts);
2833 else
2834 DemandedRHS.setBit((unsigned)M % NumElts);
2835 }
2836 // Known bits are the values that are shared by every demanded element.
2837 if (!!DemandedLHS) {
2838 SDValue LHS = Op.getOperand(0);
2839 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2840 Known = KnownBits::commonBits(Known, Known2);
2841 }
2842 // If we don't know any bits, early out.
2843 if (Known.isUnknown())
2844 break;
2845 if (!!DemandedRHS) {
2846 SDValue RHS = Op.getOperand(1);
2847 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2848 Known = KnownBits::commonBits(Known, Known2);
2849 }
2850 break;
2851 }
2852 case ISD::CONCAT_VECTORS: {
2853 // Split DemandedElts and test each of the demanded subvectors.
2854 Known.Zero.setAllBits(); Known.One.setAllBits();
2855 EVT SubVectorVT = Op.getOperand(0).getValueType();
2856 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2857 unsigned NumSubVectors = Op.getNumOperands();
2858 for (unsigned i = 0; i != NumSubVectors; ++i) {
2859 APInt DemandedSub =
2860 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
2861 if (!!DemandedSub) {
2862 SDValue Sub = Op.getOperand(i);
2863 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2864 Known = KnownBits::commonBits(Known, Known2);
2865 }
2866 // If we don't know any bits, early out.
2867 if (Known.isUnknown())
2868 break;
2869 }
2870 break;
2871 }
2872 case ISD::INSERT_SUBVECTOR: {
2873 // Demand any elements from the subvector and the remainder from the src its
2874 // inserted into.
2875 SDValue Src = Op.getOperand(0);
2876 SDValue Sub = Op.getOperand(1);
2877 uint64_t Idx = Op.getConstantOperandVal(2);
2878 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2879 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2880 APInt DemandedSrcElts = DemandedElts;
2881 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2882
2883 Known.One.setAllBits();
2884 Known.Zero.setAllBits();
2885 if (!!DemandedSubElts) {
2886 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2887 if (Known.isUnknown())
2888 break; // early-out.
2889 }
2890 if (!!DemandedSrcElts) {
2891 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2892 Known = KnownBits::commonBits(Known, Known2);
2893 }
2894 break;
2895 }
2896 case ISD::EXTRACT_SUBVECTOR: {
2897 // Offset the demanded elts by the subvector index.
2898 SDValue Src = Op.getOperand(0);
2899 // Bail until we can represent demanded elements for scalable vectors.
2900 if (Src.getValueType().isScalableVector())
2901 break;
2902 uint64_t Idx = Op.getConstantOperandVal(1);
2903 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2904 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2905 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2906 break;
2907 }
2908 case ISD::SCALAR_TO_VECTOR: {
2909 // We know about scalar_to_vector as much as we know about it source,
2910 // which becomes the first element of otherwise unknown vector.
2911 if (DemandedElts != 1)
2912 break;
2913
2914 SDValue N0 = Op.getOperand(0);
2915 Known = computeKnownBits(N0, Depth + 1);
2916 if (N0.getValueSizeInBits() != BitWidth)
2917 Known = Known.trunc(BitWidth);
2918
2919 break;
2920 }
2921 case ISD::BITCAST: {
2922 SDValue N0 = Op.getOperand(0);
2923 EVT SubVT = N0.getValueType();
2924 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2925
2926 // Ignore bitcasts from unsupported types.
2927 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
2928 break;
2929
2930 // Fast handling of 'identity' bitcasts.
2931 if (BitWidth == SubBitWidth) {
2932 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2933 break;
2934 }
2935
2936 bool IsLE = getDataLayout().isLittleEndian();
2937
2938 // Bitcast 'small element' vector to 'large element' scalar/vector.
2939 if ((BitWidth % SubBitWidth) == 0) {
2940 assert(N0.getValueType().isVector() && "Expected bitcast from vector")((void)0);
2941
2942 // Collect known bits for the (larger) output by collecting the known
2943 // bits from each set of sub elements and shift these into place.
2944 // We need to separately call computeKnownBits for each set of
2945 // sub elements as the knownbits for each is likely to be different.
2946 unsigned SubScale = BitWidth / SubBitWidth;
2947 APInt SubDemandedElts(NumElts * SubScale, 0);
2948 for (unsigned i = 0; i != NumElts; ++i)
2949 if (DemandedElts[i])
2950 SubDemandedElts.setBit(i * SubScale);
2951
2952 for (unsigned i = 0; i != SubScale; ++i) {
2953 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2954 Depth + 1);
2955 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
2956 Known.insertBits(Known2, SubBitWidth * Shifts);
2957 }
2958 }
2959
2960 // Bitcast 'large element' scalar/vector to 'small element' vector.
2961 if ((SubBitWidth % BitWidth) == 0) {
2962 assert(Op.getValueType().isVector() && "Expected bitcast to vector")((void)0);
2963
2964 // Collect known bits for the (smaller) output by collecting the known
2965 // bits from the overlapping larger input elements and extracting the
2966 // sub sections we actually care about.
2967 unsigned SubScale = SubBitWidth / BitWidth;
2968 APInt SubDemandedElts(NumElts / SubScale, 0);
2969 for (unsigned i = 0; i != NumElts; ++i)
2970 if (DemandedElts[i])
2971 SubDemandedElts.setBit(i / SubScale);
2972
2973 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2974
2975 Known.Zero.setAllBits(); Known.One.setAllBits();
2976 for (unsigned i = 0; i != NumElts; ++i)
2977 if (DemandedElts[i]) {
2978 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
2979 unsigned Offset = (Shifts % SubScale) * BitWidth;
2980 Known = KnownBits::commonBits(Known,
2981 Known2.extractBits(BitWidth, Offset));
2982 // If we don't know any bits, early out.
2983 if (Known.isUnknown())
2984 break;
2985 }
2986 }
2987 break;
2988 }
2989 case ISD::AND:
2990 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2991 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2992
2993 Known &= Known2;
2994 break;
2995 case ISD::OR:
2996 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2997 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2998
2999 Known |= Known2;
3000 break;
3001 case ISD::XOR:
3002 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3003 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3004
3005 Known ^= Known2;
3006 break;
3007 case ISD::MUL: {
3008 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3009 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3010 Known = KnownBits::mul(Known, Known2);
3011 break;
3012 }
3013 case ISD::MULHU: {
3014 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3015 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3016 Known = KnownBits::mulhu(Known, Known2);
3017 break;
3018 }
3019 case ISD::MULHS: {
3020 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3021 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3022 Known = KnownBits::mulhs(Known, Known2);
3023 break;
3024 }
3025 case ISD::UMUL_LOHI: {
3026 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result")((void)0);
3027 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3028 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3029 if (Op.getResNo() == 0)
3030 Known = KnownBits::mul(Known, Known2);
3031 else
3032 Known = KnownBits::mulhu(Known, Known2);
3033 break;
3034 }
3035 case ISD::SMUL_LOHI: {
3036 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result")((void)0);
3037 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3038 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3039 if (Op.getResNo() == 0)
3040 Known = KnownBits::mul(Known, Known2);
3041 else
3042 Known = KnownBits::mulhs(Known, Known2);
3043 break;
3044 }
3045 case ISD::UDIV: {
3046 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3047 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3048 Known = KnownBits::udiv(Known, Known2);
3049 break;
3050 }
3051 case ISD::SELECT:
3052 case ISD::VSELECT:
3053 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3054 // If we don't know any bits, early out.
3055 if (Known.isUnknown())
3056 break;
3057 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3058
3059 // Only known if known in both the LHS and RHS.
3060 Known = KnownBits::commonBits(Known, Known2);
3061 break;
3062 case ISD::SELECT_CC:
3063 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3064 // If we don't know any bits, early out.
3065 if (Known.isUnknown())
3066 break;
3067 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3068
3069 // Only known if known in both the LHS and RHS.
3070 Known = KnownBits::commonBits(Known, Known2);
3071 break;
3072 case ISD::SMULO:
3073 case ISD::UMULO:
3074 if (Op.getResNo() != 1)
3075 break;
3076 // The boolean result conforms to getBooleanContents.
3077 // If we know the result of a setcc has the top bits zero, use this info.
3078 // We know that we have an integer-based boolean since these operations
3079 // are only available for integer.
3080 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3081 TargetLowering::ZeroOrOneBooleanContent &&
3082 BitWidth > 1)
3083 Known.Zero.setBitsFrom(1);
3084 break;
3085 case ISD::SETCC:
3086 case ISD::STRICT_FSETCC:
3087 case ISD::STRICT_FSETCCS: {
3088 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3089 // If we know the result of a setcc has the top bits zero, use this info.
3090 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3091 TargetLowering::ZeroOrOneBooleanContent &&
3092 BitWidth > 1)
3093 Known.Zero.setBitsFrom(1);
3094 break;
3095 }
3096 case ISD::SHL:
3097 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3098 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3099 Known = KnownBits::shl(Known, Known2);
3100
3101 // Minimum shift low bits are known zero.
3102 if (const APInt *ShMinAmt =
3103 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3104 Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3105 break;
3106 case ISD::SRL:
3107 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3108 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3109 Known = KnownBits::lshr(Known, Known2);
3110
3111 // Minimum shift high bits are known zero.
3112 if (const APInt *ShMinAmt =
3113 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3114 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3115 break;
3116 case ISD::SRA:
3117 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3118 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3119 Known = KnownBits::ashr(Known, Known2);
3120 // TODO: Add minimum shift high known sign bits.
3121 break;
3122 case ISD::FSHL:
3123 case ISD::FSHR:
3124 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3125 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3126
3127 // For fshl, 0-shift returns the 1st arg.
3128 // For fshr, 0-shift returns the 2nd arg.
3129 if (Amt == 0) {
3130 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3131 DemandedElts, Depth + 1);
3132 break;
3133 }
3134
3135 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3136 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3137 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3138 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3139 if (Opcode == ISD::FSHL) {
3140 Known.One <<= Amt;
3141 Known.Zero <<= Amt;
3142 Known2.One.lshrInPlace(BitWidth - Amt);
3143 Known2.Zero.lshrInPlace(BitWidth - Amt);
3144 } else {
3145 Known.One <<= BitWidth - Amt;
3146 Known.Zero <<= BitWidth - Amt;
3147 Known2.One.lshrInPlace(Amt);
3148 Known2.Zero.lshrInPlace(Amt);
3149 }
3150 Known.One |= Known2.One;
3151 Known.Zero |= Known2.Zero;
3152 }
3153 break;
3154 case ISD::SIGN_EXTEND_INREG: {
3155 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3156 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3157 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3158 break;
3159 }
3160 case ISD::CTTZ:
3161 case ISD::CTTZ_ZERO_UNDEF: {
3162 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3163 // If we have a known 1, its position is our upper bound.
3164 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3165 unsigned LowBits = Log2_32(PossibleTZ) + 1;
3166 Known.Zero.setBitsFrom(LowBits);
3167 break;
3168 }
3169 case ISD::CTLZ:
3170 case ISD::CTLZ_ZERO_UNDEF: {
3171 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3172 // If we have a known 1, its position is our upper bound.
3173 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3174 unsigned LowBits = Log2_32(PossibleLZ) + 1;
3175 Known.Zero.setBitsFrom(LowBits);
3176 break;
3177 }
3178 case ISD::CTPOP: {
3179 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3180 // If we know some of the bits are zero, they can't be one.
3181 unsigned PossibleOnes = Known2.countMaxPopulation();
3182 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
3183 break;
3184 }
3185 case ISD::PARITY: {
3186 // Parity returns 0 everywhere but the LSB.
3187 Known.Zero.setBitsFrom(1);
3188 break;
3189 }
3190 case ISD::LOAD: {
3191 LoadSDNode *LD = cast<LoadSDNode>(Op);
3192 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3193 if (ISD::isNON_EXTLoad(LD) && Cst) {
3194 // Determine any common known bits from the loaded constant pool value.
3195 Type *CstTy = Cst->getType();
3196 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) {
3197 // If its a vector splat, then we can (quickly) reuse the scalar path.
3198 // NOTE: We assume all elements match and none are UNDEF.
3199 if (CstTy->isVectorTy()) {
3200 if (const Constant *Splat = Cst->getSplatValue()) {
3201 Cst = Splat;
3202 CstTy = Cst->getType();
3203 }
3204 }
3205 // TODO - do we need to handle different bitwidths?
3206 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3207 // Iterate across all vector elements finding common known bits.
3208 Known.One.setAllBits();
3209 Known.Zero.setAllBits();
3210 for (unsigned i = 0; i != NumElts; ++i) {
3211 if (!DemandedElts[i])
3212 continue;
3213 if (Constant *Elt = Cst->getAggregateElement(i)) {
3214 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3215 const APInt &Value = CInt->getValue();
3216 Known.One &= Value;
3217 Known.Zero &= ~Value;
3218 continue;
3219 }
3220 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3221 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3222 Known.One &= Value;
3223 Known.Zero &= ~Value;
3224 continue;
3225 }
3226 }
3227 Known.One.clearAllBits();
3228 Known.Zero.clearAllBits();
3229 break;
3230 }
3231 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3232 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3233 Known = KnownBits::makeConstant(CInt->getValue());
3234 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3235 Known =
3236 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
3237 }
3238 }
3239 }
3240 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
3241 // If this is a ZEXTLoad and we are looking at the loaded value.
3242 EVT VT = LD->getMemoryVT();
3243 unsigned MemBits = VT.getScalarSizeInBits();
3244 Known.Zero.setBitsFrom(MemBits);
3245 } else if (const MDNode *Ranges = LD->getRanges()) {
3246 if (LD->getExtensionType() == ISD::NON_EXTLOAD)
3247 computeKnownBitsFromRangeMetadata(*Ranges, Known);
3248 }
3249 break;
3250 }
3251 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3252 EVT InVT = Op.getOperand(0).getValueType();
3253 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3254 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3255 Known = Known.zext(BitWidth);
3256 break;
3257 }
3258 case ISD::ZERO_EXTEND: {
3259 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3260 Known = Known.zext(BitWidth);
3261 break;
3262 }
3263 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3264 EVT InVT = Op.getOperand(0).getValueType();
3265 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3266 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3267 // If the sign bit is known to be zero or one, then sext will extend
3268 // it to the top bits, else it will just zext.
3269 Known = Known.sext(BitWidth);
3270 break;
3271 }
3272 case ISD::SIGN_EXTEND: {
3273 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3274 // If the sign bit is known to be zero or one, then sext will extend
3275 // it to the top bits, else it will just zext.
3276 Known = Known.sext(BitWidth);
3277 break;
3278 }
3279 case ISD::ANY_EXTEND_VECTOR_INREG: {
3280 EVT InVT = Op.getOperand(0).getValueType();
3281 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3282 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3283 Known = Known.anyext(BitWidth);
3284 break;
3285 }
3286 case ISD::ANY_EXTEND: {
3287 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3288 Known = Known.anyext(BitWidth);
3289 break;
3290 }
3291 case ISD::TRUNCATE: {
3292 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3293 Known = Known.trunc(BitWidth);
3294 break;
3295 }
3296 case ISD::AssertZext: {
3297 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3298 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3299 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3300 Known.Zero |= (~InMask);
3301 Known.One &= (~Known.Zero);
3302 break;
3303 }
3304 case ISD::AssertAlign: {
3305 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3306 assert(LogOfAlign != 0)((void)0);
3307 // If a node is guaranteed to be aligned, set low zero bits accordingly as
3308 // well as clearing one bits.
3309 Known.Zero.setLowBits(LogOfAlign);
3310 Known.One.clearLowBits(LogOfAlign);
3311 break;
3312 }
3313 case ISD::FGETSIGN:
3314 // All bits are zero except the low bit.
3315 Known.Zero.setBitsFrom(1);
3316 break;
3317 case ISD::USUBO:
3318 case ISD::SSUBO:
3319 if (Op.getResNo() == 1) {
3320 // If we know the result of a setcc has the top bits zero, use this info.
3321 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3322 TargetLowering::ZeroOrOneBooleanContent &&
3323 BitWidth > 1)
3324 Known.Zero.setBitsFrom(1);
3325 break;
3326 }
3327 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3328 case ISD::SUB:
3329 case ISD::SUBC: {
3330 assert(Op.getResNo() == 0 &&((void)0)
3331 "We only compute knownbits for the difference here.")((void)0);
3332
3333 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3334 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3335 Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
3336 Known, Known2);
3337 break;
3338 }
3339 case ISD::UADDO:
3340 case ISD::SADDO:
3341 case ISD::ADDCARRY:
3342 if (Op.getResNo() == 1) {
3343 // If we know the result of a setcc has the top bits zero, use this info.
3344 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3345 TargetLowering::ZeroOrOneBooleanContent &&
3346 BitWidth > 1)
3347 Known.Zero.setBitsFrom(1);
3348 break;
3349 }
3350 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3351 case ISD::ADD:
3352 case ISD::ADDC:
3353 case ISD::ADDE: {
3354 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.")((void)0);
3355
3356 // With ADDE and ADDCARRY, a carry bit may be added in.
3357 KnownBits Carry(1);
3358 if (Opcode == ISD::ADDE)
3359 // Can't track carry from glue, set carry to unknown.
3360 Carry.resetAll();
3361 else if (Opcode == ISD::ADDCARRY)
3362 // TODO: Compute known bits for the carry operand. Not sure if it is worth
3363 // the trouble (how often will we find a known carry bit). And I haven't
3364 // tested this very much yet, but something like this might work:
3365 // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3366 // Carry = Carry.zextOrTrunc(1, false);
3367 Carry.resetAll();
3368 else
3369 Carry.setAllZero();
3370
3371 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3372 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3373 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3374 break;
3375 }
3376 case ISD::SREM: {
3377 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3378 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3379 Known = KnownBits::srem(Known, Known2);
3380 break;
3381 }
3382 case ISD::UREM: {
3383 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3384 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3385 Known = KnownBits::urem(Known, Known2);
3386 break;
3387 }
3388 case ISD::EXTRACT_ELEMENT: {
3389 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3390 const unsigned Index = Op.getConstantOperandVal(1);
3391 const unsigned EltBitWidth = Op.getValueSizeInBits();
3392
3393 // Remove low part of known bits mask
3394 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3395 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3396
3397 // Remove high part of known bit mask
3398 Known = Known.trunc(EltBitWidth);
3399 break;
3400 }
3401 case ISD::EXTRACT_VECTOR_ELT: {
3402 SDValue InVec = Op.getOperand(0);
3403 SDValue EltNo = Op.getOperand(1);
3404 EVT VecVT = InVec.getValueType();
3405 // computeKnownBits not yet implemented for scalable vectors.
3406 if (VecVT.isScalableVector())
3407 break;
3408 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3409 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3410
3411 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3412 // anything about the extended bits.
3413 if (BitWidth > EltBitWidth)
3414 Known = Known.trunc(EltBitWidth);
3415
3416 // If we know the element index, just demand that vector element, else for
3417 // an unknown element index, ignore DemandedElts and demand them all.
3418 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3419 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3420 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3421 DemandedSrcElts =
3422 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3423
3424 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3425 if (BitWidth > EltBitWidth)
3426 Known = Known.anyext(BitWidth);
3427 break;
3428 }
3429 case ISD::INSERT_VECTOR_ELT: {
3430 // If we know the element index, split the demand between the
3431 // source vector and the inserted element, otherwise assume we need
3432 // the original demanded vector elements and the value.
3433 SDValue InVec = Op.getOperand(0);
3434 SDValue InVal = Op.getOperand(1);
3435 SDValue EltNo = Op.getOperand(2);
3436 bool DemandedVal = true;
3437 APInt DemandedVecElts = DemandedElts;
3438 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3439 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3440 unsigned EltIdx = CEltNo->getZExtValue();
3441 DemandedVal = !!DemandedElts[EltIdx];
3442 DemandedVecElts.clearBit(EltIdx);
3443 }
3444 Known.One.setAllBits();
3445 Known.Zero.setAllBits();
3446 if (DemandedVal) {
3447 Known2 = computeKnownBits(InVal, Depth + 1);
3448 Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth));
3449 }
3450 if (!!DemandedVecElts) {
3451 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3452 Known = KnownBits::commonBits(Known, Known2);
3453 }
3454 break;
3455 }
3456 case ISD::BITREVERSE: {
3457 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3458 Known = Known2.reverseBits();
3459 break;
3460 }
3461 case ISD::BSWAP: {
3462 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3463 Known = Known2.byteSwap();
3464 break;
3465 }
3466 case ISD::ABS: {
3467 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3468 Known = Known2.abs();
3469 break;
3470 }
3471 case ISD::USUBSAT: {
3472 // The result of usubsat will never be larger than the LHS.
3473 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3474 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
3475 break;
3476 }
3477 case ISD::UMIN: {
3478 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3479 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3480 Known = KnownBits::umin(Known, Known2);
3481 break;
3482 }
3483 case ISD::UMAX: {
3484 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3485 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3486 Known = KnownBits::umax(Known, Known2);
3487 break;
3488 }
3489 case ISD::SMIN:
3490 case ISD::SMAX: {
3491 // If we have a clamp pattern, we know that the number of sign bits will be
3492 // the minimum of the clamp min/max range.
3493 bool IsMax = (Opcode == ISD::SMAX);
3494 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3495 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3496 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3497 CstHigh =
3498 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3499 if (CstLow && CstHigh) {
3500 if (!IsMax)
3501 std::swap(CstLow, CstHigh);
3502
3503 const APInt &ValueLow = CstLow->getAPIntValue();
3504 const APInt &ValueHigh = CstHigh->getAPIntValue();
3505 if (ValueLow.sle(ValueHigh)) {
3506 unsigned LowSignBits = ValueLow.getNumSignBits();
3507 unsigned HighSignBits = ValueHigh.getNumSignBits();
3508 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3509 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3510 Known.One.setHighBits(MinSignBits);
3511 break;
3512 }
3513 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3514 Known.Zero.setHighBits(MinSignBits);
3515 break;
3516 }
3517 }
3518 }
3519
3520 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3521 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3522 if (IsMax)
3523 Known = KnownBits::smax(Known, Known2);
3524 else
3525 Known = KnownBits::smin(Known, Known2);
3526 break;
3527 }
3528 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
3529 if (Op.getResNo() == 1) {
3530 // The boolean result conforms to getBooleanContents.
3531 // If we know the result of a setcc has the top bits zero, use this info.
3532 // We know that we have an integer-based boolean since these operations
3533 // are only available for integer.
3534 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3535 TargetLowering::ZeroOrOneBooleanContent &&
3536 BitWidth > 1)
3537 Known.Zero.setBitsFrom(1);
3538 break;
3539 }
3540 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3541 case ISD::ATOMIC_CMP_SWAP:
3542 case ISD::ATOMIC_SWAP:
3543 case ISD::ATOMIC_LOAD_ADD:
3544 case ISD::ATOMIC_LOAD_SUB:
3545 case ISD::ATOMIC_LOAD_AND:
3546 case ISD::ATOMIC_LOAD_CLR:
3547 case ISD::ATOMIC_LOAD_OR:
3548 case ISD::ATOMIC_LOAD_XOR:
3549 case ISD::ATOMIC_LOAD_NAND:
3550 case ISD::ATOMIC_LOAD_MIN:
3551 case ISD::ATOMIC_LOAD_MAX:
3552 case ISD::ATOMIC_LOAD_UMIN:
3553 case ISD::ATOMIC_LOAD_UMAX:
3554 case ISD::ATOMIC_LOAD: {
3555 unsigned MemBits =
3556 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
3557 // If we are looking at the loaded value.
3558 if (Op.getResNo() == 0) {
3559 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
3560 Known.Zero.setBitsFrom(MemBits);
3561 }
3562 break;
3563 }
3564 case ISD::FrameIndex:
3565 case ISD::TargetFrameIndex:
3566 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
3567 Known, getMachineFunction());
3568 break;
3569
3570 default:
3571 if (Opcode < ISD::BUILTIN_OP_END)
3572 break;
3573 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3574 case ISD::INTRINSIC_WO_CHAIN:
3575 case ISD::INTRINSIC_W_CHAIN:
3576 case ISD::INTRINSIC_VOID:
3577 // Allow the target to implement this method for its nodes.
3578 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3579 break;
3580 }
3581
3582 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
3583 return Known;
3584}
3585
3586SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3587 SDValue N1) const {
3588 // X + 0 never overflow
3589 if (isNullConstant(N1))
3590 return OFK_Never;
3591
3592 KnownBits N1Known = computeKnownBits(N1);
3593 if (N1Known.Zero.getBoolValue()) {
3594 KnownBits N0Known = computeKnownBits(N0);
3595
3596 bool overflow;
3597 (void)N0Known.getMaxValue().uadd_ov(N1Known.getMaxValue(), overflow);
3598 if (!overflow)
3599 return OFK_Never;
3600 }
3601
3602 // mulhi + 1 never overflow
3603 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
3604 (N1Known.getMaxValue() & 0x01) == N1Known.getMaxValue())
3605 return OFK_Never;
3606
3607 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
3608 KnownBits N0Known = computeKnownBits(N0);
3609
3610 if ((N0Known.getMaxValue() & 0x01) == N0Known.getMaxValue())
3611 return OFK_Never;
3612 }
3613
3614 return OFK_Sometime;
3615}
3616
3617bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3618 EVT OpVT = Val.getValueType();
3619 unsigned BitWidth = OpVT.getScalarSizeInBits();
3620
3621 // Is the constant a known power of 2?
3622 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3623 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3624
3625 // A left-shift of a constant one will have exactly one bit set because
3626 // shifting the bit off the end is undefined.
3627 if (Val.getOpcode() == ISD::SHL) {
3628 auto *C = isConstOrConstSplat(Val.getOperand(0));
3629 if (C && C->getAPIntValue() == 1)
3630 return true;
3631 }
3632
3633 // Similarly, a logical right-shift of a constant sign-bit will have exactly
3634 // one bit set.
3635 if (Val.getOpcode() == ISD::SRL) {
3636 auto *C = isConstOrConstSplat(Val.getOperand(0));
3637 if (C && C->getAPIntValue().isSignMask())
3638 return true;
3639 }
3640
3641 // Are all operands of a build vector constant powers of two?
3642 if (Val.getOpcode() == ISD::BUILD_VECTOR)
3643 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
3644 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3645 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3646 return false;
3647 }))
3648 return true;
3649
3650 // More could be done here, though the above checks are enough
3651 // to handle some common cases.
3652
3653 // Fall back to computeKnownBits to catch other known cases.
3654 KnownBits Known = computeKnownBits(Val);
3655 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
3656}
3657
3658unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3659 EVT VT = Op.getValueType();
3660
3661 // TODO: Assume we don't know anything for now.
3662 if (VT.isScalableVector())
3663 return 1;
3664
3665 APInt DemandedElts = VT.isVector()
3666 ? APInt::getAllOnesValue(VT.getVectorNumElements())
3667 : APInt(1, 1);
3668 return ComputeNumSignBits(Op, DemandedElts, Depth);
3669}
3670
3671unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3672 unsigned Depth) const {
3673 EVT VT = Op.getValueType();
3674 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!")((void)0);
3675 unsigned VTBits = VT.getScalarSizeInBits();
3676 unsigned NumElts = DemandedElts.getBitWidth();
3677 unsigned Tmp, Tmp2;
3678 unsigned FirstAnswer = 1;
3679
3680 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3681 const APInt &Val = C->getAPIntValue();
3682 return Val.getNumSignBits();
3683 }
3684
3685 if (Depth >= MaxRecursionDepth)
3686 return 1; // Limit search depth.
3687
3688 if (!DemandedElts || VT.isScalableVector())
3689 return 1; // No demanded elts, better to assume we don't know anything.
3690
3691 unsigned Opcode = Op.getOpcode();
3692 switch (Opcode) {
3693 default: break;
3694 case ISD::AssertSext:
3695 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3696 return VTBits-Tmp+1;
3697 case ISD::AssertZext:
3698 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3699 return VTBits-Tmp;
3700
3701 case ISD::BUILD_VECTOR:
3702 Tmp = VTBits;
3703 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
3704 if (!DemandedElts[i])
3705 continue;
3706
3707 SDValue SrcOp = Op.getOperand(i);
3708 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
3709
3710 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3711 if (SrcOp.getValueSizeInBits() != VTBits) {
3712 assert(SrcOp.getValueSizeInBits() > VTBits &&((void)0)
3713 "Expected BUILD_VECTOR implicit truncation")((void)0);
3714 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3715 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
3716 }
3717 Tmp = std::min(Tmp, Tmp2);
3718 }
3719 return Tmp;
3720
3721 case ISD::VECTOR_SHUFFLE: {
3722 // Collect the minimum number of sign bits that are shared by every vector
3723 // element referenced by the shuffle.
3724 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3725 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3726 assert(NumElts == SVN->getMask().size() && "Unexpected vector size")((void)0);
3727 for (unsigned i = 0; i != NumElts; ++i) {
3728 int M = SVN->getMaskElt(i);
3729 if (!DemandedElts[i])
3730 continue;
3731 // For UNDEF elements, we don't know anything about the common state of
3732 // the shuffle result.
3733 if (M < 0)
3734 return 1;
3735 if ((unsigned)M < NumElts)
3736 DemandedLHS.setBit((unsigned)M % NumElts);
3737 else
3738 DemandedRHS.setBit((unsigned)M % NumElts);
3739 }
3740 Tmp = std::numeric_limits<unsigned>::max();
3741 if (!!DemandedLHS)
3742 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3743 if (!!DemandedRHS) {
3744 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3745 Tmp = std::min(Tmp, Tmp2);
3746 }
3747 // If we don't know anything, early out and try computeKnownBits fall-back.
3748 if (Tmp == 1)
3749 break;
3750 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((void)0);
3751 return Tmp;
3752 }
3753
3754 case ISD::BITCAST: {
3755 SDValue N0 = Op.getOperand(0);
3756 EVT SrcVT = N0.getValueType();
3757 unsigned SrcBits = SrcVT.getScalarSizeInBits();
3758
3759 // Ignore bitcasts from unsupported types..
3760 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
3761 break;
3762
3763 // Fast handling of 'identity' bitcasts.
3764 if (VTBits == SrcBits)
3765 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3766
3767 bool IsLE = getDataLayout().isLittleEndian();
3768
3769 // Bitcast 'large element' scalar/vector to 'small element' vector.
3770 if ((SrcBits % VTBits) == 0) {
3771 assert(VT.isVector() && "Expected bitcast to vector")((void)0);
3772
3773 unsigned Scale = SrcBits / VTBits;
3774 APInt SrcDemandedElts(NumElts / Scale, 0);
3775 for (unsigned i = 0; i != NumElts; ++i)
3776 if (DemandedElts[i])
3777 SrcDemandedElts.setBit(i / Scale);
3778
3779 // Fast case - sign splat can be simply split across the small elements.
3780 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3781 if (Tmp == SrcBits)
3782 return VTBits;
3783
3784 // Slow case - determine how far the sign extends into each sub-element.
3785 Tmp2 = VTBits;
3786 for (unsigned i = 0; i != NumElts; ++i)
3787 if (DemandedElts[i]) {
3788 unsigned SubOffset = i % Scale;
3789 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
3790 SubOffset = SubOffset * VTBits;
3791 if (Tmp <= SubOffset)
3792 return 1;
3793 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3794 }
3795 return Tmp2;
3796 }
3797 break;
3798 }
3799
3800 case ISD::SIGN_EXTEND:
3801 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3802 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3803 case ISD::SIGN_EXTEND_INREG:
3804 // Max of the input and what this extends.
3805 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3806 Tmp = VTBits-Tmp+1;
3807 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3808 return std::max(Tmp, Tmp2);
3809 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3810 SDValue Src = Op.getOperand(0);
3811 EVT SrcVT = Src.getValueType();
3812 APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3813 Tmp = VTBits - SrcVT.getScalarSizeInBits();
3814 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3815 }
3816 case ISD::SRA:
3817 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3818 // SRA X, C -> adds C sign bits.
3819 if (const APInt *ShAmt =
3820 getValidMinimumShiftAmountConstant(Op, DemandedElts))
3821 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
3822 return Tmp;
3823 case ISD::SHL:
3824 if (const APInt *ShAmt =
3825 getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
3826 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
3827 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3828 if (ShAmt->ult(Tmp))
3829 return Tmp - ShAmt->getZExtValue();
3830 }
3831 break;
3832 case ISD::AND:
3833 case ISD::OR:
3834 case ISD::XOR: // NOT is handled here.
3835 // Logical binary ops preserve the number of sign bits at the worst.
3836 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3837 if (Tmp != 1) {
3838 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3839 FirstAnswer = std::min(Tmp, Tmp2);
3840 // We computed what we know about the sign bits as our first
3841 // answer. Now proceed to the generic code that uses
3842 // computeKnownBits, and pick whichever answer is better.
3843 }
3844 break;
3845
3846 case ISD::SELECT:
3847 case ISD::VSELECT:
3848 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3849 if (Tmp == 1) return 1; // Early out.
3850 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3851 return std::min(Tmp, Tmp2);
3852 case ISD::SELECT_CC:
3853 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3854 if (Tmp == 1) return 1; // Early out.
3855 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3856 return std::min(Tmp, Tmp2);
3857
3858 case ISD::SMIN:
3859 case ISD::SMAX: {
3860 // If we have a clamp pattern, we know that the number of sign bits will be
3861 // the minimum of the clamp min/max range.
3862 bool IsMax = (Opcode == ISD::SMAX);
3863 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3864 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3865 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3866 CstHigh =
3867 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3868 if (CstLow && CstHigh) {
3869 if (!IsMax)
3870 std::swap(CstLow, CstHigh);
3871 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3872 Tmp = CstLow->getAPIntValue().getNumSignBits();
3873 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3874 return std::min(Tmp, Tmp2);
3875 }
3876 }
3877
3878 // Fallback - just get the minimum number of sign bits of the operands.
3879 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3880 if (Tmp == 1)
3881 return 1; // Early out.
3882 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3883 return std::min(Tmp, Tmp2);
3884 }
3885 case ISD::UMIN:
3886 case ISD::UMAX:
3887 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3888 if (Tmp == 1)
3889 return 1; // Early out.
3890 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3891 return std::min(Tmp, Tmp2);
3892 case ISD::SADDO:
3893 case ISD::UADDO:
3894 case ISD::SSUBO:
3895 case ISD::USUBO:
3896 case ISD::SMULO:
3897 case ISD::UMULO:
3898 if (Op.getResNo() != 1)
3899 break;
3900 // The boolean result conforms to getBooleanContents. Fall through.
3901 // If setcc returns 0/-1, all bits are sign bits.
3902 // We know that we have an integer-based boolean since these operations
3903 // are only available for integer.
3904 if (TLI->getBooleanContents(VT.isVector(), false) ==
3905 TargetLowering::ZeroOrNegativeOneBooleanContent)
3906 return VTBits;
3907 break;
3908 case ISD::SETCC:
3909 case ISD::STRICT_FSETCC:
3910 case ISD::STRICT_FSETCCS: {
3911 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3912 // If setcc returns 0/-1, all bits are sign bits.
3913 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3914 TargetLowering::ZeroOrNegativeOneBooleanContent)
3915 return VTBits;
3916 break;
3917 }
3918 case ISD::ROTL:
3919 case ISD::ROTR:
3920 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3921
3922 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
3923 if (Tmp == VTBits)
3924 return VTBits;
3925
3926 if (ConstantSDNode *C =
3927 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3928 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3929
3930 // Handle rotate right by N like a rotate left by 32-N.
3931 if (Opcode == ISD::ROTR)
3932 RotAmt = (VTBits - RotAmt) % VTBits;
3933
3934 // If we aren't rotating out all of the known-in sign bits, return the
3935 // number that are left. This handles rotl(sext(x), 1) for example.
3936 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
3937 }
3938 break;
3939 case ISD::ADD:
3940 case ISD::ADDC:
3941 // Add can have at most one carry bit. Thus we know that the output
3942 // is, at worst, one more bit than the inputs.
3943 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3944 if (Tmp == 1) return 1; // Early out.
3945
3946 // Special case decrementing a value (ADD X, -1):
3947 if (ConstantSDNode *CRHS =
3948 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
3949 if (CRHS->isAllOnesValue()) {
3950 KnownBits Known =
3951 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3952
3953 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3954 // sign bits set.
3955 if ((Known.Zero | 1).isAllOnesValue())
3956 return VTBits;
3957
3958 // If we are subtracting one from a positive number, there is no carry
3959 // out of the result.
3960 if (Known.isNonNegative())
3961 return Tmp;
3962 }
3963
3964 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3965 if (Tmp2 == 1) return 1; // Early out.
3966 return std::min(Tmp, Tmp2) - 1;
3967 case ISD::SUB:
3968 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
3969 if (Tmp2 == 1) return 1; // Early out.
3970
3971 // Handle NEG.
3972 if (ConstantSDNode *CLHS =
3973 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
3974 if (CLHS->isNullValue()) {
3975 KnownBits Known =
3976 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3977 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3978 // sign bits set.
3979 if ((Known.Zero | 1).isAllOnesValue())
3980 return VTBits;
3981
3982 // If the input is known to be positive (the sign bit is known clear),
3983 // the output of the NEG has the same number of sign bits as the input.
3984 if (Known.isNonNegative())
3985 return Tmp2;
3986
3987 // Otherwise, we treat this like a SUB.
3988 }
3989
3990 // Sub can have at most one carry bit. Thus we know that the output
3991 // is, at worst, one more bit than the inputs.
3992 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
3993 if (Tmp == 1) return 1; // Early out.
3994 return std::min(Tmp, Tmp2) - 1;
3995 case ISD::MUL: {
3996 // The output of the Mul can be at most twice the valid bits in the inputs.
3997 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3998 if (SignBitsOp0 == 1)
3999 break;
4000 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
4001 if (SignBitsOp1 == 1)
4002 break;
4003 unsigned OutValidBits =
4004 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4005 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4006 }
4007 case ISD::SREM:
4008 // The sign bit is the LHS's sign bit, except when the result of the
4009 // remainder is zero. The magnitude of the result should be less than or
4010 // equal to the magnitude of the LHS. Therefore, the result should have
4011 // at least as many sign bits as the left hand side.
4012 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4013 case ISD::TRUNCATE: {
4014 // Check if the sign bits of source go down as far as the truncated value.
4015 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
4016 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4017 if (NumSrcSignBits > (NumSrcBits - VTBits))
4018 return NumSrcSignBits - (NumSrcBits - VTBits);
4019 break;
4020 }
4021 case ISD::EXTRACT_ELEMENT: {
4022 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
4023 const int BitWidth = Op.getValueSizeInBits();
4024 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
4025
4026 // Get reverse index (starting from 1), Op1 value indexes elements from
4027 // little end. Sign starts at big end.
4028 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
4029
4030 // If the sign portion ends in our element the subtraction gives correct
4031 // result. Otherwise it gives either negative or > bitwidth result
4032 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
4033 }
4034 case ISD::INSERT_VECTOR_ELT: {
4035 // If we know the element index, split the demand between the
4036 // source vector and the inserted element, otherwise assume we need
4037 // the original demanded vector elements and the value.
4038 SDValue InVec = Op.getOperand(0);
4039 SDValue InVal = Op.getOperand(1);
4040 SDValue EltNo = Op.getOperand(2);
4041 bool DemandedVal = true;
4042 APInt DemandedVecElts = DemandedElts;
4043 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4044 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4045 unsigned EltIdx = CEltNo->getZExtValue();
4046 DemandedVal = !!DemandedElts[EltIdx];
4047 DemandedVecElts.clearBit(EltIdx);
4048 }
4049 Tmp = std::numeric_limits<unsigned>::max();
4050 if (DemandedVal) {
4051 // TODO - handle implicit truncation of inserted elements.
4052 if (InVal.getScalarValueSizeInBits() != VTBits)
4053 break;
4054 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
4055 Tmp = std::min(Tmp, Tmp2);
4056 }
4057 if (!!DemandedVecElts) {
4058 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
4059 Tmp = std::min(Tmp, Tmp2);
4060 }
4061 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((void)0);
4062 return Tmp;
4063 }
4064 case ISD::EXTRACT_VECTOR_ELT: {
4065 SDValue InVec = Op.getOperand(0);
4066 SDValue EltNo = Op.getOperand(1);
4067 EVT VecVT = InVec.getValueType();
4068 // ComputeNumSignBits not yet implemented for scalable vectors.
4069 if (VecVT.isScalableVector())
4070 break;
4071 const unsigned BitWidth = Op.getValueSizeInBits();
4072 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
4073 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4074
4075 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4076 // anything about sign bits. But if the sizes match we can derive knowledge
4077 // about sign bits from the vector operand.
4078 if (BitWidth != EltBitWidth)
4079 break;
4080
4081 // If we know the element index, just demand that vector element, else for
4082 // an unknown element index, ignore DemandedElts and demand them all.
4083 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
4084 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4085 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4086 DemandedSrcElts =
4087 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4088
4089 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
4090 }
4091 case ISD::EXTRACT_SUBVECTOR: {
4092 // Offset the demanded elts by the subvector index.
4093 SDValue Src = Op.getOperand(0);
4094 // Bail until we can represent demanded elements for scalable vectors.
4095 if (Src.getValueType().isScalableVector())
4096 break;
4097 uint64_t Idx = Op.getConstantOperandVal(1);
4098 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4099 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
4100 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4101 }
4102 case ISD::CONCAT_VECTORS: {
4103 // Determine the minimum number of sign bits across all demanded
4104 // elts of the input vectors. Early out if the result is already 1.
4105 Tmp = std::numeric_limits<unsigned>::max();
4106 EVT SubVectorVT = Op.getOperand(0).getValueType();
4107 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4108 unsigned NumSubVectors = Op.getNumOperands();
4109 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4110 APInt DemandedSub =
4111 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
4112 if (!DemandedSub)
4113 continue;
4114 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
4115 Tmp = std::min(Tmp, Tmp2);
4116 }
4117 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((void)0);
4118 return Tmp;
4119 }
4120 case ISD::INSERT_SUBVECTOR: {
4121 // Demand any elements from the subvector and the remainder from the src its
4122 // inserted into.
4123 SDValue Src = Op.getOperand(0);
4124 SDValue Sub = Op.getOperand(1);
4125 uint64_t Idx = Op.getConstantOperandVal(2);
4126 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4127 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
4128 APInt DemandedSrcElts = DemandedElts;
4129 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
4130
4131 Tmp = std::numeric_limits<unsigned>::max();
4132 if (!!DemandedSubElts) {
4133 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
4134 if (Tmp == 1)
4135 return 1; // early-out
4136 }
4137 if (!!DemandedSrcElts) {
4138 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4139 Tmp = std::min(Tmp, Tmp2);
4140 }
4141 assert(Tmp <= VTBits && "Failed to determine minimum sign bits")((void)0);
4142 return Tmp;
4143 }
4144 case ISD::ATOMIC_CMP_SWAP:
4145 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4146 case ISD::ATOMIC_SWAP:
4147 case ISD::ATOMIC_LOAD_ADD:
4148 case ISD::ATOMIC_LOAD_SUB:
4149 case ISD::ATOMIC_LOAD_AND:
4150 case ISD::ATOMIC_LOAD_CLR:
4151 case ISD::ATOMIC_LOAD_OR:
4152 case ISD::ATOMIC_LOAD_XOR:
4153 case ISD::ATOMIC_LOAD_NAND:
4154 case ISD::ATOMIC_LOAD_MIN:
4155 case ISD::ATOMIC_LOAD_MAX:
4156 case ISD::ATOMIC_LOAD_UMIN:
4157 case ISD::ATOMIC_LOAD_UMAX:
4158 case ISD::ATOMIC_LOAD: {
4159 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4160 // If we are looking at the loaded value.
4161 if (Op.getResNo() == 0) {
4162 if (Tmp == VTBits)
4163 return 1; // early-out
4164 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
4165 return VTBits - Tmp + 1;
4166 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4167 return VTBits - Tmp;
4168 }
4169 break;
4170 }
4171 }
4172
4173 // If we are looking at the loaded value of the SDNode.
4174 if (Op.getResNo() == 0) {
4175 // Handle LOADX separately here. EXTLOAD case will fallthrough.
4176 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
4177 unsigned ExtType = LD->getExtensionType();
4178 switch (ExtType) {
4179 default: break;
4180 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4181 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4182 return VTBits - Tmp + 1;
4183 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4184 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4185 return VTBits - Tmp;
4186 case ISD::NON_EXTLOAD:
4187 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4188 // We only need to handle vectors - computeKnownBits should handle
4189 // scalar cases.
4190 Type *CstTy = Cst->getType();
4191 if (CstTy->isVectorTy() &&
4192 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) {
4193 Tmp = VTBits;
4194 for (unsigned i = 0; i != NumElts; ++i) {
4195 if (!DemandedElts[i])
4196 continue;
4197 if (Constant *Elt = Cst->getAggregateElement(i)) {
4198 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4199 const APInt &Value = CInt->getValue();
4200 Tmp = std::min(Tmp, Value.getNumSignBits());
4201 continue;
4202 }
4203 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4204 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4205 Tmp = std::min(Tmp, Value.getNumSignBits());
4206 continue;
4207 }
4208 }
4209 // Unknown type. Conservatively assume no bits match sign bit.
4210 return 1;
4211 }
4212 return Tmp;
4213 }
4214 }
4215 break;
4216 }
4217 }
4218 }
4219
4220 // Allow the target to implement this method for its nodes.
4221 if (Opcode >= ISD::BUILTIN_OP_END ||
4222 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4223 Opcode == ISD::INTRINSIC_W_CHAIN ||
4224 Opcode == ISD::INTRINSIC_VOID) {
4225 unsigned NumBits =
4226 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4227 if (NumBits > 1)
4228 FirstAnswer = std::max(FirstAnswer, NumBits);
4229 }
4230
4231 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4232 // use this information.
4233 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4234
4235 APInt Mask;
4236 if (Known.isNonNegative()) { // sign bit is 0
4237 Mask = Known.Zero;
4238 } else if (Known.isNegative()) { // sign bit is 1;
4239 Mask = Known.One;
4240 } else {
4241 // Nothing known.
4242 return FirstAnswer;
4243 }
4244
4245 // Okay, we know that the sign bit in Mask is set. Use CLO to determine
4246 // the number of identical bits in the top of the input value.
4247 Mask <<= Mask.getBitWidth()-VTBits;
4248 return std::max(FirstAnswer, Mask.countLeadingOnes());
4249}
4250
4251bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
4252 unsigned Depth) const {
4253 // Early out for FREEZE.
4254 if (Op.getOpcode() == ISD::FREEZE)
4255 return true;
4256
4257 // TODO: Assume we don't know anything for now.
4258 EVT VT = Op.getValueType();
4259 if (VT.isScalableVector())
4260 return false;
4261
4262 APInt DemandedElts = VT.isVector()
4263 ? APInt::getAllOnesValue(VT.getVectorNumElements())
4264 : APInt(1, 1);
4265 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4266}
4267
4268bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
4269 const APInt &DemandedElts,
4270 bool PoisonOnly,
4271 unsigned Depth) const {
4272 unsigned Opcode = Op.getOpcode();
4273
4274 // Early out for FREEZE.
4275 if (Opcode == ISD::FREEZE)
4276 return true;
4277
4278 if (Depth >= MaxRecursionDepth)
4279 return false; // Limit search depth.
4280
4281 if (isIntOrFPConstant(Op))
4282 return true;
4283
4284 switch (Opcode) {
4285 case ISD::UNDEF:
4286 return PoisonOnly;
4287
4288 // TODO: ISD::BUILD_VECTOR handling
4289
4290 // TODO: Search for noundef attributes from library functions.
4291
4292 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
4293
4294 default:
4295 // Allow the target to implement this method for its nodes.
4296 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
4297 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
4298 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4299 Op, DemandedElts, *this, PoisonOnly, Depth);
4300 break;
4301 }
4302
4303 return false;
4304}
4305
4306bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
4307 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
4308 !isa<ConstantSDNode>(Op.getOperand(1)))
4309 return false;
4310
4311 if (Op.getOpcode() == ISD::OR &&
4312 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
4313 return false;
4314
4315 return true;
4316}
4317
4318bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
4319 // If we're told that NaNs won't happen, assume they won't.
4320 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
4321 return true;
4322
4323 if (Depth >= MaxRecursionDepth)
4324 return false; // Limit search depth.
4325
4326 // TODO: Handle vectors.
4327 // If the value is a constant, we can obviously see if it is a NaN or not.
4328 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
4329 return !C->getValueAPF().isNaN() ||
4330 (SNaN && !C->getValueAPF().isSignaling());
4331 }
4332
4333 unsigned Opcode = Op.getOpcode();
4334 switch (Opcode) {
4335 case ISD::FADD:
4336 case ISD::FSUB:
4337 case ISD::FMUL:
4338 case ISD::FDIV:
4339 case ISD::FREM:
4340 case ISD::FSIN:
4341 case ISD::FCOS: {
4342 if (SNaN)
4343 return true;
4344 // TODO: Need isKnownNeverInfinity
4345 return false;
4346 }
4347 case ISD::FCANONICALIZE:
4348 case ISD::FEXP:
4349 case ISD::FEXP2:
4350 case ISD::FTRUNC:
4351 case ISD::FFLOOR:
4352 case ISD::FCEIL:
4353 case ISD::FROUND:
4354 case ISD::FROUNDEVEN:
4355 case ISD::FRINT:
4356 case ISD::FNEARBYINT: {
4357 if (SNaN)
4358 return true;
4359 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4360 }
4361 case ISD::FABS:
4362 case ISD::FNEG:
4363 case ISD::FCOPYSIGN: {
4364 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4365 }
4366 case ISD::SELECT:
4367 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4368 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4369 case ISD::FP_EXTEND:
4370 case ISD::FP_ROUND: {
4371 if (SNaN)
4372 return true;
4373 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4374 }
4375 case ISD::SINT_TO_FP:
4376 case ISD::UINT_TO_FP:
4377 return true;
4378 case ISD::FMA:
4379 case ISD::FMAD: {
4380 if (SNaN)
4381 return true;
4382 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4383 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4384 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4385 }
4386 case ISD::FSQRT: // Need is known positive
4387 case ISD::FLOG:
4388 case ISD::FLOG2:
4389 case ISD::FLOG10:
4390 case ISD::FPOWI:
4391 case ISD::FPOW: {
4392 if (SNaN)
4393 return true;
4394 // TODO: Refine on operand
4395 return false;
4396 }
4397 case ISD::FMINNUM:
4398 case ISD::FMAXNUM: {
4399 // Only one needs to be known not-nan, since it will be returned if the
4400 // other ends up being one.
4401 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
4402 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4403 }
4404 case ISD::FMINNUM_IEEE:
4405 case ISD::FMAXNUM_IEEE: {
4406 if (SNaN)
4407 return true;
4408 // This can return a NaN if either operand is an sNaN, or if both operands
4409 // are NaN.
4410 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
4411 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
4412 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
4413 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
4414 }
4415 case ISD::FMINIMUM:
4416 case ISD::FMAXIMUM: {
4417 // TODO: Does this quiet or return the origina NaN as-is?
4418 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4419 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4420 }
4421 case ISD::EXTRACT_VECTOR_ELT: {
4422 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4423 }
4424 default:
4425 if (Opcode >= ISD::BUILTIN_OP_END ||
4426 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4427 Opcode == ISD::INTRINSIC_W_CHAIN ||
4428 Opcode == ISD::INTRINSIC_VOID) {
4429 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
4430 }
4431
4432 return false;
4433 }
4434}
4435
4436bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4437 assert(Op.getValueType().isFloatingPoint() &&((void)0)
4438 "Floating point type expected")((void)0);
4439
4440 // If the value is a constant, we can obviously see if it is a zero or not.
4441 // TODO: Add BuildVector support.
4442 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
4443 return !C->isZero();
4444 return false;
4445}
4446
4447bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4448 assert(!Op.getValueType().isFloatingPoint() &&((void)0)
4449 "Floating point types unsupported - use isKnownNeverZeroFloat")((void)0);
4450
4451 // If the value is a constant, we can obviously see if it is a zero or not.
4452 if (ISD::matchUnaryPredicate(
2
←
Calling 'matchUnaryPredicate'
→
4453 Op, [](ConstantSDNode *C) { return !C->isNullValue(); }))
1
Value assigned to 'Op.Node'
→
4454 return true;
4455
4456 // TODO: Recognize more cases here.
4457 switch (Op.getOpcode()) {
4458 default: break;
4459 case ISD::OR:
4460 if (isKnownNeverZero(Op.getOperand(1)) ||
4461 isKnownNeverZero(Op.getOperand(0)))
4462 return true;
4463 break;
4464 }
4465
4466 return false;
4467}
4468
4469bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4470 // Check the obvious case.
4471 if (A == B) return true;
4472
4473 // For for negative and positive zero.
4474 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
4475 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
4476 if (CA->isZero() && CB->isZero()) return true;
4477
4478 // Otherwise they may not be equal.
4479 return false;
4480}
4481
4482// FIXME: unify with llvm::haveNoCommonBitsSet.
4483// FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
4484bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4485 assert(A.getValueType() == B.getValueType() &&((void)0)
4486 "Values must have the same type")((void)0);
4487 return KnownBits::haveNoCommonBitsSet(computeKnownBits(A),
4488 computeKnownBits(B));
4489}
4490
4491static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
4492 SelectionDAG &DAG) {
4493 if (cast<ConstantSDNode>(Step)->isNullValue())
4494 return DAG.getConstant(0, DL, VT);
4495
4496 return SDValue();
4497}
4498
4499static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4500 ArrayRef<SDValue> Ops,
4501 SelectionDAG &DAG) {
4502 int NumOps = Ops.size();
4503 assert(NumOps != 0 && "Can't build an empty vector!")((void)0);
4504 assert(!VT.isScalableVector() &&((void)0)
4505 "BUILD_VECTOR cannot be used with scalable types")((void)0);
4506 assert(VT.getVectorNumElements() == (unsigned)NumOps &&((void)0)
4507 "Incorrect element count in BUILD_VECTOR!")((void)0);
4508
4509 // BUILD_VECTOR of UNDEFs is UNDEF.
4510 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4511 return DAG.getUNDEF(VT);
4512
4513 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
4514 SDValue IdentitySrc;
4515 bool IsIdentity = true;
4516 for (int i = 0; i != NumOps; ++i) {
4517 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4518 Ops[i].getOperand(0).getValueType() != VT ||
4519 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
4520 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
4521 cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) {
4522 IsIdentity = false;
4523 break;
4524 }
4525 IdentitySrc = Ops[i].getOperand(0);
4526 }
4527 if (IsIdentity)
4528 return IdentitySrc;
4529
4530 return SDValue();
4531}
4532
4533/// Try to simplify vector concatenation to an input value, undef, or build
4534/// vector.
4535static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4536 ArrayRef<SDValue> Ops,
4537 SelectionDAG &DAG) {
4538 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!")((void)0);
4539 assert(llvm::all_of(Ops,((void)0)
4540 [Ops](SDValue Op) {((void)0)
4541 return Ops[0].getValueType() == Op.getValueType();((void)0)
4542 }) &&((void)0)
4543 "Concatenation of vectors with inconsistent value types!")((void)0);
4544 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==((void)0)
4545 VT.getVectorElementCount() &&((void)0)
4546 "Incorrect element count in vector concatenation!")((void)0);
4547
4548 if (Ops.size() == 1)
4549 return Ops[0];
4550
4551 // Concat of UNDEFs is UNDEF.
4552 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4553 return DAG.getUNDEF(VT);
4554
4555 // Scan the operands and look for extract operations from a single source
4556 // that correspond to insertion at the same location via this concatenation:
4557 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
4558 SDValue IdentitySrc;
4559 bool IsIdentity = true;
4560 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
4561 SDValue Op = Ops[i];
4562 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
4563 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4564 Op.getOperand(0).getValueType() != VT ||
4565 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
4566 Op.getConstantOperandVal(1) != IdentityIndex) {
4567 IsIdentity = false;
4568 break;
4569 }
4570 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&((void)0)
4571 "Unexpected identity source vector for concat of extracts")((void)0);
4572 IdentitySrc = Op.getOperand(0);
4573 }
4574 if (IsIdentity) {
4575 assert(IdentitySrc && "Failed to set source vector of extracts")((void)0);
4576 return IdentitySrc;
4577 }
4578
4579 // The code below this point is only designed to work for fixed width
4580 // vectors, so we bail out for now.
4581 if (VT.isScalableVector())
4582 return SDValue();
4583
4584 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
4585 // simplified to one big BUILD_VECTOR.
4586 // FIXME: Add support for SCALAR_TO_VECTOR as well.
4587 EVT SVT = VT.getScalarType();
4588 SmallVector<SDValue, 16> Elts;
4589 for (SDValue Op : Ops) {
4590 EVT OpVT = Op.getValueType();
4591 if (Op.isUndef())
4592 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
4593 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
4594 Elts.append(Op->op_begin(), Op->op_end());
4595 else
4596 return SDValue();
4597 }
4598
4599 // BUILD_VECTOR requires all inputs to be of the same type, find the
4600 // maximum type and extend them all.
4601 for (SDValue Op : Elts)
4602 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
4603
4604 if (SVT.bitsGT(VT.getScalarType())) {
4605 for (SDValue &Op : Elts) {
4606 if (Op.isUndef())
4607 Op = DAG.getUNDEF(SVT);
4608 else
4609 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
4610 ? DAG.getZExtOrTrunc(Op, DL, SVT)
4611 : DAG.getSExtOrTrunc(Op, DL, SVT);
4612 }
4613 }
4614
4615 SDValue V = DAG.getBuildVector(VT, DL, Elts);
4616 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
4617 return V;
4618}
4619
4620/// Gets or creates the specified node.
4621SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4622 FoldingSetNodeID ID;
4623 AddNodeIDNode(ID, Opcode, getVTList(VT), None);
4624 void *IP = nullptr;
4625 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4626 return SDValue(E, 0);
4627
4628 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4629 getVTList(VT));
4630 CSEMap.InsertNode(N, IP);
4631
4632 InsertNode(N);
4633 SDValue V = SDValue(N, 0);
4634 NewSDValueDbgMsg(V, "Creating new node: ", this);
4635 return V;
4636}
4637
4638SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4639 SDValue Operand) {
4640 SDNodeFlags Flags;
4641 if (Inserter)
4642 Flags = Inserter->getFlags();
4643 return getNode(Opcode, DL, VT, Operand, Flags);
4644}
4645
4646SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4647 SDValue Operand, const SDNodeFlags Flags) {
4648 assert(Operand.getOpcode() != ISD::DELETED_NODE &&((void)0)
4649 "Operand is DELETED_NODE!")((void)0);
4650 // Constant fold unary operations with an integer constant operand. Even
4651 // opaque constant will be folded, because the folding of unary operations
4652 // doesn't create new constants with different values. Nevertheless, the
4653 // opaque flag is preserved during folding to prevent future folding with
4654 // other constants.
4655 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
4656 const APInt &Val = C->getAPIntValue();
4657 switch (Opcode) {
4658 default: break;
4659 case ISD::SIGN_EXTEND:
4660 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4661 C->isTargetOpcode(), C->isOpaque());
4662 case ISD::TRUNCATE:
4663 if (C->isOpaque())
4664 break;
4665 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4666 case ISD::ZERO_EXTEND:
4667 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4668 C->isTargetOpcode(), C->isOpaque());
4669 case ISD::ANY_EXTEND:
4670 // Some targets like RISCV prefer to sign extend some types.
4671 if (TLI->isSExtCheaperThanZExt(Operand.getValueType(), VT))
4672 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4673 C->isTargetOpcode(), C->isOpaque());
4674 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4675 C->isTargetOpcode(), C->isOpaque());
4676 case ISD::UINT_TO_FP:
4677 case ISD::SINT_TO_FP: {
4678 APFloat apf(EVTToAPFloatSemantics(VT),
4679 APInt::getNullValue(VT.getSizeInBits()));
4680 (void)apf.convertFromAPInt(Val,
4681 Opcode==ISD::SINT_TO_FP,
4682 APFloat::rmNearestTiesToEven);
4683 return getConstantFP(apf, DL, VT);
4684 }
4685 case ISD::BITCAST:
4686 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
4687 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
4688 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
4689 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
4690 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
4691 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
4692 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
4693 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
4694 break;
4695 case ISD::ABS:
4696 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
4697 C->isOpaque());
4698 case ISD::BITREVERSE:
4699 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
4700 C->isOpaque());
4701 case ISD::BSWAP:
4702 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
4703 C->isOpaque());
4704 case ISD::CTPOP:
4705 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
4706 C->isOpaque());
4707 case ISD::CTLZ:
4708 case ISD::CTLZ_ZERO_UNDEF:
4709 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
4710 C->isOpaque());
4711 case ISD::CTTZ:
4712 case ISD::CTTZ_ZERO_UNDEF:
4713 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
4714 C->isOpaque());
4715 case ISD::FP16_TO_FP: {
4716 bool Ignored;
4717 APFloat FPV(APFloat::IEEEhalf(),
4718 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
4719
4720 // This can return overflow, underflow, or inexact; we don't care.
4721 // FIXME need to be more flexible about rounding mode.
4722 (void)FPV.convert(EVTToAPFloatSemantics(VT),
4723 APFloat::rmNearestTiesToEven, &Ignored);
4724 return getConstantFP(FPV, DL, VT);
4725 }
4726 case ISD::STEP_VECTOR: {
4727 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Operand, *this))
4728 return V;
4729 break;
4730 }
4731 }
4732 }
4733
4734 // Constant fold unary operations with a floating point constant operand.
4735 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
4736 APFloat V = C->getValueAPF(); // make copy
4737 switch (Opcode) {
4738 case ISD::FNEG:
4739 V.changeSign();
4740 return getConstantFP(V, DL, VT);
4741 case ISD::FABS:
4742 V.clearSign();
4743 return getConstantFP(V, DL, VT);
4744 case ISD::FCEIL: {
4745 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
4746 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4747 return getConstantFP(V, DL, VT);
4748 break;
4749 }
4750 case ISD::FTRUNC: {
4751 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
4752 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4753 return getConstantFP(V, DL, VT);
4754 break;
4755 }
4756 case ISD::FFLOOR: {
4757 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
4758 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4759 return getConstantFP(V, DL, VT);
4760 break;
4761 }
4762 case ISD::FP_EXTEND: {
4763 bool ignored;
4764 // This can return overflow, underflow, or inexact; we don't care.
4765 // FIXME need to be more flexible about rounding mode.
4766 (void)V.convert(EVTToAPFloatSemantics(VT),
4767 APFloat::rmNearestTiesToEven, &ignored);
4768 return getConstantFP(V, DL, VT);
4769 }
4770 case ISD::FP_TO_SINT:
4771 case ISD::FP_TO_UINT: {
4772 bool ignored;
4773 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
4774 // FIXME need to be more flexible about rounding mode.
4775 APFloat::opStatus s =
4776 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
4777 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
4778 break;
4779 return getConstant(IntVal, DL, VT);
4780 }
4781 case ISD::BITCAST:
4782 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
4783 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4784 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
4785 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4786 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
4787 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4788 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
4789 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4790 break;
4791 case ISD::FP_TO_FP16: {
4792 bool Ignored;
4793 // This can return overflow, underflow, or inexact; we don't care.
4794 // FIXME need to be more flexible about rounding mode.
4795 (void)V.convert(APFloat::IEEEhalf(),
4796 APFloat::rmNearestTiesToEven, &Ignored);
4797 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4798 }
4799 }
4800 }
4801
4802 // Constant fold unary operations with a vector integer or float operand.
4803 switch (Opcode) {
4804 default:
4805 // FIXME: Entirely reasonable to perform folding of other unary
4806 // operations here as the need arises.
4807 break;
4808 case ISD::FNEG:
4809 case ISD::FABS:
4810 case ISD::FCEIL:
4811 case ISD::FTRUNC:
4812 case ISD::FFLOOR:
4813 case ISD::FP_EXTEND:
4814 case ISD::FP_TO_SINT:
4815 case ISD::FP_TO_UINT:
4816 case ISD::TRUNCATE:
4817 case ISD::ANY_EXTEND:
4818 case ISD::ZERO_EXTEND:
4819 case ISD::SIGN_EXTEND:
4820 case ISD::UINT_TO_FP:
4821 case ISD::SINT_TO_FP:
4822 case ISD::ABS:
4823 case ISD::BITREVERSE:
4824 case ISD::BSWAP:
4825 case ISD::CTLZ:
4826 case ISD::CTLZ_ZERO_UNDEF:
4827 case ISD::CTTZ:
4828 case ISD::CTTZ_ZERO_UNDEF:
4829 case ISD::CTPOP: {
4830 SDValue Ops = {Operand};
4831 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4832 return Fold;
4833 }
4834 }
4835
4836 unsigned OpOpcode = Operand.getNode()->getOpcode();
4837 switch (Opcode) {
4838 case ISD::STEP_VECTOR:
4839 assert(VT.isScalableVector() &&((void)0)
4840 "STEP_VECTOR can only be used with scalable types")((void)0);
4841 assert(OpOpcode == ISD::TargetConstant &&((void)0)
4842 VT.getVectorElementType() == Operand.getValueType() &&((void)0)
4843 "Unexpected step operand")((void)0);
4844 break;
4845 case ISD::FREEZE:
4846 assert(VT == Operand.getValueType() && "Unexpected VT!")((void)0);
4847 break;
4848 case ISD::TokenFactor:
4849 case ISD::MERGE_VALUES:
4850 case ISD::CONCAT_VECTORS:
4851 return Operand; // Factor, merge or concat of one node? No need.
4852 case ISD::BUILD_VECTOR: {
4853 // Attempt to simplify BUILD_VECTOR.
4854 SDValue Ops[] = {Operand};
4855 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4856 return V;
4857 break;
4858 }
4859 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node")__builtin_unreachable();
4860 case ISD::FP_EXTEND:
4861 assert(VT.isFloatingPoint() &&((void)0)
4862 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!")((void)0);
4863 if (Operand.getValueType() == VT) return Operand; // noop conversion.
4864 assert((!VT.isVector() ||((void)0)
4865 VT.getVectorElementCount() ==((void)0)
4866 Operand.getValueType().getVectorElementCount()) &&((void)0)
4867 "Vector element count mismatch!")((void)0);
4868 assert(Operand.getValueType().bitsLT(VT) &&((void)0)
4869 "Invalid fpext node, dst < src!")((void)0);
4870 if (Operand.isUndef())
4871 return getUNDEF(VT);
4872 break;
4873 case ISD::FP_TO_SINT:
4874 case ISD::FP_TO_UINT:
4875 if (Operand.isUndef())
4876 return getUNDEF(VT);
4877 break;
4878 case ISD::SINT_TO_FP:
4879 case ISD::UINT_TO_FP:
4880 // [us]itofp(undef) = 0, because the result value is bounded.
4881 if (Operand.isUndef())
4882 return getConstantFP(0.0, DL, VT);
4883 break;
4884 case ISD::SIGN_EXTEND:
4885 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((void)0)
4886 "Invalid SIGN_EXTEND!")((void)0);
4887 assert(VT.isVector() == Operand.getValueType().isVector() &&((void)0)
4888 "SIGN_EXTEND result type type should be vector iff the operand "((void)0)
4889 "type is vector!")((void)0);
4890 if (Operand.getValueType() == VT) return Operand; // noop extension
4891 assert((!VT.isVector() ||((void)0)
4892 VT.getVectorElementCount() ==((void)0)
4893 Operand.getValueType().getVectorElementCount()) &&((void)0)
4894 "Vector element count mismatch!")((void)0);
4895 assert(Operand.getValueType().bitsLT(VT) &&((void)0)
4896 "Invalid sext node, dst < src!")((void)0);
4897 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
4898 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4899 if (OpOpcode == ISD::UNDEF)
4900 // sext(undef) = 0, because the top bits will all be the same.
4901 return getConstant(0, DL, VT);
4902 break;
4903 case ISD::ZERO_EXTEND:
4904 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((void)0)
4905 "Invalid ZERO_EXTEND!")((void)0);
4906 assert(VT.isVector() == Operand.getValueType().isVector() &&((void)0)
4907 "ZERO_EXTEND result type type should be vector iff the operand "((void)0)
4908 "type is vector!")((void)0);
4909 if (Operand.getValueType() == VT) return Operand; // noop extension
4910 assert((!VT.isVector() ||((void)0)
4911 VT.getVectorElementCount() ==((void)0)
4912 Operand.getValueType().getVectorElementCount()) &&((void)0)
4913 "Vector element count mismatch!")((void)0);
4914 assert(Operand.getValueType().bitsLT(VT) &&((void)0)
4915 "Invalid zext node, dst < src!")((void)0);
4916 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
4917 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4918 if (OpOpcode == ISD::UNDEF)
4919 // zext(undef) = 0, because the top bits will be zero.
4920 return getConstant(0, DL, VT);
4921 break;
4922 case ISD::ANY_EXTEND:
4923 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((void)0)
4924 "Invalid ANY_EXTEND!")((void)0);
4925 assert(VT.isVector() == Operand.getValueType().isVector() &&((void)0)
4926 "ANY_EXTEND result type type should be vector iff the operand "((void)0)
4927 "type is vector!")((void)0);
4928 if (Operand.getValueType() == VT) return Operand; // noop extension
4929 assert((!VT.isVector() ||((void)0)
4930 VT.getVectorElementCount() ==((void)0)
4931 Operand.getValueType().getVectorElementCount()) &&((void)0)
4932 "Vector element count mismatch!")((void)0);
4933 assert(Operand.getValueType().bitsLT(VT) &&((void)0)
4934 "Invalid anyext node, dst < src!")((void)0);
4935
4936 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4937 OpOpcode == ISD::ANY_EXTEND)
4938 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
4939 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4940 if (OpOpcode == ISD::UNDEF)
4941 return getUNDEF(VT);
4942
4943 // (ext (trunc x)) -> x
4944 if (OpOpcode == ISD::TRUNCATE) {
4945 SDValue OpOp = Operand.getOperand(0);
4946 if (OpOp.getValueType() == VT) {
4947 transferDbgValues(Operand, OpOp);
4948 return OpOp;
4949 }
4950 }
4951 break;
4952 case ISD::TRUNCATE:
4953 assert(VT.isInteger() && Operand.getValueType().isInteger() &&((void)0)
4954 "Invalid TRUNCATE!")((void)0);
4955 assert(VT.isVector() == Operand.getValueType().isVector() &&((void)0)
4956 "TRUNCATE result type type should be vector iff the operand "((void)0)
4957 "type is vector!")((void)0);
4958 if (Operand.getValueType() == VT) return Operand; // noop truncate
4959 assert((!VT.isVector() ||((void)0)
4960 VT.getVectorElementCount() ==((void)0)
4961 Operand.getValueType().getVectorElementCount()) &&((void)0)
4962 "Vector element count mismatch!")((void)0);
4963 assert(Operand.getValueType().bitsGT(VT) &&((void)0)
4964 "Invalid truncate node, src < dst!")((void)0);
4965 if (OpOpcode == ISD::TRUNCATE)
4966 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4967 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4968 OpOpcode == ISD::ANY_EXTEND) {
4969 // If the source is smaller than the dest, we still need an extend.
4970 if (Operand.getOperand(0).getValueType().getScalarType()
4971 .bitsLT(VT.getScalarType()))
4972 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4973 if (Operand.getOperand(0).getValueType().bitsGT(VT))
4974 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4975 return Operand.getOperand(0);
4976 }
4977 if (OpOpcode == ISD::UNDEF)
4978 return getUNDEF(VT);
4979 break;
4980 case ISD::ANY_EXTEND_VECTOR_INREG:
4981 case ISD::ZERO_EXTEND_VECTOR_INREG:
4982 case ISD::SIGN_EXTEND_VECTOR_INREG:
4983 assert(VT.isVector() && "This DAG node is restricted to vector types.")((void)0);
4984 assert(Operand.getValueType().bitsLE(VT) &&((void)0)
4985 "The input must be the same size or smaller than the result.")((void)0);
4986 assert(VT.getVectorMinNumElements() <((void)0)
4987 Operand.getValueType().getVectorMinNumElements() &&((void)0)
4988 "The destination vector type must have fewer lanes than the input.")((void)0);
4989 break;
4990 case ISD::ABS:
4991 assert(VT.isInteger() && VT == Operand.getValueType() &&((void)0)
4992 "Invalid ABS!")((void)0);
4993 if (OpOpcode == ISD::UNDEF)
4994 return getUNDEF(VT);
4995 break;
4996 case ISD::BSWAP:
4997 assert(VT.isInteger() && VT == Operand.getValueType() &&((void)0)
4998 "Invalid BSWAP!")((void)0);
4999 assert((VT.getScalarSizeInBits() % 16 == 0) &&((void)0)
5000 "BSWAP types must be a multiple of 16 bits!")((void)0);
5001 if (OpOpcode == ISD::UNDEF)
5002 return getUNDEF(VT);
5003 break;
5004 case ISD::BITREVERSE:
5005 assert(VT.isInteger() && VT == Operand.getValueType() &&((void)0)
5006 "Invalid BITREVERSE!")((void)0);
5007 if (OpOpcode == ISD::UNDEF)
5008 return getUNDEF(VT);
5009 break;
5010 case ISD::BITCAST:
5011 // Basic sanity checking.
5012 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&((void)0)
5013 "Cannot BITCAST between types of different sizes!")((void)0);
5014 if (VT == Operand.getValueType()) return Operand; // noop conversion.
5015 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5016 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
5017 if (OpOpcode == ISD::UNDEF)
5018 return getUNDEF(VT);
5019 break;
5020 case ISD::SCALAR_TO_VECTOR:
5021 assert(VT.isVector() && !Operand.getValueType().isVector() &&((void)0)
5022 (VT.getVectorElementType() == Operand.getValueType() ||((void)0)
5023 (VT.getVectorElementType().isInteger() &&((void)0)
5024 Operand.getValueType().isInteger() &&((void)0)
5025 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&((void)0)
5026 "Illegal SCALAR_TO_VECTOR node!")((void)0);
5027 if (OpOpcode == ISD::UNDEF)
5028 return getUNDEF(VT);
5029 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5030 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5031 isa<ConstantSDNode>(Operand.getOperand(1)) &&
5032 Operand.getConstantOperandVal(1) == 0 &&
5033 Operand.getOperand(0).getValueType() == VT)
5034 return Operand.getOperand(0);
5035 break;
5036 case ISD::FNEG:
5037 // Negation of an unknown bag of bits is still completely undefined.
5038 if (OpOpcode == ISD::UNDEF)
5039 return getUNDEF(VT);
5040
5041 if (OpOpcode == ISD::FNEG) // --X -> X
5042 return Operand.getOperand(0);
5043 break;
5044 case ISD::FABS:
5045 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5046 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
5047 break;
5048 case ISD::VSCALE:
5049 assert(VT == Operand.getValueType() && "Unexpected VT!")((void)0);
5050 break;
5051 case ISD::CTPOP:
5052 if (Operand.getValueType().getScalarType() == MVT::i1)
5053 return Operand;
5054 break;
5055 case ISD::CTLZ:
5056 case ISD::CTTZ:
5057 if (Operand.getValueType().getScalarType() == MVT::i1)
5058 return getNOT(DL, Operand, Operand.getValueType());
5059 break;
5060 case ISD::VECREDUCE_SMIN:
5061 case ISD::VECREDUCE_UMAX:
5062 if (Operand.getValueType().getScalarType() == MVT::i1)
5063 return getNode(ISD::VECREDUCE_OR, DL, VT, Operand);
5064 break;
5065 case ISD::VECREDUCE_SMAX:
5066 case ISD::VECREDUCE_UMIN:
5067 if (Operand.getValueType().getScalarType() == MVT::i1)
5068 return getNode(ISD::VECREDUCE_AND, DL, VT, Operand);
5069 break;
5070 }
5071
5072 SDNode *N;
5073 SDVTList VTs = getVTList(VT);
5074 SDValue Ops[] = {Operand};
5075 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
5076 FoldingSetNodeID ID;
5077 AddNodeIDNode(ID, Opcode, VTs, Ops);
5078 void *IP = nullptr;
5079 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5080 E->intersectFlagsWith(Flags);
5081 return SDValue(E, 0);
5082 }
5083
5084 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5085 N->setFlags(Flags);
5086 createOperands(N, Ops);
5087 CSEMap.InsertNode(N, IP);
5088 } else {
5089 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5090 createOperands(N, Ops);
5091 }
5092
5093 InsertNode(N);
5094 SDValue V = SDValue(N, 0);
5095 NewSDValueDbgMsg(V, "Creating new node: ", this);
5096 return V;
5097}
5098
5099static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
5100 const APInt &C2) {
5101 switch (Opcode) {
5102 case ISD::ADD: return C1 + C2;
5103 case ISD::SUB: return C1 - C2;
5104 case ISD::MUL: return C1 * C2;
5105 case ISD::AND: return C1 & C2;
5106 case ISD::OR: return C1 | C2;
5107 case ISD::XOR: return C1 ^ C2;
5108 case ISD::SHL: return C1 << C2;
5109 case ISD::SRL: return C1.lshr(C2);
5110 case ISD::SRA: return C1.ashr(C2);
5111 case ISD::ROTL: return C1.rotl(C2);
5112 case ISD::ROTR: return C1.rotr(C2);
5113 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
5114 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
5115 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
5116 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
5117 case ISD::SADDSAT: return C1.sadd_sat(C2);
5118 case ISD::UADDSAT: return C1.uadd_sat(C2);
5119 case ISD::SSUBSAT: return C1.ssub_sat(C2);
5120 case ISD::USUBSAT: return C1.usub_sat(C2);
5121 case ISD::UDIV:
5122 if (!C2.getBoolValue())
5123 break;
5124 return C1.udiv(C2);
5125 case ISD::UREM:
5126 if (!C2.getBoolValue())
5127 break;
5128 return C1.urem(C2);
5129 case ISD::SDIV:
5130 if (!C2.getBoolValue())
5131 break;
5132 return C1.sdiv(C2);
5133 case ISD::SREM:
5134 if (!C2.getBoolValue())
5135 break;
5136 return C1.srem(C2);
5137 case ISD::MULHS: {
5138 unsigned FullWidth = C1.getBitWidth() * 2;
5139 APInt C1Ext = C1.sext(FullWidth);
5140 APInt C2Ext = C2.sext(FullWidth);
5141 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5142 }
5143 case ISD::MULHU: {
5144 unsigned FullWidth = C1.getBitWidth() * 2;
5145 APInt C1Ext = C1.zext(FullWidth);
5146 APInt C2Ext = C2.zext(FullWidth);
5147 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5148 }
5149 }
5150 return llvm::None;
5151}
5152
5153SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
5154 const GlobalAddressSDNode *GA,
5155 const SDNode *N2) {
5156 if (GA->getOpcode() != ISD::GlobalAddress)
5157 return SDValue();
5158 if (!TLI->isOffsetFoldingLegal(GA))
5159 return SDValue();
5160 auto *C2 = dyn_cast<ConstantSDNode>(N2);
5161 if (!C2)
5162 return SDValue();
5163 int64_t Offset = C2->getSExtValue();
5164 switch (Opcode) {
5165 case ISD::ADD: break;
5166 case ISD::SUB: Offset = -uint64_t(Offset); break;
5167 default: return SDValue();
5168 }
5169 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
5170 GA->getOffset() + uint64_t(Offset));
5171}
5172
5173bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
5174 switch (Opcode) {
5175 case ISD::SDIV:
5176 case ISD::UDIV:
5177 case ISD::SREM:
5178 case ISD::UREM: {
5179 // If a divisor is zero/undef or any element of a divisor vector is
5180 // zero/undef, the whole op is undef.
5181 assert(Ops.size() == 2 && "Div/rem should have 2 operands")((void)0);
5182 SDValue Divisor = Ops[1];
5183 if (Divisor.isUndef() || isNullConstant(Divisor))
5184 return true;
5185
5186 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
5187 llvm::any_of(Divisor->op_values(),
5188 [](SDValue V) { return V.isUndef() ||
5189 isNullConstant(V); });
5190 // TODO: Handle signed overflow.
5191 }
5192 // TODO: Handle oversized shifts.
5193 default:
5194 return false;
5195 }
5196}
5197
5198SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
5199 EVT VT, ArrayRef<SDValue> Ops) {
5200 // If the opcode is a target-specific ISD node, there's nothing we can
5201 // do here and the operand rules may not line up with the below, so
5202 // bail early.
5203 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
5204 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
5205 // foldCONCAT_VECTORS in getNode before this is called.
5206 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
5207 return SDValue();
5208
5209 // For now, the array Ops should only contain two values.
5210 // This enforcement will be removed once this function is merged with
5211 // FoldConstantVectorArithmetic
5212 if (Ops.size() != 2)
5213 return SDValue();
5214
5215 if (isUndef(Opcode, Ops))
5216 return getUNDEF(VT);
5217
5218 SDNode *N1 = Ops[0].getNode();
5219 SDNode *N2 = Ops[1].getNode();
5220
5221 // Handle the case of two scalars.
5222 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
5223 if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
5224 if (C1->isOpaque() || C2->isOpaque())
5225 return SDValue();
5226
5227 Optional<APInt> FoldAttempt =
5228 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
5229 if (!FoldAttempt)
5230 return SDValue();
5231
5232 SDValue Folded = getConstant(FoldAttempt.getValue(), DL, VT);
5233 assert((!Folded || !VT.isVector()) &&((void)0)
5234 "Can't fold vectors ops with scalar operands")((void)0);
5235 return Folded;
5236 }
5237 }
5238
5239 // fold (add Sym, c) -> Sym+c
5240 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
5241 return FoldSymbolOffset(Opcode, VT, GA, N2);
5242 if (TLI->isCommutativeBinOp(Opcode))
5243 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
5244 return FoldSymbolOffset(Opcode, VT, GA, N1);
5245
5246 // For fixed width vectors, extract each constant element and fold them
5247 // individually. Either input may be an undef value.
5248 bool IsBVOrSV1 = N1->getOpcode() == ISD::BUILD_VECTOR ||
5249 N1->getOpcode() == ISD::SPLAT_VECTOR;
5250 if (!IsBVOrSV1 && !N1->isUndef())
5251 return SDValue();
5252 bool IsBVOrSV2 = N2->getOpcode() == ISD::BUILD_VECTOR ||
5253 N2->getOpcode() == ISD::SPLAT_VECTOR;
5254 if (!IsBVOrSV2 && !N2->isUndef())
5255 return SDValue();
5256 // If both operands are undef, that's handled the same way as scalars.
5257 if (!IsBVOrSV1 && !IsBVOrSV2)
5258 return SDValue();
5259
5260 EVT SVT = VT.getScalarType();
5261 EVT LegalSVT = SVT;
5262 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
5263 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
5264 if (LegalSVT.bitsLT(SVT))
5265 return SDValue();
5266 }
5267
5268 SmallVector<SDValue, 4> Outputs;
5269 unsigned NumOps = 0;
5270 if (IsBVOrSV1)
5271 NumOps = std::max(NumOps, N1->getNumOperands());
5272 if (IsBVOrSV2)
5273 NumOps = std::max(NumOps, N2->getNumOperands());
5274 assert(NumOps != 0 && "Expected non-zero operands")((void)0);
5275 // Scalable vectors should only be SPLAT_VECTOR or UNDEF here. We only need
5276 // one iteration for that.
5277 assert((!VT.isScalableVector() || NumOps == 1) &&((void)0)
5278 "Scalable vector should only have one scalar")((void)0);
5279
5280 for (unsigned I = 0; I != NumOps; ++I) {
5281 // We can have a fixed length SPLAT_VECTOR and a BUILD_VECTOR so we need
5282 // to use operand 0 of the SPLAT_VECTOR for each fixed element.
5283 SDValue V1;
5284 if (N1->getOpcode() == ISD::BUILD_VECTOR)
5285 V1 = N1->getOperand(I);
5286 else if (N1->getOpcode() == ISD::SPLAT_VECTOR)
5287 V1 = N1->getOperand(0);
5288 else
5289 V1 = getUNDEF(SVT);
5290
5291 SDValue V2;
5292 if (N2->getOpcode() == ISD::BUILD_VECTOR)
5293 V2 = N2->getOperand(I);
5294 else if (N2->getOpcode() == ISD::SPLAT_VECTOR)
5295 V2 = N2->getOperand(0);
5296 else
5297 V2 = getUNDEF(SVT);
5298
5299 if (SVT.isInteger()) {
5300 if (V1.getValueType().bitsGT(SVT))
5301 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
5302 if (V2.getValueType().bitsGT(SVT))
5303 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
5304 }
5305
5306 if (V1.getValueType() != SVT || V2.getValueType() != SVT)
5307 return SDValue();
5308
5309 // Fold one vector element.
5310 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
5311 if (LegalSVT != SVT)
5312 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
5313
5314 // Scalar folding only succeeded if the result is a constant or UNDEF.
5315 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
5316 ScalarResult.getOpcode() != ISD::ConstantFP)
5317 return SDValue();
5318 Outputs.push_back(ScalarResult);
5319 }
5320
5321 if (N1->getOpcode() == ISD::BUILD_VECTOR ||
5322 N2->getOpcode() == ISD::BUILD_VECTOR) {
5323 assert(VT.getVectorNumElements() == Outputs.size() &&((void)0)
5324 "Vector size mismatch!")((void)0);
5325
5326 // Build a big vector out of the scalar elements we generated.
5327 return getBuildVector(VT, SDLoc(), Outputs);
5328 }
5329
5330 assert((N1->getOpcode() == ISD::SPLAT_VECTOR ||((void)0)
5331 N2->getOpcode() == ISD::SPLAT_VECTOR) &&((void)0)
5332 "One operand should be a splat vector")((void)0);
5333
5334 assert(Outputs.size() == 1 && "Vector size mismatch!")((void)0);
5335 return getSplatVector(VT, SDLoc(), Outputs[0]);
5336}
5337
5338// TODO: Merge with FoldConstantArithmetic
5339SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
5340 const SDLoc &DL, EVT VT,
5341 ArrayRef<SDValue> Ops,
5342 const SDNodeFlags Flags) {
5343 // If the opcode is a target-specific ISD node, there's nothing we can
5344 // do here and the operand rules may not line up with the below, so
5345 // bail early.
5346 if (Opcode >= ISD::BUILTIN_OP_END)
5347 return SDValue();
5348
5349 if (isUndef(Opcode, Ops))
5350 return getUNDEF(VT);
5351
5352 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
5353 if (!VT.isVector())
5354 return SDValue();
5355
5356 ElementCount NumElts = VT.getVectorElementCount();
5357
5358 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
5359 return !Op.getValueType().isVector() ||
5360 Op.getValueType().getVectorElementCount() == NumElts;
5361 };
5362
5363 auto IsConstantBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
5364 APInt SplatVal;
5365 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
5366 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
5367 (BV && BV->isConstant()) ||
5368 (Op.getOpcode() == ISD::SPLAT_VECTOR &&
5369 ISD::isConstantSplatVector(Op.getNode(), SplatVal));
5370 };
5371
5372 // All operands must be vector types with the same number of elements as
5373 // the result type and must be either UNDEF or a build vector of constant
5374 // or UNDEF scalars.
5375 if (!llvm::all_of(Ops, IsConstantBuildVectorSplatVectorOrUndef) ||
5376 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
5377 return SDValue();
5378
5379 // If we are comparing vectors, then the result needs to be a i1 boolean
5380 // that is then sign-extended back to the legal result type.
5381 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
5382
5383 // Find legal integer scalar type for constant promotion and
5384 // ensure that its scalar size is at least as large as source.
5385 EVT LegalSVT = VT.getScalarType();
5386 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
5387 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
5388 if (LegalSVT.bitsLT(VT.getScalarType()))
5389 return SDValue();
5390 }
5391
5392 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
5393 // only have one operand to check. For fixed-length vector types we may have
5394 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
5395 unsigned NumOperands = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
5396
5397 // Constant fold each scalar lane separately.
5398 SmallVector<SDValue, 4> ScalarResults;
5399 for (unsigned I = 0; I != NumOperands; I++) {
5400 SmallVector<SDValue, 4> ScalarOps;
5401 for (SDValue Op : Ops) {
5402 EVT InSVT = Op.getValueType().getScalarType();
5403 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
5404 Op.getOpcode() != ISD::SPLAT_VECTOR) {
5405 // We've checked that this is UNDEF or a constant of some kind.
5406 if (Op.isUndef())
5407 ScalarOps.push_back(getUNDEF(InSVT));
5408 else
5409 ScalarOps.push_back(Op);
5410 continue;
5411 }
5412
5413 SDValue ScalarOp =
5414 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
5415 EVT ScalarVT = ScalarOp.getValueType();
5416
5417 // Build vector (integer) scalar operands may need implicit
5418 // truncation - do this before constant folding.
5419 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
5420 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
5421
5422 ScalarOps.push_back(ScalarOp);
5423 }
5424
5425 // Constant fold the scalar operands.
5426 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
5427
5428 // Legalize the (integer) scalar constant if necessary.
5429 if (LegalSVT != SVT)
5430 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
5431
5432 // Scalar folding only succeeded if the result is a constant or UNDEF.
5433 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
5434 ScalarResult.getOpcode() != ISD::ConstantFP)
5435 return SDValue();
5436 ScalarResults.push_back(ScalarResult);
5437 }
5438
5439 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
5440 : getBuildVector(VT, DL, ScalarResults);
5441 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
5442 return V;
5443}
5444
5445SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
5446 EVT VT, SDValue N1, SDValue N2) {
5447 // TODO: We don't do any constant folding for strict FP opcodes here, but we
5448 // should. That will require dealing with a potentially non-default
5449 // rounding mode, checking the "opStatus" return value from the APFloat
5450 // math calculations, and possibly other variations.
5451 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
5452 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
5453 if (N1CFP && N2CFP) {
5454 APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
5455 switch (Opcode) {
5456 case ISD::FADD:
5457 C1.add(C2, APFloat::rmNearestTiesToEven);
5458 return getConstantFP(C1, DL, VT);
5459 case ISD::FSUB:
5460 C1.subtract(C2, APFloat::rmNearestTiesToEven);
5461 return getConstantFP(C1, DL, VT);
5462 case ISD::FMUL:
5463 C1.multiply(C2, APFloat::rmNearestTiesToEven);
5464 return getConstantFP(C1, DL, VT);
5465 case ISD::FDIV:
5466 C1.divide(C2, APFloat::rmNearestTiesToEven);
5467 return getConstantFP(C1, DL, VT);
5468 case ISD::FREM:
5469 C1.mod(C2);
5470 return getConstantFP(C1, DL, VT);
5471 case ISD::FCOPYSIGN:
5472 C1.copySign(C2);
5473 return getConstantFP(C1, DL, VT);
5474 default: break;
5475 }
5476 }
5477 if (N1CFP && Opcode == ISD::FP_ROUND) {
5478 APFloat C1 = N1CFP->getValueAPF(); // make copy
5479 bool Unused;
5480 // This can return overflow, underflow, or inexact; we don't care.
5481 // FIXME need to be more flexible about rounding mode.
5482 (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
5483 &Unused);
5484 return getConstantFP(C1, DL, VT);
5485 }
5486
5487 switch (Opcode) {
5488 case ISD::FSUB:
5489 // -0.0 - undef --> undef (consistent with "fneg undef")
5490 if (N1CFP && N1CFP->getValueAPF().isNegZero() && N2.isUndef())
5491 return getUNDEF(VT);
5492 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5493
5494 case ISD::FADD:
5495 case ISD::FMUL:
5496 case ISD::FDIV:
5497 case ISD::FREM:
5498 // If both operands are undef, the result is undef. If 1 operand is undef,
5499 // the result is NaN. This should match the behavior of the IR optimizer.
5500 if (N1.isUndef() && N2.isUndef())
5501 return getUNDEF(VT);
5502 if (N1.isUndef() || N2.isUndef())
5503 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
5504 }
5505 return SDValue();
5506}
5507
5508SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
5509 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!")((void)0);
5510
5511 // There's no need to assert on a byte-aligned pointer. All pointers are at
5512 // least byte aligned.
5513 if (A == Align(1))
5514 return Val;
5515
5516 FoldingSetNodeID ID;
5517 AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val});
5518 ID.AddInteger(A.value());
5519
5520 void *IP = nullptr;
5521 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
5522 return SDValue(E, 0);
5523
5524 auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
5525 Val.getValueType(), A);
5526 createOperands(N, {Val});
5527
5528 CSEMap.InsertNode(N, IP);
5529 InsertNode(N);
5530
5531 SDValue V(N, 0);
5532 NewSDValueDbgMsg(V, "Creating new node: ", this);
5533 return V;
5534}
5535
5536SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5537 SDValue N1, SDValue N2) {
5538 SDNodeFlags Flags;
5539 if (Inserter)
5540 Flags = Inserter->getFlags();
5541 return getNode(Opcode, DL, VT, N1, N2, Flags);
5542}
5543
5544SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5545 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5546 assert(N1.getOpcode() != ISD::DELETED_NODE &&((void)0)
5547 N2.getOpcode() != ISD::DELETED_NODE &&((void)0)
5548 "Operand is DELETED_NODE!")((void)0);
5549 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5550 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
5551 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5552 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5553
5554 // Canonicalize constant to RHS if commutative.
5555 if (TLI->isCommutativeBinOp(Opcode)) {
5556 if (N1C && !N2C) {
5557 std::swap(N1C, N2C);
5558 std::swap(N1, N2);
5559 } else if (N1CFP && !N2CFP) {
5560 std::swap(N1CFP, N2CFP);
5561 std::swap(N1, N2);
5562 }
5563 }
5564
5565 switch (Opcode) {
5566 default: break;
5567 case ISD::TokenFactor:
5568 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&((void)0)
5569 N2.getValueType() == MVT::Other && "Invalid token factor!")((void)0);
5570 // Fold trivial token factors.
5571 if (N1.getOpcode() == ISD::EntryToken) return N2;
5572 if (N2.getOpcode() == ISD::EntryToken) return N1;
5573 if (N1 == N2) return N1;
5574 break;
5575 case ISD::BUILD_VECTOR: {
5576 // Attempt to simplify BUILD_VECTOR.
5577 SDValue Ops[] = {N1, N2};
5578 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5579 return V;
5580 break;
5581 }
5582 case ISD::CONCAT_VECTORS: {
5583 SDValue Ops[] = {N1, N2};
5584 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5585 return V;
5586 break;
5587 }
5588 case ISD::AND:
5589 assert(VT.isInteger() && "This operator does not apply to FP types!")((void)0);
5590 assert(N1.getValueType() == N2.getValueType() &&((void)0)
5591 N1.getValueType() == VT && "Binary operator types must match!")((void)0);
5592 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
5593 // worth handling here.
5594 if (N2C && N2C->isNullValue())
5595 return N2;
5596 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
5597 return N1;
5598 break;
5599 case ISD::OR:
5600 case ISD::XOR:
5601 case ISD::ADD:
5602 case ISD::SUB:
5603 assert(VT.isInteger() && "This operator does not apply to FP types!")((void)0);
5604 assert(N1.getValueType() == N2.getValueType() &&((void)0)
5605 N1.getValueType() == VT && "Binary operator types must match!")((void)0);
5606 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
5607 // it's worth handling here.
5608 if (N2C && N2C->isNullValue())
5609 return N1;
5610 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
5611 VT.getVectorElementType() == MVT::i1)
5612 return getNode(ISD::XOR, DL, VT, N1, N2);
5613 break;
5614 case ISD::MUL:
5615 assert(VT.isInteger() && "This operator does not apply to FP types!")((void)0);
5616 assert(N1.getValueType() == N2.getValueType() &&((void)0)
5617 N1.getValueType() == VT && "Binary operator types must match!")((void)0);
5618 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5619 return getNode(ISD::AND, DL, VT, N1, N2);
5620 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5621 const APInt &MulImm = N1->getConstantOperandAPInt(0);
5622 const APInt &N2CImm = N2C->getAPIntValue();
5623 return getVScale(DL, VT, MulImm * N2CImm);
5624 }
5625 break;
5626 case ISD::UDIV:
5627 case ISD::UREM:
5628 case ISD::MULHU:
5629 case ISD::MULHS:
5630 case ISD::SDIV:
5631 case ISD::SREM:
5632 case ISD::SADDSAT:
5633 case ISD::SSUBSAT:
5634 case ISD::UADDSAT:
5635 case ISD::USUBSAT:
5636 assert(VT.isInteger() && "This operator does not apply to FP types!")((void)0);
5637 assert(N1.getValueType() == N2.getValueType() &&((void)0)
5638 N1.getValueType() == VT && "Binary operator types must match!")((void)0);
5639 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
5640 // fold (add_sat x, y) -> (or x, y) for bool types.
5641 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
5642 return getNode(ISD::OR, DL, VT, N1, N2);
5643 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
5644 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
5645 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
5646 }
5647 break;
5648 case ISD::SMIN:
5649 case ISD::UMAX:
5650 assert(VT.isInteger() && "This operator does not apply to FP types!")((void)0);
5651 assert(N1.getValueType() == N2.getValueType() &&((void)0)
5652 N1.getValueType() == VT && "Binary operator types must match!")((void)0);
5653 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5654 return getNode(ISD::OR, DL, VT, N1, N2);
5655 break;
5656 case ISD::SMAX:
5657 case ISD::UMIN:
5658 assert(VT.isInteger() && "This operator does not apply to FP types!")((void)0);
5659 assert(N1.getValueType() == N2.getValueType() &&((void)0)
5660 N1.getValueType() == VT && "Binary operator types must match!")((void)0);
5661 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
5662 return getNode(ISD::AND, DL, VT, N1, N2);
5663 break;
5664 case ISD::FADD:
5665 case ISD::FSUB:
5666 case ISD::FMUL:
5667 case ISD::FDIV:
5668 case ISD::FREM:
5669 assert(VT.isFloatingPoint() && "This operator only applies to FP types!")((void)0);
5670 assert(N1.getValueType() == N2.getValueType() &&((void)0)
5671 N1.getValueType() == VT && "Binary operator types must match!")((void)0);
5672 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
5673 return V;
5674 break;
5675 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
5676 assert(N1.getValueType() == VT &&((void)0)
5677 N1.getValueType().isFloatingPoint() &&((void)0)
5678 N2.getValueType().isFloatingPoint() &&((void)0)
5679 "Invalid FCOPYSIGN!")((void)0);
5680 break;
5681 case ISD::SHL:
5682 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
5683 const APInt &MulImm = N1->getConstantOperandAPInt(0);
5684 const APInt &ShiftImm = N2C->getAPIntValue();
5685 return getVScale(DL, VT, MulImm << ShiftImm);
5686 }
5687 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5688 case ISD::SRA:
5689 case ISD::SRL:
5690 if (SDValue V = simplifyShift(N1, N2))
5691 return V;
5692 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5693 case ISD::ROTL:
5694 case ISD::ROTR:
5695 assert(VT == N1.getValueType() &&((void)0)
5696 "Shift operators return type must be the same as their first arg")((void)0);
5697 assert(VT.isInteger() && N2.getValueType().isInteger() &&((void)0)
5698 "Shifts only work on integers")((void)0);
5699 assert((!VT.isVector() || VT == N2.getValueType()) &&((void)0)
5700 "Vector shift amounts must be in the same as their first arg")((void)0);
5701 // Verify that the shift amount VT is big enough to hold valid shift
5702 // amounts. This catches things like trying to shift an i1024 value by an
5703 // i8, which is easy to fall into in generic code that uses
5704 // TLI.getShiftAmount().
5705 assert(N2.getValueType().getScalarSizeInBits() >=((void)0)
5706 Log2_32_Ceil(VT.getScalarSizeInBits()) &&((void)0)
5707 "Invalid use of small shift amount with oversized value!")((void)0);
5708
5709 // Always fold shifts of i1 values so the code generator doesn't need to
5710 // handle them. Since we know the size of the shift has to be less than the
5711 // size of the value, the shift/rotate count is guaranteed to be zero.
5712 if (VT == MVT::i1)
5713 return N1;
5714 if (N2C && N2C->isNullValue())
5715 return N1;
5716 break;
5717 case ISD::FP_ROUND:
5718 assert(VT.isFloatingPoint() &&((void)0)
5719 N1.getValueType().isFloatingPoint() &&((void)0)
5720 VT.bitsLE(N1.getValueType()) &&((void)0)
5721 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&((void)0)
5722 "Invalid FP_ROUND!")((void)0);
5723 if (N1.getValueType() == VT) return N1; // noop conversion.
5724 break;
5725 case ISD::AssertSext:
5726 case ISD::AssertZext: {
5727 EVT EVT = cast<VTSDNode>(N2)->getVT();
5728 assert(VT == N1.getValueType() && "Not an inreg extend!")((void)0);
5729 assert(VT.isInteger() && EVT.isInteger() &&((void)0)
5730 "Cannot *_EXTEND_INREG FP types")((void)0);
5731 assert(!EVT.isVector() &&((void)0)
5732 "AssertSExt/AssertZExt type should be the vector element type "((void)0)
5733 "rather than the vector type!")((void)0);
5734 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!")((void)0);
5735 if (VT.getScalarType() == EVT) return N1; // noop assertion.
5736 break;
5737 }
5738 case ISD::SIGN_EXTEND_INREG: {
5739 EVT EVT = cast<VTSDNode>(N2)->getVT();
5740 assert(VT == N1.getValueType() && "Not an inreg extend!")((void)0);
5741 assert(VT.isInteger() && EVT.isInteger() &&((void)0)
5742 "Cannot *_EXTEND_INREG FP types")((void)0);
5743 assert(EVT.isVector() == VT.isVector() &&((void)0)
5744 "SIGN_EXTEND_INREG type should be vector iff the operand "((void)0)
5745 "type is vector!")((void)0);
5746 assert((!EVT.isVector() ||((void)0)
5747 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&((void)0)
5748 "Vector element counts must match in SIGN_EXTEND_INREG")((void)0);
5749 assert(EVT.bitsLE(VT) && "Not extending!")((void)0);
5750 if (EVT == VT) return N1; // Not actually extending
5751
5752 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
5753 unsigned FromBits = EVT.getScalarSizeInBits();
5754 Val <<= Val.getBitWidth() - FromBits;
5755 Val.ashrInPlace(Val.getBitWidth() - FromBits);
5756 return getConstant(Val, DL, ConstantVT);
5757 };
5758
5759 if (N1C) {
5760 const APInt &Val = N1C->getAPIntValue();
5761 return SignExtendInReg(Val, VT);
5762 }
5763
5764 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
5765 SmallVector<SDValue, 8> Ops;
5766 llvm::EVT OpVT = N1.getOperand(0).getValueType();
5767 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
5768 SDValue Op = N1.getOperand(i);
5769 if (Op.isUndef()) {
5770 Ops.push_back(getUNDEF(OpVT));
5771 continue;
5772 }
5773 ConstantSDNode *C = cast<ConstantSDNode>(Op);
5774 APInt Val = C->getAPIntValue();
5775 Ops.push_back(SignExtendInReg(Val, OpVT));
5776 }
5777 return getBuildVector(VT, DL, Ops);
5778 }
5779 break;
5780 }
5781 case ISD::FP_TO_SINT_SAT:
5782 case ISD::FP_TO_UINT_SAT: {
5783 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&((void)0)
5784 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT")((void)0);
5785 assert(N1.getValueType().isVector() == VT.isVector() &&((void)0)
5786 "FP_TO_*INT_SAT type should be vector iff the operand type is "((void)0)
5787 "vector!")((void)0);
5788 assert((!VT.isVector() || VT.getVectorNumElements() ==((void)0)
5789 N1.getValueType().getVectorNumElements()) &&((void)0)
5790 "Vector element counts must match in FP_TO_*INT_SAT")((void)0);
5791 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&((void)0)
5792 "Type to saturate to must be a scalar.")((void)0);
5793 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&((void)0)
5794 "Not extending!")((void)0);
5795 break;
5796 }
5797 case ISD::EXTRACT_VECTOR_ELT:
5798 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&((void)0)
5799 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \((void)0)
5800 element type of the vector.")((void)0);
5801
5802 // Extract from an undefined value or using an undefined index is undefined.
5803 if (N1.isUndef() || N2.isUndef())
5804 return getUNDEF(VT);
5805
5806 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
5807 // vectors. For scalable vectors we will provide appropriate support for
5808 // dealing with arbitrary indices.
5809 if (N2C && N1.getValueType().isFixedLengthVector() &&
5810 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
5811 return getUNDEF(VT);
5812
5813 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
5814 // expanding copies of large vectors from registers. This only works for
5815 // fixed length vectors, since we need to know the exact number of
5816 // elements.
5817 if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() &&
5818 N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) {
5819 unsigned Factor =
5820 N1.getOperand(0).getValueType().getVectorNumElements();
5821 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
5822 N1.getOperand(N2C->getZExtValue() / Factor),
5823 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
5824 }
5825
5826 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
5827 // lowering is expanding large vector constants.
5828 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
5829 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
5830 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||((void)0)
5831 N1.getValueType().isFixedLengthVector()) &&((void)0)
5832 "BUILD_VECTOR used for scalable vectors")((void)0);
5833 unsigned Index =
5834 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
5835 SDValue Elt = N1.getOperand(Index);
5836
5837 if (VT != Elt.getValueType())
5838 // If the vector element type is not legal, the BUILD_VECTOR operands
5839 // are promoted and implicitly truncated, and the result implicitly
5840 // extended. Make that explicit here.
5841 Elt = getAnyExtOrTrunc(Elt, DL, VT);
5842
5843 return Elt;
5844 }
5845
5846 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
5847 // operations are lowered to scalars.
5848 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
5849 // If the indices are the same, return the inserted element else
5850 // if the indices are known different, extract the element from
5851 // the original vector.
5852 SDValue N1Op2 = N1.getOperand(2);
5853 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
5854
5855 if (N1Op2C && N2C) {
5856 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
5857 if (VT == N1.getOperand(1).getValueType())
5858 return N1.getOperand(1);
5859 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
5860 }
5861 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
5862 }
5863 }
5864
5865 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
5866 // when vector types are scalarized and v1iX is legal.
5867 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
5868 // Here we are completely ignoring the extract element index (N2),
5869 // which is fine for fixed width vectors, since any index other than 0
5870 // is undefined anyway. However, this cannot be ignored for scalable
5871 // vectors - in theory we could support this, but we don't want to do this
5872 // without a profitability check.
5873 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5874 N1.getValueType().isFixedLengthVector() &&
5875 N1.getValueType().getVectorNumElements() == 1) {
5876 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
5877 N1.getOperand(1));
5878 }
5879 break;
5880 case ISD::EXTRACT_ELEMENT:
5881 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!")((void)0);
5882 assert(!N1.getValueType().isVector() && !VT.isVector() &&((void)0)
5883 (N1.getValueType().isInteger() == VT.isInteger()) &&((void)0)
5884 N1.getValueType() != VT &&((void)0)
5885 "Wrong types for EXTRACT_ELEMENT!")((void)0);
5886
5887 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
5888 // 64-bit integers into 32-bit parts. Instead of building the extract of
5889 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
5890 if (N1.getOpcode() == ISD::BUILD_PAIR)
5891 return N1.getOperand(N2C->getZExtValue());
5892
5893 // EXTRACT_ELEMENT of a constant int is also very common.
5894 if (N1C) {
5895 unsigned ElementSize = VT.getSizeInBits();
5896 unsigned Shift = ElementSize * N2C->getZExtValue();
5897 const APInt &Val = N1C->getAPIntValue();
5898 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
5899 }
5900 break;
5901 case ISD::EXTRACT_SUBVECTOR: {
5902 EVT N1VT = N1.getValueType();
5903 assert(VT.isVector() && N1VT.isVector() &&((void)0)
5904 "Extract subvector VTs must be vectors!")((void)0);
5905 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&((void)0)
5906 "Extract subvector VTs must have the same element type!")((void)0);
5907 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&((void)0)
5908 "Cannot extract a scalable vector from a fixed length vector!")((void)0);
5909 assert((VT.isScalableVector() != N1VT.isScalableVector() ||((void)0)
5910 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&((void)0)
5911 "Extract subvector must be from larger vector to smaller vector!")((void)0);
5912 assert(N2C && "Extract subvector index must be a constant")((void)0);
5913 assert((VT.isScalableVector() != N1VT.isScalableVector() ||((void)0)
5914 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=((void)0)
5915 N1VT.getVectorMinNumElements()) &&((void)0)
5916 "Extract subvector overflow!")((void)0);
5917 assert(N2C->getAPIntValue().getBitWidth() ==((void)0)
5918 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&((void)0)
5919 "Constant index for EXTRACT_SUBVECTOR has an invalid size")((void)0);
5920
5921 // Trivial extraction.
5922 if (VT == N1VT)
5923 return N1;
5924
5925 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
5926 if (N1.isUndef())
5927 return getUNDEF(VT);
5928
5929 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
5930 // the concat have the same type as the extract.
5931 if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 &&
5932 VT == N1.getOperand(0).getValueType()) {
5933 unsigned Factor = VT.getVectorMinNumElements();
5934 return N1.getOperand(N2C->getZExtValue() / Factor);
5935 }
5936
5937 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
5938 // during shuffle legalization.
5939 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
5940 VT == N1.getOperand(1).getValueType())
5941 return N1.getOperand(1);
5942 break;
5943 }
5944 }
5945
5946 // Perform trivial constant folding.
5947 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
5948 return SV;
5949
5950 if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
5951 return V;
5952
5953 // Canonicalize an UNDEF to the RHS, even over a constant.
5954 if (N1.isUndef()) {
5955 if (TLI->isCommutativeBinOp(Opcode)) {
5956 std::swap(N1, N2);
5957 } else {
5958 switch (Opcode) {
5959 case ISD::SIGN_EXTEND_INREG:
5960 case ISD::SUB:
5961 return getUNDEF(VT); // fold op(undef, arg2) -> undef
5962 case ISD::UDIV:
5963 case ISD::SDIV:
5964 case ISD::UREM:
5965 case ISD::SREM:
5966 case ISD::SSUBSAT:
5967 case ISD::USUBSAT:
5968 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
5969 }
5970 }
5971 }
5972
5973 // Fold a bunch of operators when the RHS is undef.
5974 if (N2.isUndef()) {
5975 switch (Opcode) {
5976 case ISD::XOR:
5977 if (N1.isUndef())
5978 // Handle undef ^ undef -> 0 special case. This is a common
5979 // idiom (misuse).
5980 return getConstant(0, DL, VT);
5981 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5982 case ISD::ADD:
5983 case ISD::SUB:
5984 case ISD::UDIV:
5985 case ISD::SDIV:
5986 case ISD::UREM:
5987 case ISD::SREM:
5988 return getUNDEF(VT); // fold op(arg1, undef) -> undef
5989 case ISD::MUL:
5990 case ISD::AND:
5991 case ISD::SSUBSAT:
5992 case ISD::USUBSAT:
5993 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
5994 case ISD::OR:
5995 case ISD::SADDSAT:
5996 case ISD::UADDSAT:
5997 return getAllOnesConstant(DL, VT);
5998 }
5999 }
6000
6001 // Memoize this node if possible.
6002 SDNode *N;
6003 SDVTList VTs = getVTList(VT);
6004 SDValue Ops[] = {N1, N2};
6005 if (VT != MVT::Glue) {
6006 FoldingSetNodeID ID;
6007 AddNodeIDNode(ID, Opcode, VTs, Ops);
6008 void *IP = nullptr;
6009 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
6010 E->intersectFlagsWith(Flags);
6011 return SDValue(E, 0);
6012 }
6013
6014 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6015 N->setFlags(Flags);
6016 createOperands(N, Ops);
6017 CSEMap.InsertNode(N, IP);
6018 } else {
6019 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6020 createOperands(N, Ops);
6021 }
6022
6023 InsertNode(N);
6024 SDValue V = SDValue(N, 0);
6025 NewSDValueDbgMsg(V, "Creating new node: ", this);
6026 return V;
6027}
6028
6029SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6030 SDValue N1, SDValue N2, SDValue N3) {
6031 SDNodeFlags Flags;
6032 if (Inserter)
6033 Flags = Inserter->getFlags();
6034 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
6035}
6036
6037SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6038 SDValue N1, SDValue N2, SDValue N3,
6039 const SDNodeFlags Flags) {
6040 assert(N1.getOpcode() != ISD::DELETED_NODE &&((void)0)
6041 N2.getOpcode() != ISD::DELETED_NODE &&((void)0)
6042 N3.getOpcode() != ISD::DELETED_NODE &&((void)0)
6043 "Operand is DELETED_NODE!")((void)0);
6044 // Perform various simplifications.
6045 switch (Opcode) {
6046 case ISD::FMA: {
6047 assert(VT.isFloatingPoint() && "This operator only applies to FP types!")((void)0);
6048 assert(N1.getValueType() == VT && N2.getValueType() == VT &&((void)0)
6049 N3.getValueType() == VT && "FMA types must match!")((void)0);
6050 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
6051 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
6052 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
6053 if (N1CFP && N2CFP && N3CFP) {
6054 APFloat V1 = N1CFP->getValueAPF();
6055 const APFloat &V2 = N2CFP->getValueAPF();
6056 const APFloat &V3 = N3CFP->getValueAPF();
6057 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
6058 return getConstantFP(V1, DL, VT);
6059 }
6060 break;
6061 }
6062 case ISD::BUILD_VECTOR: {
6063 // Attempt to simplify BUILD_VECTOR.
6064 SDValue Ops[] = {N1, N2, N3};
6065 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
6066 return V;
6067 break;
6068 }
6069 case ISD::CONCAT_VECTORS: {
6070 SDValue Ops[] = {N1, N2, N3};
6071 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
6072 return V;
6073 break;
6074 }
6075 case ISD::SETCC: {
6076 assert(VT.isInteger() && "SETCC result type must be an integer!")((void)0);
6077 assert(N1.getValueType() == N2.getValueType() &&((void)0)
6078 "SETCC operands must have the same type!")((void)0);
6079 assert(VT.isVector() == N1.getValueType().isVector() &&((void)0)
6080 "SETCC type should be vector iff the operand type is vector!")((void)0);
6081 assert((!VT.isVector() || VT.getVectorElementCount() ==((void)0)
6082 N1.getValueType().getVectorElementCount()) &&((void)0)
6083 "SETCC vector element counts must match!")((void)0);
6084 // Use FoldSetCC to simplify SETCC's.
6085 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
6086 return V;
6087 // Vector constant folding.
6088 SDValue Ops[] = {N1, N2, N3};
6089 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
6090 NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
6091 return V;
6092 }
6093 break;
6094 }
6095 case ISD::SELECT:
6096 case ISD::VSELECT:
6097 if (SDValue V = simplifySelect(N1, N2, N3))
6098 return V;
6099 break;
6100 case ISD::VECTOR_SHUFFLE:
6101 llvm_unreachable("should use getVectorShuffle constructor!")__builtin_unreachable();
6102 case ISD::INSERT_VECTOR_ELT: {
6103 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
6104 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
6105 // for scalable vectors where we will generate appropriate code to
6106 // deal with out-of-bounds cases correctly.
6107 if (N3C && N1.getValueType().isFixedLengthVector() &&
6108 N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
6109 return getUNDEF(VT);
6110
6111 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
6112 if (N3.isUndef())
6113 return getUNDEF(VT);
6114
6115 // If the inserted element is an UNDEF, just use the input vector.
6116 if (N2.isUndef())
6117 return N1;
6118
6119 break;
6120 }
6121 case ISD::INSERT_SUBVECTOR: {
6122 // Inserting undef into undef is still undef.
6123 if (N1.isUndef() && N2.isUndef())
6124 return getUNDEF(VT);
6125
6126 EVT N2VT = N2.getValueType();
6127 assert(VT == N1.getValueType() &&((void)0)
6128 "Dest and insert subvector source types must match!")((void)0);
6129 assert(VT.isVector() && N2VT.isVector() &&((void)0)
6130 "Insert subvector VTs must be vectors!")((void)0);
6131 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&((void)0)
6132 "Cannot insert a scalable vector into a fixed length vector!")((void)0);
6133 assert((VT.isScalableVector() != N2VT.isScalableVector() ||((void)0)
6134 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&((void)0)
6135 "Insert subvector must be from smaller vector to larger vector!")((void)0);
6136 assert(isa<ConstantSDNode>(N3) &&((void)0)
6137 "Insert subvector index must be constant")((void)0);
6138 assert((VT.isScalableVector() != N2VT.isScalableVector() ||((void)0)
6139 (N2VT.getVectorMinNumElements() +((void)0)
6140 cast<ConstantSDNode>(N3)->getZExtValue()) <=((void)0)
6141 VT.getVectorMinNumElements()) &&((void)0)
6142 "Insert subvector overflow!")((void)0);
6143 assert(cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() ==((void)0)
6144 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&((void)0)
6145 "Constant index for INSERT_SUBVECTOR has an invalid size")((void)0);
6146
6147 // Trivial insertion.
6148 if (VT == N2VT)
6149 return N2;
6150
6151 // If this is an insert of an extracted vector into an undef vector, we
6152 // can just use the input to the extract.
6153 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6154 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
6155 return N2.getOperand(0);
6156 break;
6157 }
6158 case ISD::BITCAST:
6159 // Fold bit_convert nodes from a type to themselves.
6160 if (N1.getValueType() == VT)
6161 return N1;
6162 break;
6163 }
6164
6165 // Memoize node if it doesn't produce a flag.
6166 SDNode *N;
6167 SDVTList VTs = getVTList(VT);
6168 SDValue Ops[] = {N1, N2, N3};
6169 if (VT != MVT::Glue) {
6170 FoldingSetNodeID ID;
6171 AddNodeIDNode(ID, Opcode, VTs, Ops);
6172 void *IP = nullptr;
6173 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
6174 E->intersectFlagsWith(Flags);
6175 return SDValue(E, 0);
6176 }
6177
6178 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6179 N->setFlags(Flags);
6180 createOperands(N, Ops);
6181 CSEMap.InsertNode(N, IP);
6182 } else {
6183 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6184 createOperands(N, Ops);
6185 }
6186
6187 InsertNode(N);
6188 SDValue V = SDValue(N, 0);
6189 NewSDValueDbgMsg(V, "Creating new node: ", this);
6190 return V;
6191}
6192
6193SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6194 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
6195 SDValue Ops[] = { N1, N2, N3, N4 };
6196 return getNode(Opcode, DL, VT, Ops);
6197}
6198
6199SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6200 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
6201 SDValue N5) {
6202 SDValue Ops[] = { N1, N2, N3, N4, N5 };
6203 return getNode(Opcode, DL, VT, Ops);
6204}
6205
6206/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
6207/// the incoming stack arguments to be loaded from the stack.
6208SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
6209 SmallVector<SDValue, 8> ArgChains;
6210
6211 // Include the original chain at the beginning of the list. When this is
6212 // used by target LowerCall hooks, this helps legalize find the
6213 // CALLSEQ_BEGIN node.
6214 ArgChains.push_back(Chain);
6215
6216 // Add a chain value for each stack argument.
6217 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
6218 UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
6219 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
6220 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
6221 if (FI->getIndex() < 0)
6222 ArgChains.push_back(SDValue(L, 1));
6223
6224 // Build a tokenfactor for all the chains.
6225 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
6226}
6227
6228/// getMemsetValue - Vectorized representation of the memset value
6229/// operand.
6230static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
6231 const SDLoc &dl) {
6232 assert(!Value.isUndef())((void)0);
6233
6234 unsigned NumBits = VT.getScalarSizeInBits();
6235 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
6236 assert(C->getAPIntValue().getBitWidth() == 8)((void)0);
6237 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
6238 if (VT.isInteger()) {
6239 bool IsOpaque = VT.getSizeInBits() > 64 ||
6240 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
6241 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
6242 }
6243 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
6244 VT);
6245 }
6246
6247 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?")((void)0);
6248 EVT IntVT = VT.getScalarType();
6249 if (!IntVT.isInteger())
6250 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
6251
6252 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
6253 if (NumBits > 8) {
6254 // Use a multiplication with 0x010101... to extend the input to the
6255 // required length.
6256 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
6257 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
6258 DAG.getConstant(Magic, dl, IntVT));
6259 }
6260
6261 if (VT != Value.getValueType() && !VT.isInteger())
6262 Value = DAG.getBitcast(VT.getScalarType(), Value);
6263 if (VT != Value.getValueType())
6264 Value = DAG.getSplatBuildVector(VT, dl, Value);
6265
6266 return Value;
6267}
6268
6269/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
6270/// used when a memcpy is turned into a memset when the source is a constant
6271/// string ptr.
6272static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
6273 const TargetLowering &TLI,
6274 const ConstantDataArraySlice &Slice) {
6275 // Handle vector with all elements zero.
6276 if (Slice.Array == nullptr) {
6277 if (VT.isInteger())
6278 return DAG.getConstant(0, dl, VT);
6279 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
6280 return DAG.getConstantFP(0.0, dl, VT);
6281 if (VT.isVector()) {
6282 unsigned NumElts = VT.getVectorNumElements();
6283 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
6284 return DAG.getNode(ISD::BITCAST, dl, VT,
6285 DAG.getConstant(0, dl,
6286 EVT::getVectorVT(*DAG.getContext(),
6287 EltVT, NumElts)));
6288 }
6289 llvm_unreachable("Expected type!")__builtin_unreachable();
6290 }
6291
6292 assert(!VT.isVector() && "Can't handle vector type here!")((void)0);
6293 unsigned NumVTBits = VT.getSizeInBits();
6294 unsigned NumVTBytes = NumVTBits / 8;
6295 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
6296
6297 APInt Val(NumVTBits, 0);
6298 if (DAG.getDataLayout().isLittleEndian()) {
6299 for (unsigned i = 0; i != NumBytes; ++i)
6300 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
6301 } else {
6302 for (unsigned i = 0; i != NumBytes; ++i)
6303 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
6304 }
6305
6306 // If the "cost" of materializing the integer immediate is less than the cost
6307 // of a load, then it is cost effective to turn the load into the immediate.
6308 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
6309 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
6310 return DAG.getConstant(Val, dl, VT);
6311 return SDValue(nullptr, 0);
6312}
6313
6314SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
6315 const SDLoc &DL,
6316 const SDNodeFlags Flags) {
6317 EVT VT = Base.getValueType();
6318 SDValue Index;
6319
6320 if (Offset.isScalable())
6321 Index = getVScale(DL, Base.getValueType(),
6322 APInt(Base.getValueSizeInBits().getFixedSize(),
6323 Offset.getKnownMinSize()));
6324 else
6325 Index = getConstant(Offset.getFixedSize(), DL, VT);
6326
6327 return getMemBasePlusOffset(Base, Index, DL, Flags);
6328}
6329
6330SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
6331 const SDLoc &DL,
6332 const SDNodeFlags Flags) {
6333 assert(Offset.getValueType().isInteger())((void)0);
6334 EVT BasePtrVT = Ptr.getValueType();
6335 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
6336}
6337
6338/// Returns true if memcpy source is constant data.
6339static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
6340 uint64_t SrcDelta = 0;
6341 GlobalAddressSDNode *G = nullptr;
6342 if (Src.getOpcode() == ISD::GlobalAddress)
6343 G = cast<GlobalAddressSDNode>(Src);
6344 else if (Src.getOpcode() == ISD::ADD &&
6345 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
6346 Src.getOperand(1).getOpcode() == ISD::Constant) {
6347 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
6348 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
6349 }
6350 if (!G)
6351 return false;
6352
6353 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
6354 SrcDelta + G->getOffset());
6355}
6356
6357static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
6358 SelectionDAG &DAG) {
6359 // On Darwin, -Os means optimize for size without hurting performance, so
6360 // only really optimize for size when -Oz (MinSize) is used.
6361 if (MF.getTarget().getTargetTriple().isOSDarwin())
6362 return MF.getFunction().hasMinSize();
6363 return DAG.shouldOptForSize();
6364}
6365
6366static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
6367 SmallVector<SDValue, 32> &OutChains, unsigned From,
6368 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
6369 SmallVector<SDValue, 16> &OutStoreChains) {
6370 assert(OutLoadChains.size() && "Missing loads in memcpy inlining")((void)0);
6371 assert(OutStoreChains.size() && "Missing stores in memcpy inlining")((void)0);
6372 SmallVector<SDValue, 16> GluedLoadChains;
6373 for (unsigned i = From; i < To; ++i) {
6374 OutChains.push_back(OutLoadChains[i]);
6375 GluedLoadChains.push_back(OutLoadChains[i]);
6376 }
6377
6378 // Chain for all loads.
6379 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
6380 GluedLoadChains);
6381
6382 for (unsigned i = From; i < To; ++i) {
6383 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
6384 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
6385 ST->getBasePtr(), ST->getMemoryVT(),
6386 ST->getMemOperand());
6387 OutChains.push_back(NewStore);
6388 }
6389}
6390
6391static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
6392 SDValue Chain, SDValue Dst, SDValue Src,
6393 uint64_t Size, Align Alignment,
6394 bool isVol, bool AlwaysInline,
6395 MachinePointerInfo DstPtrInfo,
6396 MachinePointerInfo SrcPtrInfo,
6397 const AAMDNodes &AAInfo) {
6398 // Turn a memcpy of undef to nop.
6399 // FIXME: We need to honor volatile even is Src is undef.
6400 if (Src.isUndef())
6401 return Chain;
6402
6403 // Expand memcpy to a series of load and store ops if the size operand falls
6404 // below a certain threshold.
6405 // TODO: In the AlwaysInline case, if the size is big then generate a loop
6406 // rather than maybe a humongous number of loads and stores.
6407 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6408 const DataLayout &DL = DAG.getDataLayout();
6409 LLVMContext &C = *DAG.getContext();
6410 std::vector<EVT> MemOps;
6411 bool DstAlignCanChange = false;
6412 MachineFunction &MF = DAG.getMachineFunction();
6413 MachineFrameInfo &MFI = MF.getFrameInfo();
6414 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6415 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6416 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6417 DstAlignCanChange = true;
6418 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
6419 if (!SrcAlign || Alignment > *SrcAlign)
6420 SrcAlign = Alignment;
6421 assert(SrcAlign && "SrcAlign must be set")((void)0);
6422 ConstantDataArraySlice Slice;
6423 // If marked as volatile, perform a copy even when marked as constant.
6424 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
6425 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
6426 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
6427 const MemOp Op = isZeroConstant
6428 ? MemOp::Set(Size, DstAlignCanChange, Alignment,
6429 /*IsZeroMemset*/ true, isVol)
6430 : MemOp::Copy(Size, DstAlignCanChange, Alignment,
6431 *SrcAlign, isVol, CopyFromConstant);
6432 if (!TLI.findOptimalMemOpLowering(
6433 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
6434 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
6435 return SDValue();
6436
6437 if (DstAlignCanChange) {
6438 Type *Ty = MemOps[0].getTypeForEVT(C);
6439 Align NewAlign = DL.getABITypeAlign(Ty);
6440
6441 // Don't promote to an alignment that would require dynamic stack
6442 // realignment.
6443 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6444 if (!TRI->hasStackRealignment(MF))
6445 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
6446 NewAlign = NewAlign / 2;
6447
6448 if (NewAlign > Alignment) {
6449 // Give the stack frame object a larger alignment if needed.
6450 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6451 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6452 Alignment = NewAlign;
6453 }
6454 }
6455
6456 // Prepare AAInfo for loads/stores after lowering this memcpy.
6457 AAMDNodes NewAAInfo = AAInfo;
6458 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
6459
6460 MachineMemOperand::Flags MMOFlags =
6461 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
6462 SmallVector<SDValue, 16> OutLoadChains;
6463 SmallVector<SDValue, 16> OutStoreChains;
6464 SmallVector<SDValue, 32> OutChains;
6465 unsigned NumMemOps = MemOps.size();
6466 uint64_t SrcOff = 0, DstOff = 0;
6467 for (unsigned i = 0; i != NumMemOps; ++i) {
6468 EVT VT = MemOps[i];
6469 unsigned VTSize = VT.getSizeInBits() / 8;
6470 SDValue Value, Store;
6471
6472 if (VTSize > Size) {
6473 // Issuing an unaligned load / store pair that overlaps with the previous
6474 // pair. Adjust the offset accordingly.
6475 assert(i == NumMemOps-1 && i != 0)((void)0);
6476 SrcOff -= VTSize - Size;
6477 DstOff -= VTSize - Size;
6478 }
6479
6480 if (CopyFromConstant &&
6481 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
6482 // It's unlikely a store of a vector immediate can be done in a single
6483 // instruction. It would require a load from a constantpool first.
6484 // We only handle zero vectors here.
6485 // FIXME: Handle other cases where store of vector immediate is done in
6486 // a single instruction.
6487 ConstantDataArraySlice SubSlice;
6488 if (SrcOff < Slice.Length) {
6489 SubSlice = Slice;
6490 SubSlice.move(SrcOff);
6491 } else {
6492 // This is an out-of-bounds access and hence UB. Pretend we read zero.
6493 SubSlice.Array = nullptr;
6494 SubSlice.Offset = 0;
6495 SubSlice.Length = VTSize;
6496 }
6497 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
6498 if (Value.getNode()) {
6499 Store = DAG.getStore(
6500 Chain, dl, Value,
6501 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6502 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
6503 OutChains.push_back(Store);
6504 }
6505 }
6506
6507 if (!Store.getNode()) {
6508 // The type might not be legal for the target. This should only happen
6509 // if the type is smaller than a legal type, as on PPC, so the right
6510 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
6511 // to Load/Store if NVT==VT.
6512 // FIXME does the case above also need this?
6513 EVT NVT = TLI.getTypeToTransformTo(C, VT);
6514 assert(NVT.bitsGE(VT))((void)0);
6515
6516 bool isDereferenceable =
6517 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
6518 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
6519 if (isDereferenceable)
6520 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
6521
6522 Value = DAG.getExtLoad(
6523 ISD::EXTLOAD, dl, NVT, Chain,
6524 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
6525 SrcPtrInfo.getWithOffset(SrcOff), VT,
6526 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
6527 OutLoadChains.push_back(Value.getValue(1));
6528
6529 Store = DAG.getTruncStore(
6530 Chain, dl, Value,
6531 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6532 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
6533 OutStoreChains.push_back(Store);
6534 }
6535 SrcOff += VTSize;
6536 DstOff += VTSize;
6537 Size -= VTSize;
6538 }
6539
6540 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
6541 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
6542 unsigned NumLdStInMemcpy = OutStoreChains.size();
6543
6544 if (NumLdStInMemcpy) {
6545 // It may be that memcpy might be converted to memset if it's memcpy
6546 // of constants. In such a case, we won't have loads and stores, but
6547 // just stores. In the absence of loads, there is nothing to gang up.
6548 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
6549 // If target does not care, just leave as it.
6550 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
6551 OutChains.push_back(OutLoadChains[i]);
6552 OutChains.push_back(OutStoreChains[i]);
6553 }
6554 } else {
6555 // Ld/St less than/equal limit set by target.
6556 if (NumLdStInMemcpy <= GluedLdStLimit) {
6557 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6558 NumLdStInMemcpy, OutLoadChains,
6559 OutStoreChains);
6560 } else {
6561 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
6562 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
6563 unsigned GlueIter = 0;
6564
6565 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
6566 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
6567 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
6568
6569 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
6570 OutLoadChains, OutStoreChains);
6571 GlueIter += GluedLdStLimit;
6572 }
6573
6574 // Residual ld/st.
6575 if (RemainingLdStInMemcpy) {
6576 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6577 RemainingLdStInMemcpy, OutLoadChains,
6578 OutStoreChains);
6579 }
6580 }
6581 }
6582 }
6583 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6584}
6585
6586static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
6587 SDValue Chain, SDValue Dst, SDValue Src,
6588 uint64_t Size, Align Alignment,
6589 bool isVol, bool AlwaysInline,
6590 MachinePointerInfo DstPtrInfo,
6591 MachinePointerInfo SrcPtrInfo,
6592 const AAMDNodes &AAInfo) {
6593 // Turn a memmove of undef to nop.
6594 // FIXME: We need to honor volatile even is Src is undef.
6595 if (Src.isUndef())
6596 return Chain;
6597
6598 // Expand memmove to a series of load and store ops if the size operand falls
6599 // below a certain threshold.
6600 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6601 const DataLayout &DL = DAG.getDataLayout();
6602 LLVMContext &C = *DAG.getContext();
6603 std::vector<EVT> MemOps;
6604 bool DstAlignCanChange = false;
6605 MachineFunction &MF = DAG.getMachineFunction();
6606 MachineFrameInfo &MFI = MF.getFrameInfo();
6607 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6608 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6609 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6610 DstAlignCanChange = true;
6611 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
6612 if (!SrcAlign || Alignment > *SrcAlign)
6613 SrcAlign = Alignment;
6614 assert(SrcAlign && "SrcAlign must be set")((void)0);
6615 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
6616 if (!TLI.findOptimalMemOpLowering(
6617 MemOps, Limit,
6618 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
6619 /*IsVolatile*/ true),
6620 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
6621 MF.getFunction().getAttributes()))
6622 return SDValue();
6623
6624 if (DstAlignCanChange) {
6625 Type *Ty = MemOps[0].getTypeForEVT(C);
6626 Align NewAlign = DL.getABITypeAlign(Ty);
6627 if (NewAlign > Alignment) {
6628 // Give the stack frame object a larger alignment if needed.
6629 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6630 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6631 Alignment = NewAlign;
6632 }
6633 }
6634
6635 // Prepare AAInfo for loads/stores after lowering this memmove.
6636 AAMDNodes NewAAInfo = AAInfo;
6637 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
6638
6639 MachineMemOperand::Flags MMOFlags =
6640 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
6641 uint64_t SrcOff = 0, DstOff = 0;
6642 SmallVector<SDValue, 8> LoadValues;
6643 SmallVector<SDValue, 8> LoadChains;
6644 SmallVector<SDValue, 8> OutChains;
6645 unsigned NumMemOps = MemOps.size();
6646 for (unsigned i = 0; i < NumMemOps; i++) {
6647 EVT VT = MemOps[i];
6648 unsigned VTSize = VT.getSizeInBits() / 8;
6649 SDValue Value;
6650
6651 bool isDereferenceable =
6652 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
6653 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
6654 if (isDereferenceable)
6655 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
6656
6657 Value = DAG.getLoad(
6658 VT, dl, Chain,
6659 DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl),
6660 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
6661 LoadValues.push_back(Value);
6662 LoadChains.push_back(Value.getValue(1));
6663 SrcOff += VTSize;
6664 }
6665 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
6666 OutChains.clear();
6667 for (unsigned i = 0; i < NumMemOps; i++) {
6668 EVT VT = MemOps[i];
6669 unsigned VTSize = VT.getSizeInBits() / 8;
6670 SDValue Store;
6671
6672 Store = DAG.getStore(
6673 Chain, dl, LoadValues[i],
6674 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6675 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
6676 OutChains.push_back(Store);
6677 DstOff += VTSize;
6678 }
6679
6680 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6681}
6682
6683/// Lower the call to 'memset' intrinsic function into a series of store
6684/// operations.
6685///
6686/// \param DAG Selection DAG where lowered code is placed.
6687/// \param dl Link to corresponding IR location.
6688/// \param Chain Control flow dependency.
6689/// \param Dst Pointer to destination memory location.
6690/// \param Src Value of byte to write into the memory.
6691/// \param Size Number of bytes to write.
6692/// \param Alignment Alignment of the destination in bytes.
6693/// \param isVol True if destination is volatile.
6694/// \param DstPtrInfo IR information on the memory pointer.
6695/// \returns New head in the control flow, if lowering was successful, empty
6696/// SDValue otherwise.
6697///
6698/// The function tries to replace 'llvm.memset' intrinsic with several store
6699/// operations and value calculation code. This is usually profitable for small
6700/// memory size.
6701static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
6702 SDValue Chain, SDValue Dst, SDValue Src,
6703 uint64_t Size, Align Alignment, bool isVol,
6704 MachinePointerInfo DstPtrInfo,
6705 const AAMDNodes &AAInfo) {
6706 // Turn a memset of undef to nop.
6707 // FIXME: We need to honor volatile even is Src is undef.
6708 if (Src.isUndef())
6709 return Chain;
6710
6711 // Expand memset to a series of load/store ops if the size operand
6712 // falls below a certain threshold.
6713 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6714 std::vector<EVT> MemOps;
6715 bool DstAlignCanChange = false;
6716 MachineFunction &MF = DAG.getMachineFunction();
6717 MachineFrameInfo &MFI = MF.getFrameInfo();
6718 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
6719 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6720 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6721 DstAlignCanChange = true;
6722 bool IsZeroVal =
6723 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
6724 if (!TLI.findOptimalMemOpLowering(
6725 MemOps, TLI.getMaxStoresPerMemset(OptSize),
6726 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
6727 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
6728 return SDValue();
6729
6730 if (DstAlignCanChange) {
6731 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
6732 Align NewAlign = DAG.getDataLayout().getABITypeAlign(Ty);
6733 if (NewAlign > Alignment) {
6734 // Give the stack frame object a larger alignment if needed.
6735 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
6736 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6737 Alignment = NewAlign;
6738 }
6739 }
6740
6741 SmallVector<SDValue, 8> OutChains;
6742 uint64_t DstOff = 0;
6743 unsigned NumMemOps = MemOps.size();
6744
6745 // Find the largest store and generate the bit pattern for it.
6746 EVT LargestVT = MemOps[0];
6747 for (unsigned i = 1; i < NumMemOps; i++)
6748 if (MemOps[i].bitsGT(LargestVT))
6749 LargestVT = MemOps[i];
6750 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
6751
6752 // Prepare AAInfo for loads/stores after lowering this memset.
6753 AAMDNodes NewAAInfo = AAInfo;
6754 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
6755
6756 for (unsigned i = 0; i < NumMemOps; i++) {
6757 EVT VT = MemOps[i];
6758 unsigned VTSize = VT.getSizeInBits() / 8;
6759 if (VTSize > Size) {
6760 // Issuing an unaligned load / store pair that overlaps with the previous
6761 // pair. Adjust the offset accordingly.
6762 assert(i == NumMemOps-1 && i != 0)((void)0);
6763 DstOff -= VTSize - Size;
6764 }
6765
6766 // If this store is smaller than the largest store see whether we can get
6767 // the smaller value for free with a truncate.
6768 SDValue Value = MemSetValue;
6769 if (VT.bitsLT(LargestVT)) {
6770 if (!LargestVT.isVector() && !VT.isVector() &&
6771 TLI.isTruncateFree(LargestVT, VT))
6772 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
6773 else
6774 Value = getMemsetValue(Src, VT, DAG, dl);
6775 }
6776 assert(Value.getValueType() == VT && "Value with wrong type.")((void)0);
6777 SDValue Store = DAG.getStore(
6778 Chain, dl, Value,
6779 DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl),
6780 DstPtrInfo.getWithOffset(DstOff), Alignment,
6781 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
6782 NewAAInfo);
6783 OutChains.push_back(Store);
6784 DstOff += VT.getSizeInBits() / 8;
6785 Size -= VTSize;
6786 }
6787
6788 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6789}
6790
6791static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
6792 unsigned AS) {
6793 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
6794 // pointer operands can be losslessly bitcasted to pointers of address space 0
6795 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
6796 report_fatal_error("cannot lower memory intrinsic in address space " +
6797 Twine(AS));
6798 }
6799}
6800
6801SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
6802 SDValue Src, SDValue Size, Align Alignment,
6803 bool isVol, bool AlwaysInline, bool isTailCall,
6804 MachinePointerInfo DstPtrInfo,
6805 MachinePointerInfo SrcPtrInfo,
6806 const AAMDNodes &AAInfo) {
6807 // Check to see if we should lower the memcpy to loads and stores first.
6808 // For cases within the target-specified limits, this is the best choice.
6809 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6810 if (ConstantSize) {
6811 // Memcpy with size zero? Just return the original chain.
6812 if (ConstantSize->isNullValue())
6813 return Chain;
6814
6815 SDValue Result = getMemcpyLoadsAndStores(
6816 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
6817 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
6818 if (Result.getNode())
6819 return Result;
6820 }
6821
6822 // Then check to see if we should lower the memcpy with target-specific
6823 // code. If the target chooses to do this, this is the next best.
6824 if (TSI) {
6825 SDValue Result = TSI->EmitTargetCodeForMemcpy(
6826 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
6827 DstPtrInfo, SrcPtrInfo);
6828 if (Result.getNode())
6829 return Result;
6830 }
6831
6832 // If we really need inline code and the target declined to provide it,
6833 // use a (potentially long) sequence of loads and stores.
6834 if (AlwaysInline) {
6835 assert(ConstantSize && "AlwaysInline requires a constant size!")((void)0);
6836 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
6837 ConstantSize->getZExtValue(), Alignment,
6838 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo);
6839 }
6840
6841 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6842 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6843
6844 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
6845 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
6846 // respect volatile, so they may do things like read or write memory
6847 // beyond the given memory regions. But fixing this isn't easy, and most
6848 // people don't care.
6849
6850 // Emit a library call.
6851 TargetLowering::ArgListTy Args;
6852 TargetLowering::ArgListEntry Entry;
6853 Entry.Ty = Type::getInt8PtrTy(*getContext());
6854 Entry.Node = Dst; Args.push_back(Entry);
6855 Entry.Node = Src; Args.push_back(Entry);
6856
6857 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6858 Entry.Node = Size; Args.push_back(Entry);
6859 // FIXME: pass in SDLoc
6860 TargetLowering::CallLoweringInfo CLI(*this);
6861 CLI.setDebugLoc(dl)
6862 .setChain(Chain)
6863 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
6864 Dst.getValueType().getTypeForEVT(*getContext()),
6865 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
6866 TLI->getPointerTy(getDataLayout())),
6867 std::move(Args))
6868 .setDiscardResult()
6869 .setTailCall(isTailCall);
6870
6871 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6872 return CallResult.second;
6873}
6874
6875SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
6876 SDValue Dst, unsigned DstAlign,
6877 SDValue Src, unsigned SrcAlign,
6878 SDValue Size, Type *SizeTy,
6879 unsigned ElemSz, bool isTailCall,
6880 MachinePointerInfo DstPtrInfo,
6881 MachinePointerInfo SrcPtrInfo) {
6882 // Emit a library call.
6883 TargetLowering::ArgListTy Args;
6884 TargetLowering::ArgListEntry Entry;
6885 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6886 Entry.Node = Dst;
6887 Args.push_back(Entry);
6888
6889 Entry.Node = Src;
6890 Args.push_back(Entry);
6891
6892 Entry.Ty = SizeTy;
6893 Entry.Node = Size;
6894 Args.push_back(Entry);
6895
6896 RTLIB::Libcall LibraryCall =
6897 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6898 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6899 report_fatal_error("Unsupported element size");
6900
6901 TargetLowering::CallLoweringInfo CLI(*this);
6902 CLI.setDebugLoc(dl)
6903 .setChain(Chain)
6904 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6905 Type::getVoidTy(*getContext()),
6906 getExternalSymbol(TLI->getLibcallName(LibraryCall),
6907 TLI->getPointerTy(getDataLayout())),
6908 std::move(Args))
6909 .setDiscardResult()
6910 .setTailCall(isTailCall);
6911
6912 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6913 return CallResult.second;
6914}
6915
6916SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
6917 SDValue Src, SDValue Size, Align Alignment,
6918 bool isVol, bool isTailCall,
6919 MachinePointerInfo DstPtrInfo,
6920 MachinePointerInfo SrcPtrInfo,
6921 const AAMDNodes &AAInfo) {
6922 // Check to see if we should lower the memmove to loads and stores first.
6923 // For cases within the target-specified limits, this is the best choice.
6924 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6925 if (ConstantSize) {
6926 // Memmove with size zero? Just return the original chain.
6927 if (ConstantSize->isNullValue())
6928 return Chain;
6929
6930 SDValue Result = getMemmoveLoadsAndStores(
6931 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
6932 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
6933 if (Result.getNode())
6934 return Result;
6935 }
6936
6937 // Then check to see if we should lower the memmove with target-specific
6938 // code. If the target chooses to do this, this is the next best.
6939 if (TSI) {
6940 SDValue Result =
6941 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
6942 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
6943 if (Result.getNode())
6944 return Result;
6945 }
6946
6947 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6948 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6949
6950 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
6951 // not be safe. See memcpy above for more details.
6952
6953 // Emit a library call.
6954 TargetLowering::ArgListTy Args;
6955 TargetLowering::ArgListEntry Entry;
6956 Entry.Ty = Type::getInt8PtrTy(*getContext());
6957 Entry.Node = Dst; Args.push_back(Entry);
6958 Entry.Node = Src; Args.push_back(Entry);
6959
6960 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6961 Entry.Node = Size; Args.push_back(Entry);
6962 // FIXME: pass in SDLoc
6963 TargetLowering::CallLoweringInfo CLI(*this);
6964 CLI.setDebugLoc(dl)
6965 .setChain(Chain)
6966 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
6967 Dst.getValueType().getTypeForEVT(*getContext()),
6968 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
6969 TLI->getPointerTy(getDataLayout())),
6970 std::move(Args))
6971 .setDiscardResult()
6972 .setTailCall(isTailCall);
6973
6974 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6975 return CallResult.second;
6976}
6977
6978SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
6979 SDValue Dst, unsigned DstAlign,
6980 SDValue Src, unsigned SrcAlign,
6981 SDValue Size, Type *SizeTy,
6982 unsigned ElemSz, bool isTailCall,
6983 MachinePointerInfo DstPtrInfo,
6984 MachinePointerInfo SrcPtrInfo) {
6985 // Emit a library call.
6986 TargetLowering::ArgListTy Args;
6987 TargetLowering::ArgListEntry Entry;
6988 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6989 Entry.Node = Dst;
6990 Args.push_back(Entry);
6991
6992 Entry.Node = Src;
6993 Args.push_back(Entry);
6994
6995 Entry.Ty = SizeTy;
6996 Entry.Node = Size;
6997 Args.push_back(Entry);
6998
6999 RTLIB::Libcall LibraryCall =
7000 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
7001 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
7002 report_fatal_error("Unsupported element size");
7003
7004 TargetLowering::CallLoweringInfo CLI(*this);
7005 CLI.setDebugLoc(dl)
7006 .setChain(Chain)
7007 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
7008 Type::getVoidTy(*getContext()),
7009 getExternalSymbol(TLI->getLibcallName(LibraryCall),
7010 TLI->getPointerTy(getDataLayout())),
7011 std::move(Args))
7012 .setDiscardResult()
7013 .setTailCall(isTailCall);
7014
7015 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
7016 return CallResult.second;
7017}
7018
7019SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
7020 SDValue Src, SDValue Size, Align Alignment,
7021 bool isVol, bool isTailCall,
7022 MachinePointerInfo DstPtrInfo,
7023 const AAMDNodes &AAInfo) {
7024 // Check to see if we should lower the memset to stores first.
7025 // For cases within the target-specified limits, this is the best choice.
7026 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
7027 if (ConstantSize) {
7028 // Memset with size zero? Just return the original chain.
7029 if (ConstantSize->isNullValue())
7030 return Chain;
7031
7032 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
7033 ConstantSize->getZExtValue(), Alignment,
7034 isVol, DstPtrInfo, AAInfo);
7035
7036 if (Result.getNode())
7037 return Result;
7038 }
7039
7040 // Then check to see if we should lower the memset with target-specific
7041 // code. If the target chooses to do this, this is the next best.
7042 if (TSI) {
7043 SDValue Result = TSI->EmitTargetCodeForMemset(
7044 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, DstPtrInfo);
7045 if (Result.getNode())
7046 return Result;
7047 }
7048
7049 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
7050
7051 // Emit a library call.
7052 TargetLowering::ArgListTy Args;
7053 TargetLowering::ArgListEntry Entry;
7054 Entry.Node = Dst; Entry.Ty = Type::getInt8PtrTy(*getContext());
7055 Args.push_back(Entry);
7056 Entry.Node = Src;
7057 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
7058 Args.push_back(Entry);
7059 Entry.Node = Size;
7060 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7061 Args.push_back(Entry);
7062
7063 // FIXME: pass in SDLoc
7064 TargetLowering::CallLoweringInfo CLI(*this);
7065 CLI.setDebugLoc(dl)
7066 .setChain(Chain)
7067 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
7068 Dst.getValueType().getTypeForEVT(*getContext()),
7069 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
7070 TLI->getPointerTy(getDataLayout())),
7071 std::move(Args))
7072 .setDiscardResult()
7073 .setTailCall(isTailCall);
7074
7075 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
7076 return CallResult.second;
7077}
7078
7079SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
7080 SDValue Dst, unsigned DstAlign,
7081 SDValue Value, SDValue Size, Type *SizeTy,
7082 unsigned ElemSz, bool isTailCall,
7083 MachinePointerInfo DstPtrInfo) {
7084 // Emit a library call.
7085 TargetLowering::ArgListTy Args;
7086 TargetLowering::ArgListEntry Entry;
7087 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7088 Entry.Node = Dst;
7089 Args.push_back(Entry);
7090
7091 Entry.Ty = Type::getInt8Ty(*getContext());
7092 Entry.Node = Value;
7093 Args.push_back(Entry);
7094
7095 Entry.Ty = SizeTy;
7096 Entry.Node = Size;
7097 Args.push_back(Entry);
7098
7099 RTLIB::Libcall LibraryCall =
7100 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
7101 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
7102 report_fatal_error("Unsupported element size");
7103
7104 TargetLowering::CallLoweringInfo CLI(*this);
7105 CLI.setDebugLoc(dl)
7106 .setChain(Chain)
7107 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
7108 Type::getVoidTy(*getContext()),
7109 getExternalSymbol(TLI->getLibcallName(LibraryCall),
7110 TLI->getPointerTy(getDataLayout())),
7111 std::move(Args))
7112 .setDiscardResult()
7113 .setTailCall(isTailCall);
7114
7115 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
7116 return CallResult.second;
7117}
7118
7119SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
7120 SDVTList VTList, ArrayRef<SDValue> Ops,
7121 MachineMemOperand *MMO) {
7122 FoldingSetNodeID ID;
7123 ID.AddInteger(MemVT.getRawBits());
7124 AddNodeIDNode(ID, Opcode, VTList, Ops);
7125 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7126 void* IP = nullptr;
7127 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7128 cast<AtomicSDNode>(E)->refineAlignment(MMO);
7129 return SDValue(E, 0);
7130 }
7131
7132 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
7133 VTList, MemVT, MMO);
7134 createOperands(N, Ops);
7135
7136 CSEMap.InsertNode(N, IP);
7137 InsertNode(N);
7138 return SDValue(N, 0);
7139}
7140
7141SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
7142 EVT MemVT, SDVTList VTs, SDValue Chain,
7143 SDValue Ptr, SDValue Cmp, SDValue Swp,
7144 MachineMemOperand *MMO) {
7145 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||((void)0)
7146 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS)((void)0);
7147 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types")((void)0);
7148
7149 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
7150 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
7151}
7152
7153SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
7154 SDValue Chain, SDValue Ptr, SDValue Val,
7155 MachineMemOperand *MMO) {
7156 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||((void)0)
7157 Opcode == ISD::ATOMIC_LOAD_SUB ||((void)0)
7158 Opcode == ISD::ATOMIC_LOAD_AND ||((void)0)
7159 Opcode == ISD::ATOMIC_LOAD_CLR ||((void)0)
7160 Opcode == ISD::ATOMIC_LOAD_OR ||((void)0)
7161 Opcode == ISD::ATOMIC_LOAD_XOR ||((void)0)
7162 Opcode == ISD::ATOMIC_LOAD_NAND ||((void)0)
7163 Opcode == ISD::ATOMIC_LOAD_MIN ||((void)0)
7164 Opcode == ISD::ATOMIC_LOAD_MAX ||((void)0)
7165 Opcode == ISD::ATOMIC_LOAD_UMIN ||((void)0)
7166 Opcode == ISD::ATOMIC_LOAD_UMAX ||((void)0)
7167 Opcode == ISD::ATOMIC_LOAD_FADD ||((void)0)
7168 Opcode == ISD::ATOMIC_LOAD_FSUB ||((void)0)
7169 Opcode == ISD::ATOMIC_SWAP ||((void)0)
7170 Opcode == ISD::ATOMIC_STORE) &&((void)0)
7171 "Invalid Atomic Op")((void)0);
7172
7173 EVT VT = Val.getValueType();
7174
7175 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
7176 getVTList(VT, MVT::Other);
7177 SDValue Ops[] = {Chain, Ptr, Val};
7178 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
7179}
7180
7181SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
7182 EVT VT, SDValue Chain, SDValue Ptr,
7183 MachineMemOperand *MMO) {
7184 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op")((void)0);
7185
7186 SDVTList VTs = getVTList(VT, MVT::Other);
7187 SDValue Ops[] = {Chain, Ptr};
7188 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
7189}
7190
7191/// getMergeValues - Create a MERGE_VALUES node from the given operands.
7192SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
7193 if (Ops.size() == 1)
7194 return Ops[0];
7195
7196 SmallVector<EVT, 4> VTs;
7197 VTs.reserve(Ops.size());
7198 for (const SDValue &Op : Ops)
7199 VTs.push_back(Op.getValueType());
7200 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
7201}
7202
7203SDValue SelectionDAG::getMemIntrinsicNode(
7204 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
7205 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
7206 MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
7207 if (!Size && MemVT.isScalableVector())
7208 Size = MemoryLocation::UnknownSize;
7209 else if (!Size)
7210 Size = MemVT.getStoreSize();
7211
7212 MachineFunction &MF = getMachineFunction();
7213 MachineMemOperand *MMO =
7214 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
7215
7216 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
7217}
7218
7219SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
7220 SDVTList VTList,
7221 ArrayRef<SDValue> Ops, EVT MemVT,
7222 MachineMemOperand *MMO) {
7223 assert((Opcode == ISD::INTRINSIC_VOID ||((void)0)
7224 Opcode == ISD::INTRINSIC_W_CHAIN ||((void)0)
7225 Opcode == ISD::PREFETCH ||((void)0)
7226 ((int)Opcode <= std::numeric_limits<int>::max() &&((void)0)
7227 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&((void)0)
7228 "Opcode is not a memory-accessing opcode!")((void)0);
7229
7230 // Memoize the node unless it returns a flag.
7231 MemIntrinsicSDNode *N;
7232 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
7233 FoldingSetNodeID ID;
7234 AddNodeIDNode(ID, Opcode, VTList, Ops);
7235 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
7236 Opcode, dl.getIROrder(), VTList, MemVT, MMO));
7237 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7238 void *IP = nullptr;
7239 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7240 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
7241 return SDValue(E, 0);
7242 }
7243
7244 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
7245 VTList, MemVT, MMO);
7246 createOperands(N, Ops);
7247
7248 CSEMap.InsertNode(N, IP);
7249 } else {
7250 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
7251 VTList, MemVT, MMO);
7252 createOperands(N, Ops);
7253 }
7254 InsertNode(N);
7255 SDValue V(N, 0);
7256 NewSDValueDbgMsg(V, "Creating new node: ", this);
7257 return V;
7258}
7259
7260SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
7261 SDValue Chain, int FrameIndex,
7262 int64_t Size, int64_t Offset) {
7263 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
7264 const auto VTs = getVTList(MVT::Other);
7265 SDValue Ops[2] = {
7266 Chain,
7267 getFrameIndex(FrameIndex,
7268 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
7269 true)};
7270
7271 FoldingSetNodeID ID;
7272 AddNodeIDNode(ID, Opcode, VTs, Ops);
7273 ID.AddInteger(FrameIndex);
7274 ID.AddInteger(Size);
7275 ID.AddInteger(Offset);
7276 void *IP = nullptr;
7277 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
7278 return SDValue(E, 0);
7279
7280 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
7281 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
7282 createOperands(N, Ops);
7283 CSEMap.InsertNode(N, IP);
7284 InsertNode(N);
7285 SDValue V(N, 0);
7286 NewSDValueDbgMsg(V, "Creating new node: ", this);
7287 return V;
7288}
7289
7290SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
7291 uint64_t Guid, uint64_t Index,
7292 uint32_t Attr) {
7293 const unsigned Opcode = ISD::PSEUDO_PROBE;
7294 const auto VTs = getVTList(MVT::Other);
7295 SDValue Ops[] = {Chain};
7296 FoldingSetNodeID ID;
7297 AddNodeIDNode(ID, Opcode, VTs, Ops);
7298 ID.AddInteger(Guid);
7299 ID.AddInteger(Index);
7300 void *IP = nullptr;
7301 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))
7302 return SDValue(E, 0);
7303
7304 auto *N = newSDNode<PseudoProbeSDNode>(
7305 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
7306 createOperands(N, Ops);
7307 CSEMap.InsertNode(N, IP);
7308 InsertNode(N);
7309 SDValue V(N, 0);
7310 NewSDValueDbgMsg(V, "Creating new node: ", this);
7311 return V;
7312}
7313
7314/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
7315/// MachinePointerInfo record from it. This is particularly useful because the
7316/// code generator has many cases where it doesn't bother passing in a
7317/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
7318static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
7319 SelectionDAG &DAG, SDValue Ptr,
7320 int64_t Offset = 0) {
7321 // If this is FI+Offset, we can model it.
7322 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
7323 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
7324 FI->getIndex(), Offset);
7325
7326 // If this is (FI+Offset1)+Offset2, we can model it.
7327 if (Ptr.getOpcode() != ISD::ADD ||
7328 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
7329 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
7330 return Info;
7331
7332 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
7333 return MachinePointerInfo::getFixedStack(
7334 DAG.getMachineFunction(), FI,
7335 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
7336}
7337
7338/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
7339/// MachinePointerInfo record from it. This is particularly useful because the
7340/// code generator has many cases where it doesn't bother passing in a
7341/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
7342static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
7343 SelectionDAG &DAG, SDValue Ptr,
7344 SDValue OffsetOp) {
7345 // If the 'Offset' value isn't a constant, we can't handle this.
7346 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
7347 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
7348 if (OffsetOp.isUndef())
7349 return InferPointerInfo(Info, DAG, Ptr);
7350 return Info;
7351}
7352
7353SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
7354 EVT VT, const SDLoc &dl, SDValue Chain,
7355 SDValue Ptr, SDValue Offset,
7356 MachinePointerInfo PtrInfo, EVT MemVT,
7357 Align Alignment,
7358 MachineMemOperand::Flags MMOFlags,
7359 const AAMDNodes &AAInfo, const MDNode *Ranges) {
7360 assert(Chain.getValueType() == MVT::Other &&((void)0)
7361 "Invalid chain type")((void)0);
7362
7363 MMOFlags |= MachineMemOperand::MOLoad;
7364 assert((MMOFlags & MachineMemOperand::MOStore) == 0)((void)0);
7365 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
7366 // clients.
7367 if (PtrInfo.V.isNull())
7368 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
7369
7370 uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
7371 MachineFunction &MF = getMachineFunction();
7372 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
7373 Alignment, AAInfo, Ranges);
7374 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
7375}
7376
7377SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
7378 EVT VT, const SDLoc &dl, SDValue Chain,
7379 SDValue Ptr, SDValue Offset, EVT MemVT,
7380 MachineMemOperand *MMO) {
7381 if (VT == MemVT) {
7382 ExtType = ISD::NON_EXTLOAD;
7383 } else if (ExtType == ISD::NON_EXTLOAD) {
7384 assert(VT == MemVT && "Non-extending load from different memory type!")((void)0);
7385 } else {
7386 // Extending load.
7387 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&((void)0)
7388 "Should only be an extending load, not truncating!")((void)0);
7389 assert(VT.isInteger() == MemVT.isInteger() &&((void)0)
7390 "Cannot convert from FP to Int or Int -> FP!")((void)0);
7391 assert(VT.isVector() == MemVT.isVector() &&((void)0)
7392 "Cannot use an ext load to convert to or from a vector!")((void)0);
7393 assert((!VT.isVector() ||((void)0)
7394 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&((void)0)
7395 "Cannot use an ext load to change the number of vector elements!")((void)0);
7396 }
7397
7398 bool Indexed = AM != ISD::UNINDEXED;
7399 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!")((void)0);
7400
7401 SDVTList VTs = Indexed ?
7402 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
7403 SDValue Ops[] = { Chain, Ptr, Offset };
7404 FoldingSetNodeID ID;
7405 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
7406 ID.AddInteger(MemVT.getRawBits());
7407 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
7408 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
7409 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7410 void *IP = nullptr;
7411 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7412 cast<LoadSDNode>(E)->refineAlignment(MMO);
7413 return SDValue(E, 0);
7414 }
7415 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7416 ExtType, MemVT, MMO);
7417 createOperands(N, Ops);
7418
7419 CSEMap.InsertNode(N, IP);
7420 InsertNode(N);
7421 SDValue V(N, 0);
7422 NewSDValueDbgMsg(V, "Creating new node: ", this);
7423 return V;
7424}
7425
7426SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7427 SDValue Ptr, MachinePointerInfo PtrInfo,
7428 MaybeAlign Alignment,
7429 MachineMemOperand::Flags MMOFlags,
7430 const AAMDNodes &AAInfo, const MDNode *Ranges) {
7431 SDValue Undef = getUNDEF(Ptr.getValueType());
7432 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
7433 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
7434}
7435
7436SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7437 SDValue Ptr, MachineMemOperand *MMO) {
7438 SDValue Undef = getUNDEF(Ptr.getValueType());
7439 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
7440 VT, MMO);
7441}
7442
7443SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
7444 EVT VT, SDValue Chain, SDValue Ptr,
7445 MachinePointerInfo PtrInfo, EVT MemVT,
7446 MaybeAlign Alignment,
7447 MachineMemOperand::Flags MMOFlags,
7448 const AAMDNodes &AAInfo) {
7449 SDValue Undef = getUNDEF(Ptr.getValueType());
7450 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
7451 MemVT, Alignment, MMOFlags, AAInfo);
7452}
7453
7454SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
7455 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
7456 MachineMemOperand *MMO) {
7457 SDValue Undef = getUNDEF(Ptr.getValueType());
7458 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
7459 MemVT, MMO);
7460}
7461
7462SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
7463 SDValue Base, SDValue Offset,
7464 ISD::MemIndexedMode AM) {
7465 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
7466 assert(LD->getOffset().isUndef() && "Load is already a indexed load!")((void)0);
7467 // Don't propagate the invariant or dereferenceable flags.
7468 auto MMOFlags =
7469 LD->getMemOperand()->getFlags() &
7470 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
7471 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
7472 LD->getChain(), Base, Offset, LD->getPointerInfo(),
7473 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
7474}
7475
7476SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7477 SDValue Ptr, MachinePointerInfo PtrInfo,
7478 Align Alignment,
7479 MachineMemOperand::Flags MMOFlags,
7480 const AAMDNodes &AAInfo) {
7481 assert(Chain.getValueType() == MVT::Other && "Invalid chain type")((void)0);
7482
7483 MMOFlags |= MachineMemOperand::MOStore;
7484 assert((MMOFlags & MachineMemOperand::MOLoad) == 0)((void)0);
7485
7486 if (PtrInfo.V.isNull())
7487 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7488
7489 MachineFunction &MF = getMachineFunction();
7490 uint64_t Size =
7491 MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
7492 MachineMemOperand *MMO =
7493 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
7494 return getStore(Chain, dl, Val, Ptr, MMO);
7495}
7496
7497SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7498 SDValue Ptr, MachineMemOperand *MMO) {
7499 assert(Chain.getValueType() == MVT::Other &&((void)0)
7500 "Invalid chain type")((void)0);
7501 EVT VT = Val.getValueType();
7502 SDVTList VTs = getVTList(MVT::Other);
7503 SDValue Undef = getUNDEF(Ptr.getValueType());
7504 SDValue Ops[] = { Chain, Val, Ptr, Undef };
7505 FoldingSetNodeID ID;
7506 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7507 ID.AddInteger(VT.getRawBits());
7508 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7509 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
7510 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7511 void *IP = nullptr;
7512 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7513 cast<StoreSDNode>(E)->refineAlignment(MMO);
7514 return SDValue(E, 0);
7515 }
7516 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7517 ISD::UNINDEXED, false, VT, MMO);
7518 createOperands(N, Ops);
7519
7520 CSEMap.InsertNode(N, IP);
7521 InsertNode(N);
7522 SDValue V(N, 0);
7523 NewSDValueDbgMsg(V, "Creating new node: ", this);
7524 return V;
7525}
7526
7527SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7528 SDValue Ptr, MachinePointerInfo PtrInfo,
7529 EVT SVT, Align Alignment,
7530 MachineMemOperand::Flags MMOFlags,
7531 const AAMDNodes &AAInfo) {
7532 assert(Chain.getValueType() == MVT::Other &&((void)0)
7533 "Invalid chain type")((void)0);
7534
7535 MMOFlags |= MachineMemOperand::MOStore;
7536 assert((MMOFlags & MachineMemOperand::MOLoad) == 0)((void)0);
7537
7538 if (PtrInfo.V.isNull())
7539 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7540
7541 MachineFunction &MF = getMachineFunction();
7542 MachineMemOperand *MMO = MF.getMachineMemOperand(
7543 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
7544 Alignment, AAInfo);
7545 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
7546}
7547
7548SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7549 SDValue Ptr, EVT SVT,
7550 MachineMemOperand *MMO) {
7551 EVT VT = Val.getValueType();
7552
7553 assert(Chain.getValueType() == MVT::Other &&((void)0)
7554 "Invalid chain type")((void)0);
7555 if (VT == SVT)
7556 return getStore(Chain, dl, Val, Ptr, MMO);
7557
7558 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&((void)0)
7559 "Should only be a truncating store, not extending!")((void)0);
7560 assert(VT.isInteger() == SVT.isInteger() &&((void)0)
7561 "Can't do FP-INT conversion!")((void)0);
7562 assert(VT.isVector() == SVT.isVector() &&((void)0)
7563 "Cannot use trunc store to convert to or from a vector!")((void)0);
7564 assert((!VT.isVector() ||((void)0)
7565 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&((void)0)
7566 "Cannot use trunc store to change the number of vector elements!")((void)0);
7567
7568 SDVTList VTs = getVTList(MVT::Other);
7569 SDValue Undef = getUNDEF(Ptr.getValueType());
7570 SDValue Ops[] = { Chain, Val, Ptr, Undef };
7571 FoldingSetNodeID ID;
7572 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7573 ID.AddInteger(SVT.getRawBits());
7574 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7575 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
7576 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7577 void *IP = nullptr;
7578 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7579 cast<StoreSDNode>(E)->refineAlignment(MMO);
7580 return SDValue(E, 0);
7581 }
7582 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7583 ISD::UNINDEXED, true, SVT, MMO);
7584 createOperands(N, Ops);
7585
7586 CSEMap.InsertNode(N, IP);
7587 InsertNode(N);
7588 SDValue V(N, 0);
7589 NewSDValueDbgMsg(V, "Creating new node: ", this);
7590 return V;
7591}
7592
7593SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
7594 SDValue Base, SDValue Offset,
7595 ISD::MemIndexedMode AM) {
7596 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
7597 assert(ST->getOffset().isUndef() && "Store is already a indexed store!")((void)0);
7598 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
7599 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
7600 FoldingSetNodeID ID;
7601 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7602 ID.AddInteger(ST->getMemoryVT().getRawBits());
7603 ID.AddInteger(ST->getRawSubclassData());
7604 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
7605 void *IP = nullptr;
7606 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
7607 return SDValue(E, 0);
7608
7609 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7610 ST->isTruncatingStore(), ST->getMemoryVT(),
7611 ST->getMemOperand());
7612 createOperands(N, Ops);
7613
7614 CSEMap.InsertNode(N, IP);
7615 InsertNode(N);
7616 SDValue V(N, 0);
7617 NewSDValueDbgMsg(V, "Creating new node: ", this);
7618 return V;
7619}
7620
7621SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7622 SDValue Base, SDValue Offset, SDValue Mask,
7623 SDValue PassThru, EVT MemVT,
7624 MachineMemOperand *MMO,
7625 ISD::MemIndexedMode AM,
7626 ISD::LoadExtType ExtTy, bool isExpanding) {
7627 bool Indexed = AM != ISD::UNINDEXED;
7628 assert((Indexed || Offset.isUndef()) &&((void)0)
7629 "Unindexed masked load with an offset!")((void)0);
7630 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
7631 : getVTList(VT, MVT::Other);
7632 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
7633 FoldingSetNodeID ID;
7634 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
7635 ID.AddInteger(MemVT.getRawBits());
7636 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
7637 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
7638 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7639 void *IP = nullptr;
7640 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7641 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
7642 return SDValue(E, 0);
7643 }
7644 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7645 AM, ExtTy, isExpanding, MemVT, MMO);
7646 createOperands(N, Ops);
7647
7648 CSEMap.InsertNode(N, IP);
7649 InsertNode(N);
7650 SDValue V(N, 0);
7651 NewSDValueDbgMsg(V, "Creating new node: ", this);
7652 return V;
7653}
7654
7655SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
7656 SDValue Base, SDValue Offset,
7657 ISD::MemIndexedMode AM) {
7658 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
7659 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!")((void)0);
7660 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
7661 Offset, LD->getMask(), LD->getPassThru(),
7662 LD->getMemoryVT(), LD->getMemOperand(), AM,
7663 LD->getExtensionType(), LD->isExpandingLoad());
7664}
7665
7666SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
7667 SDValue Val, SDValue Base, SDValue Offset,
7668 SDValue Mask, EVT MemVT,
7669 MachineMemOperand *MMO,
7670 ISD::MemIndexedMode AM, bool IsTruncating,
7671 bool IsCompressing) {
7672 assert(Chain.getValueType() == MVT::Other &&((void)0)
7673 "Invalid chain type")((void)0);
7674 bool Indexed = AM != ISD::UNINDEXED;
7675 assert((Indexed || Offset.isUndef()) &&((void)0)
7676 "Unindexed masked store with an offset!")((void)0);
7677 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
7678 : getVTList(MVT::Other);
7679 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
7680 FoldingSetNodeID ID;
7681 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
7682 ID.AddInteger(MemVT.getRawBits());
7683 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
7684 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
7685 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7686 void *IP = nullptr;
7687 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7688 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
7689 return SDValue(E, 0);
7690 }
7691 auto *N =
7692 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7693 IsTruncating, IsCompressing, MemVT, MMO);
7694 createOperands(N, Ops);
7695
7696 CSEMap.InsertNode(N, IP);
7697 InsertNode(N);
7698 SDValue V(N, 0);
7699 NewSDValueDbgMsg(V, "Creating new node: ", this);
7700 return V;
7701}
7702
7703SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
7704 SDValue Base, SDValue Offset,
7705 ISD::MemIndexedMode AM) {
7706 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
7707 assert(ST->getOffset().isUndef() &&((void)0)
7708 "Masked store is already a indexed store!")((void)0);
7709 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
7710 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
7711 AM, ST->isTruncatingStore(), ST->isCompressingStore());
7712}
7713
7714SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
7715 ArrayRef<SDValue> Ops,
7716 MachineMemOperand *MMO,
7717 ISD::MemIndexType IndexType,
7718 ISD::LoadExtType ExtTy) {
7719 assert(Ops.size() == 6 && "Incompatible number of operands")((void)0);
7720
7721 FoldingSetNodeID ID;
7722 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
7723 ID.AddInteger(MemVT.getRawBits());
7724 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
7725 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
7726 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7727 void *IP = nullptr;
7728 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7729 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
7730 return SDValue(E, 0);
7731 }
7732
7733 IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]);
7734 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7735 VTs, MemVT, MMO, IndexType, ExtTy);
7736 createOperands(N, Ops);
7737
7738 assert(N->getPassThru().getValueType() == N->getValueType(0) &&((void)0)
7739 "Incompatible type of the PassThru value in MaskedGatherSDNode")((void)0);
7740 assert(N->getMask().getValueType().getVectorElementCount() ==((void)0)
7741 N->getValueType(0).getVectorElementCount() &&((void)0)
7742 "Vector width mismatch between mask and data")((void)0);
7743 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==((void)0)
7744 N->getValueType(0).getVectorElementCount().isScalable() &&((void)0)
7745 "Scalable flags of index and data do not match")((void)0);
7746 assert(ElementCount::isKnownGE(((void)0)
7747 N->getIndex().getValueType().getVectorElementCount(),((void)0)
7748 N->getValueType(0).getVectorElementCount()) &&((void)0)
7749 "Vector width mismatch between index and data")((void)0);
7750 assert(isa<ConstantSDNode>(N->getScale()) &&((void)0)
7751 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&((void)0)
7752 "Scale should be a constant power of 2")((void)0);
7753
7754 CSEMap.InsertNode(N, IP);
7755 InsertNode(N);
7756 SDValue V(N, 0);
7757 NewSDValueDbgMsg(V, "Creating new node: ", this);
7758 return V;
7759}
7760
7761SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
7762 ArrayRef<SDValue> Ops,
7763 MachineMemOperand *MMO,
7764 ISD::MemIndexType IndexType,
7765 bool IsTrunc) {
7766 assert(Ops.size() == 6 && "Incompatible number of operands")((void)0);
7767
7768 FoldingSetNodeID ID;
7769 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
7770 ID.AddInteger(MemVT.getRawBits());
7771 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
7772 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
7773 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7774 void *IP = nullptr;
7775 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7776 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
7777 return SDValue(E, 0);
7778 }
7779
7780 IndexType = TLI->getCanonicalIndexType(IndexType, MemVT, Ops[4]);
7781 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7782 VTs, MemVT, MMO, IndexType, IsTrunc);
7783 createOperands(N, Ops);
7784
7785 assert(N->getMask().getValueType().getVectorElementCount() ==((void)0)
7786 N->getValue().getValueType().getVectorElementCount() &&((void)0)
7787 "Vector width mismatch between mask and data")((void)0);
7788 assert(((void)0)
7789 N->getIndex().getValueType().getVectorElementCount().isScalable() ==((void)0)
7790 N->getValue().getValueType().getVectorElementCount().isScalable() &&((void)0)
7791 "Scalable flags of index and data do not match")((void)0);
7792 assert(ElementCount::isKnownGE(((void)0)
7793 N->getIndex().getValueType().getVectorElementCount(),((void)0)
7794 N->getValue().getValueType().getVectorElementCount()) &&((void)0)
7795 "Vector width mismatch between index and data")((void)0);
7796 assert(isa<ConstantSDNode>(N->getScale()) &&((void)0)
7797 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&((void)0)
7798 "Scale should be a constant power of 2")((void)0);
7799
7800 CSEMap.InsertNode(N, IP);
7801 InsertNode(N);
7802 SDValue V(N, 0);
7803 NewSDValueDbgMsg(V, "Creating new node: ", this);
7804 return V;
7805}
7806
7807SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7808 // select undef, T, F --> T (if T is a constant), otherwise F
7809 // select, ?, undef, F --> F
7810 // select, ?, T, undef --> T
7811 if (Cond.isUndef())
7812 return isConstantValueOfAnyType(T) ? T : F;
7813 if (T.isUndef())
7814 return F;
7815 if (F.isUndef())
7816 return T;
7817
7818 // select true, T, F --> T
7819 // select false, T, F --> F
7820 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
7821 return CondC->isNullValue() ? F : T;
7822
7823 // TODO: This should simplify VSELECT with constant condition using something
7824 // like this (but check boolean contents to be complete?):
7825 // if (ISD::isBuildVectorAllOnes(Cond.getNode()))
7826 // return T;
7827 // if (ISD::isBuildVectorAllZeros(Cond.getNode()))
7828 // return F;
7829
7830 // select ?, T, T --> T
7831 if (T == F)
7832 return T;
7833
7834 return SDValue();
7835}
7836
7837SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
7838 // shift undef, Y --> 0 (can always assume that the undef value is 0)
7839 if (X.isUndef())
7840 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
7841 // shift X, undef --> undef (because it may shift by the bitwidth)
7842 if (Y.isUndef())
7843 return getUNDEF(X.getValueType());
7844
7845 // shift 0, Y --> 0
7846 // shift X, 0 --> X
7847 if (isNullOrNullSplat(X) || isNullOrNullSplat(Y))
7848 return X;
7849
7850 // shift X, C >= bitwidth(X) --> undef
7851 // All vector elements must be too big (or undef) to avoid partial undefs.
7852 auto isShiftTooBig = [X](ConstantSDNode *Val) {
7853 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
7854 };
7855 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
7856 return getUNDEF(X.getValueType());
7857
7858 return SDValue();
7859}
7860
7861SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
7862 SDNodeFlags Flags) {
7863 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
7864 // (an undef operand can be chosen to be Nan/Inf), then the result of this
7865 // operation is poison. That result can be relaxed to undef.
7866 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
7867 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
7868 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
7869 (YC && YC->getValueAPF().isNaN());
7870 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
7871 (YC && YC->getValueAPF().isInfinity());
7872
7873 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
7874 return getUNDEF(X.getValueType());
7875
7876 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
7877 return getUNDEF(X.getValueType());
7878
7879 if (!YC)
7880 return SDValue();
7881
7882 // X + -0.0 --> X
7883 if (Opcode == ISD::FADD)
7884 if (YC->getValueAPF().isNegZero())
7885 return X;
7886
7887 // X - +0.0 --> X
7888 if (Opcode == ISD::FSUB)
7889 if (YC->getValueAPF().isPosZero())
7890 return X;
7891
7892 // X * 1.0 --> X
7893 // X / 1.0 --> X
7894 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
7895 if (YC->getValueAPF().isExactlyValue(1.0))
7896 return X;
7897
7898 // X * 0.0 --> 0.0
7899 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
7900 if (YC->getValueAPF().isZero())
7901 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
7902
7903 return SDValue();
7904}
7905
7906SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
7907 SDValue Ptr, SDValue SV, unsigned Align) {
7908 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
7909 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
7910}
7911
7912SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7913 ArrayRef<SDUse> Ops) {
7914 switch (Ops.size()) {
7915 case 0: return getNode(Opcode, DL, VT);
7916 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
7917 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
7918 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
7919 default: break;
7920 }
7921
7922 // Copy from an SDUse array into an SDValue array for use with
7923 // the regular getNode logic.
7924 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
7925 return getNode(Opcode, DL, VT, NewOps);
7926}
7927
7928SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7929 ArrayRef<SDValue> Ops) {
7930 SDNodeFlags Flags;
7931 if (Inserter)
7932 Flags = Inserter->getFlags();
7933 return getNode(Opcode, DL, VT, Ops, Flags);
7934}
7935
7936SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7937 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
7938 unsigned NumOps = Ops.size();
7939 switch (NumOps) {
7940 case 0: return getNode(Opcode, DL, VT);
7941 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
7942 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
7943 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
7944 default: break;
7945 }
7946
7947#ifndef NDEBUG1
7948 for (auto &Op : Ops)
7949 assert(Op.getOpcode() != ISD::DELETED_NODE &&((void)0)
7950 "Operand is DELETED_NODE!")((void)0);
7951#endif
7952
7953 switch (Opcode) {
7954 default: break;
7955 case ISD::BUILD_VECTOR:
7956 // Attempt to simplify BUILD_VECTOR.
7957 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7958 return V;
7959 break;
7960 case ISD::CONCAT_VECTORS:
7961 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7962 return V;
7963 break;
7964 case ISD::SELECT_CC:
7965 assert(NumOps == 5 && "SELECT_CC takes 5 operands!")((void)0);
7966 assert(Ops[0].getValueType() == Ops[1].getValueType() &&((void)0)
7967 "LHS and RHS of condition must have same type!")((void)0);
7968 assert(Ops[2].getValueType() == Ops[3].getValueType() &&((void)0)
7969 "True and False arms of SelectCC must have same type!")((void)0);
7970 assert(Ops[2].getValueType() == VT &&((void)0)
7971 "select_cc node must be of same type as true and false value!")((void)0);
7972 break;
7973 case ISD::BR_CC:
7974 assert(NumOps == 5 && "BR_CC takes 5 operands!")((void)0);
7975 assert(Ops[2].getValueType() == Ops[3].getValueType() &&((void)0)
7976 "LHS/RHS of comparison should match types!")((void)0);
7977 break;
7978 }
7979
7980 // Memoize nodes.
7981 SDNode *N;
7982 SDVTList VTs = getVTList(VT);
7983
7984 if (VT != MVT::Glue) {
7985 FoldingSetNodeID ID;
7986 AddNodeIDNode(ID, Opcode, VTs, Ops);
7987 void *IP = nullptr;
7988
7989 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
7990 return SDValue(E, 0);
7991
7992 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7993 createOperands(N, Ops);
7994
7995 CSEMap.InsertNode(N, IP);
7996 } else {
7997 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7998 createOperands(N, Ops);
7999 }
8000
8001 N->setFlags(Flags);
8002 InsertNode(N);
8003 SDValue V(N, 0);
8004 NewSDValueDbgMsg(V, "Creating new node: ", this);
8005 return V;
8006}
8007
8008SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
8009 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
8010 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
8011}
8012
8013SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8014 ArrayRef<SDValue> Ops) {
8015 SDNodeFlags Flags;
8016 if (Inserter)
8017 Flags = Inserter->getFlags();
8018 return getNode(Opcode, DL, VTList, Ops, Flags);
8019}
8020
8021SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8022 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
8023 if (VTList.NumVTs == 1)
8024 return getNode(Opcode, DL, VTList.VTs[0], Ops);
8025
8026#ifndef NDEBUG1
8027 for (auto &Op : Ops)
8028 assert(Op.getOpcode() != ISD::DELETED_NODE &&((void)0)
8029 "Operand is DELETED_NODE!")((void)0);
8030#endif
8031
8032 switch (Opcode) {
8033 case ISD::STRICT_FP_EXTEND:
8034 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&((void)0)
8035 "Invalid STRICT_FP_EXTEND!")((void)0);
8036 assert(VTList.VTs[0].isFloatingPoint() &&((void)0)
8037 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!")((void)0);
8038 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&((void)0)
8039 "STRICT_FP_EXTEND result type should be vector iff the operand "((void)0)
8040 "type is vector!")((void)0);
8041 assert((!VTList.VTs[0].isVector() ||((void)0)
8042 VTList.VTs[0].getVectorNumElements() ==((void)0)
8043 Ops[1].getValueType().getVectorNumElements()) &&((void)0)
8044 "Vector element count mismatch!")((void)0);
8045 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&((void)0)
8046 "Invalid fpext node, dst <= src!")((void)0);
8047 break;
8048 case ISD::STRICT_FP_ROUND:
8049 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!")((void)0);
8050 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&((void)0)
8051 "STRICT_FP_ROUND result type should be vector iff the operand "((void)0)
8052 "type is vector!")((void)0);
8053 assert((!VTList.VTs[0].isVector() ||((void)0)
8054 VTList.VTs[0].getVectorNumElements() ==((void)0)
8055 Ops[1].getValueType().getVectorNumElements()) &&((void)0)
8056 "Vector element count mismatch!")((void)0);
8057 assert(VTList.VTs[0].isFloatingPoint() &&((void)0)
8058 Ops[1].getValueType().isFloatingPoint() &&((void)0)
8059 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&((void)0)
8060 isa<ConstantSDNode>(Ops[2]) &&((void)0)
8061 (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 ||((void)0)
8062 cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) &&((void)0)
8063 "Invalid STRICT_FP_ROUND!")((void)0);
8064 break;
8065#if 0
8066 // FIXME: figure out how to safely handle things like
8067 // int foo(int x) { return 1 << (x & 255); }
8068 // int bar() { return foo(256); }
8069 case ISD::SRA_PARTS:
8070 case ISD::SRL_PARTS:
8071 case ISD::SHL_PARTS:
8072 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
8073 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
8074 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
8075 else if (N3.getOpcode() == ISD::AND)
8076 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
8077 // If the and is only masking out bits that cannot effect the shift,
8078 // eliminate the and.
8079 unsigned NumBits = VT.getScalarSizeInBits()*2;
8080 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
8081 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
8082 }
8083 break;
8084#endif
8085 }
8086
8087 // Memoize the node unless it returns a flag.
8088 SDNode *N;
8089 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8090 FoldingSetNodeID ID;
8091 AddNodeIDNode(ID, Opcode, VTList, Ops);
8092 void *IP = nullptr;
8093 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
8094 return SDValue(E, 0);
8095
8096 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
8097 createOperands(N, Ops);
8098 CSEMap.InsertNode(N, IP);
8099 } else {
8100 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
8101 createOperands(N, Ops);
8102 }
8103
8104 N->setFlags(Flags);
8105 InsertNode(N);
8106 SDValue V(N, 0);
8107 NewSDValueDbgMsg(V, "Creating new node: ", this);
8108 return V;
8109}
8110
8111SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
8112 SDVTList VTList) {
8113 return getNode(Opcode, DL, VTList, None);
8114}
8115
8116SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8117 SDValue N1) {
8118 SDValue Ops[] = { N1 };
8119 return getNode(Opcode, DL, VTList, Ops);
8120}
8121
8122SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8123 SDValue N1, SDValue N2) {
8124 SDValue Ops[] = { N1, N2 };
8125 return getNode(Opcode, DL, VTList, Ops);
8126}
8127
8128SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8129 SDValue N1, SDValue N2, SDValue N3) {
8130 SDValue Ops[] = { N1, N2, N3 };
8131 return getNode(Opcode, DL, VTList, Ops);
8132}
8133
8134SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8135 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
8136 SDValue Ops[] = { N1, N2, N3, N4 };
8137 return getNode(Opcode, DL, VTList, Ops);
8138}
8139
8140SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
8141 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8142 SDValue N5) {
8143 SDValue Ops[] = { N1, N2, N3, N4, N5 };
8144 return getNode(Opcode, DL, VTList, Ops);
8145}
8146
8147SDVTList SelectionDAG::getVTList(EVT VT) {
8148 return makeVTList(SDNode::getValueTypeList(VT), 1);
8149}
8150
8151SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
8152 FoldingSetNodeID ID;
8153 ID.AddInteger(2U);
8154 ID.AddInteger(VT1.getRawBits());
8155 ID.AddInteger(VT2.getRawBits());
8156
8157 void *IP = nullptr;
8158 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8159 if (!Result) {
8160 EVT *Array = Allocator.Allocate<EVT>(2);
8161 Array[0] = VT1;
8162 Array[1] = VT2;
8163 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
8164 VTListMap.InsertNode(Result, IP);
8165 }
8166 return Result->getSDVTList();
8167}
8168
8169SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
8170 FoldingSetNodeID ID;
8171 ID.AddInteger(3U);
8172 ID.AddInteger(VT1.getRawBits());
8173 ID.AddInteger(VT2.getRawBits());
8174 ID.AddInteger(VT3.getRawBits());
8175
8176 void *IP = nullptr;
8177 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8178 if (!Result) {
8179 EVT *Array = Allocator.Allocate<EVT>(3);
8180 Array[0] = VT1;
8181 Array[1] = VT2;
8182 Array[2] = VT3;
8183 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
8184 VTListMap.InsertNode(Result, IP);
8185 }
8186 return Result->getSDVTList();
8187}
8188
8189SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
8190 FoldingSetNodeID ID;
8191 ID.AddInteger(4U);
8192 ID.AddInteger(VT1.getRawBits());
8193 ID.AddInteger(VT2.getRawBits());
8194 ID.AddInteger(VT3.getRawBits());
8195 ID.AddInteger(VT4.getRawBits());
8196
8197 void *IP = nullptr;
8198 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8199 if (!Result) {
8200 EVT *Array = Allocator.Allocate<EVT>(4);
8201 Array[0] = VT1;
8202 Array[1] = VT2;
8203 Array[2] = VT3;
8204 Array[3] = VT4;
8205 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
8206 VTListMap.InsertNode(Result, IP);
8207 }
8208 return Result->getSDVTList();
8209}
8210
8211SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
8212 unsigned NumVTs = VTs.size();
8213 FoldingSetNodeID ID;
8214 ID.AddInteger(NumVTs);
8215 for (unsigned index = 0; index < NumVTs; index++) {
8216 ID.AddInteger(VTs[index].getRawBits());
8217 }
8218
8219 void *IP = nullptr;
8220 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
8221 if (!Result) {
8222 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
8223 llvm::copy(VTs, Array);
8224 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
8225 VTListMap.InsertNode(Result, IP);
8226 }
8227 return Result->getSDVTList();
8228}
8229
8230
8231/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
8232/// specified operands. If the resultant node already exists in the DAG,
8233/// this does not modify the specified node, instead it returns the node that
8234/// already exists. If the resultant node does not exist in the DAG, the
8235/// input node is returned. As a degenerate case, if you specify the same
8236/// input operands as the node already has, the input node is returned.
8237SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
8238 assert(N->getNumOperands() == 1 && "Update with wrong number of operands")((void)0);
8239
8240 // Check to see if there is no change.
8241 if (Op == N->getOperand(0)) return N;
8242
8243 // See if the modified node already exists.
8244 void *InsertPos = nullptr;
8245 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
8246 return Existing;
8247
8248 // Nope it doesn't. Remove the node from its current place in the maps.
8249 if (InsertPos)
8250 if (!RemoveNodeFromCSEMaps(N))
8251 InsertPos = nullptr;
8252
8253 // Now we update the operands.
8254 N->OperandList[0].set(Op);
8255
8256 updateDivergence(N);
8257 // If this gets put into a CSE map, add it.
8258 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
8259 return N;
8260}
8261
8262SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
8263 assert(N->getNumOperands() == 2 && "Update with wrong number of operands")((void)0);
8264
8265 // Check to see if there is no change.
8266 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
8267 return N; // No operands changed, just return the input node.
8268
8269 // See if the modified node already exists.
8270 void *InsertPos = nullptr;
8271 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
8272 return Existing;
8273
8274 // Nope it doesn't. Remove the node from its current place in the maps.
8275 if (InsertPos)
8276 if (!RemoveNodeFromCSEMaps(N))
8277 InsertPos = nullptr;
8278
8279 // Now we update the operands.
8280 if (N->OperandList[0] != Op1)
8281 N->OperandList[0].set(Op1);
8282 if (N->OperandList[1] != Op2)
8283 N->OperandList[1].set(Op2);
8284
8285 updateDivergence(N);
8286 // If this gets put into a CSE map, add it.
8287 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
8288 return N;
8289}
8290
8291SDNode *SelectionDAG::
8292UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
8293 SDValue Ops[] = { Op1, Op2, Op3 };
8294 return UpdateNodeOperands(N, Ops);
8295}
8296
8297SDNode *SelectionDAG::
8298UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
8299 SDValue Op3, SDValue Op4) {
8300 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
8301 return UpdateNodeOperands(N, Ops);
8302}
8303
8304SDNode *SelectionDAG::
8305UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
8306 SDValue Op3, SDValue Op4, SDValue Op5) {
8307 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
8308 return UpdateNodeOperands(N, Ops);
8309}
8310
8311SDNode *SelectionDAG::
8312UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
8313 unsigned NumOps = Ops.size();
8314 assert(N->getNumOperands() == NumOps &&((void)0)
8315 "Update with wrong number of operands")((void)0);
8316
8317 // If no operands changed just return the input node.
8318 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
8319 return N;
8320
8321 // See if the modified node already exists.
8322 void *InsertPos = nullptr;
8323 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
8324 return Existing;
8325
8326 // Nope it doesn't. Remove the node from its current place in the maps.
8327 if (InsertPos)
8328 if (!RemoveNodeFromCSEMaps(N))
8329 InsertPos = nullptr;
8330
8331 // Now we update the operands.
8332 for (unsigned i = 0; i != NumOps; ++i)
8333 if (N->OperandList[i] != Ops[i])
8334 N->OperandList[i].set(Ops[i]);
8335
8336 updateDivergence(N);
8337 // If this gets put into a CSE map, add it.
8338 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
8339 return N;
8340}
8341
8342/// DropOperands - Release the operands and set this node to have
8343/// zero operands.
8344void SDNode::DropOperands() {
8345 // Unlike the code in MorphNodeTo that does this, we don't need to
8346 // watch for dead nodes here.
8347 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
8348 SDUse &Use = *I++;
8349 Use.set(SDValue());
8350 }
8351}
8352
8353void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
8354 ArrayRef<MachineMemOperand *> NewMemRefs) {
8355 if (NewMemRefs.empty()) {
8356 N->clearMemRefs();
8357 return;
8358 }
8359
8360 // Check if we can avoid allocating by storing a single reference directly.
8361 if (NewMemRefs.size() == 1) {
8362 N->MemRefs = NewMemRefs[0];
8363 N->NumMemRefs = 1;
8364 return;
8365 }
8366
8367 MachineMemOperand **MemRefsBuffer =
8368 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
8369 llvm::copy(NewMemRefs, MemRefsBuffer);
8370 N->MemRefs = MemRefsBuffer;
8371 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
8372}
8373
8374/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
8375/// machine opcode.
8376///
8377SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8378 EVT VT) {
8379 SDVTList VTs = getVTList(VT);
8380 return SelectNodeTo(N, MachineOpc, VTs, None);
8381}
8382
8383SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8384 EVT VT, SDValue Op1) {
8385 SDVTList VTs = getVTList(VT);
8386 SDValue Ops[] = { Op1 };
8387 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8388}
8389
8390SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8391 EVT VT, SDValue Op1,
8392 SDValue Op2) {
8393 SDVTList VTs = getVTList(VT);
8394 SDValue Ops[] = { Op1, Op2 };
8395 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8396}
8397
8398SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8399 EVT VT, SDValue Op1,
8400 SDValue Op2, SDValue Op3) {
8401 SDVTList VTs = getVTList(VT);
8402 SDValue Ops[] = { Op1, Op2, Op3 };
8403 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8404}
8405
8406SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8407 EVT VT, ArrayRef<SDValue> Ops) {
8408 SDVTList VTs = getVTList(VT);
8409 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8410}
8411
8412SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8413 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
8414 SDVTList VTs = getVTList(VT1, VT2);
8415 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8416}
8417
8418SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8419 EVT VT1, EVT VT2) {
8420 SDVTList VTs = getVTList(VT1, VT2);
8421 return SelectNodeTo(N, MachineOpc, VTs, None);
8422}
8423
8424SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8425 EVT VT1, EVT VT2, EVT VT3,
8426 ArrayRef<SDValue> Ops) {
8427 SDVTList VTs = getVTList(VT1, VT2, VT3);
8428 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8429}
8430
8431SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8432 EVT VT1, EVT VT2,
8433 SDValue Op1, SDValue Op2) {
8434 SDVTList VTs = getVTList(VT1, VT2);
8435 SDValue Ops[] = { Op1, Op2 };
8436 return SelectNodeTo(N, MachineOpc, VTs, Ops);
8437}
8438
8439SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
8440 SDVTList VTs,ArrayRef<SDValue> Ops) {
8441 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
8442 // Reset the NodeID to -1.
8443 New->setNodeId(-1);
8444 if (New != N) {
8445 ReplaceAllUsesWith(N, New);
8446 RemoveDeadNode(N);
8447 }
8448 return New;
8449}
8450
8451/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
8452/// the line number information on the merged node since it is not possible to
8453/// preserve the information that operation is associated with multiple lines.
8454/// This will make the debugger working better at -O0, were there is a higher
8455/// probability having other instructions associated with that line.
8456///
8457/// For IROrder, we keep the smaller of the two
8458SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
8459 DebugLoc NLoc = N->getDebugLoc();
8460 if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
8461 N->setDebugLoc(DebugLoc());
8462 }
8463 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
8464 N->setIROrder(Order);
8465 return N;
8466}
8467
8468/// MorphNodeTo - This *mutates* the specified node to have the specified
8469/// return type, opcode, and operands.
8470///
8471/// Note that MorphNodeTo returns the resultant node. If there is already a
8472/// node of the specified opcode and operands, it returns that node instead of
8473/// the current one. Note that the SDLoc need not be the same.
8474///
8475/// Using MorphNodeTo is faster than creating a new node and swapping it in
8476/// with ReplaceAllUsesWith both because it often avoids allocating a new
8477/// node, and because it doesn't require CSE recalculation for any of
8478/// the node's users.
8479///
8480/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
8481/// As a consequence it isn't appropriate to use from within the DAG combiner or
8482/// the legalizer which maintain worklists that would need to be updated when
8483/// deleting things.
8484SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
8485 SDVTList VTs, ArrayRef<SDValue> Ops) {
8486 // If an identical node already exists, use it.
8487 void *IP = nullptr;
8488 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
8489 FoldingSetNodeID ID;
8490 AddNodeIDNode(ID, Opc, VTs, Ops);
8491 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
8492 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
8493 }
8494
8495 if (!RemoveNodeFromCSEMaps(N))
8496 IP = nullptr;
8497
8498 // Start the morphing.
8499 N->NodeType = Opc;
8500 N->ValueList = VTs.VTs;
8501 N->NumValues = VTs.NumVTs;
8502
8503 // Clear the operands list, updating used nodes to remove this from their
8504 // use list. Keep track of any operands that become dead as a result.
8505 SmallPtrSet<SDNode*, 16> DeadNodeSet;
8506 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
8507 SDUse &Use = *I++;
8508 SDNode *Used = Use.getNode();
8509 Use.set(SDValue());
8510 if (Used->use_empty())
8511 DeadNodeSet.insert(Used);
8512 }
8513
8514 // For MachineNode, initialize the memory references information.
8515 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
8516 MN->clearMemRefs();
8517
8518 // Swap for an appropriately sized array from the recycler.
8519 removeOperands(N);
8520 createOperands(N, Ops);
8521
8522 // Delete any nodes that are still dead after adding the uses for the
8523 // new operands.
8524 if (!DeadNodeSet.empty()) {
8525 SmallVector<SDNode *, 16> DeadNodes;
8526 for (SDNode *N : DeadNodeSet)
8527 if (N->use_empty())
8528 DeadNodes.push_back(N);
8529 RemoveDeadNodes(DeadNodes);
8530 }
8531
8532 if (IP)
8533 CSEMap.InsertNode(N, IP); // Memoize the new node.
8534 return N;
8535}
8536
8537SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
8538 unsigned OrigOpc = Node->getOpcode();
8539 unsigned NewOpc;
8540 switch (OrigOpc) {
8541 default:
8542 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!")__builtin_unreachable();
8543#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8544 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
8545#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8546 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
8547#include "llvm/IR/ConstrainedOps.def"
8548 }
8549
8550 assert(Node->getNumValues() == 2 && "Unexpected number of results!")((void)0);
8551
8552 // We're taking this node out of the chain, so we need to re-link things.
8553 SDValue InputChain = Node->getOperand(0);
8554 SDValue OutputChain = SDValue(Node, 1);
8555 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
8556
8557 SmallVector<SDValue, 3> Ops;
8558 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
8559 Ops.push_back(Node->getOperand(i));
8560
8561 SDVTList VTs = getVTList(Node->getValueType(0));
8562 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
8563
8564 // MorphNodeTo can operate in two ways: if an existing node with the
8565 // specified operands exists, it can just return it. Otherwise, it
8566 // updates the node in place to have the requested operands.
8567 if (Res == Node) {
8568 // If we updated the node in place, reset the node ID. To the isel,
8569 // this should be just like a newly allocated machine node.
8570 Res->setNodeId(-1);
8571 } else {
8572 ReplaceAllUsesWith(Node, Res);
8573 RemoveDeadNode(Node);
8574 }
8575
8576 return Res;
8577}
8578
8579/// getMachineNode - These are used for target selectors to create a new node
8580/// with specified return type(s), MachineInstr opcode, and operands.
8581///
8582/// Note that getMachineNode returns the resultant node. If there is already a
8583/// node of the specified opcode and operands, it returns that node instead of
8584/// the current one.
8585MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8586 EVT VT) {
8587 SDVTList VTs = getVTList(VT);
8588 return getMachineNode(Opcode, dl, VTs, None);
8589}
8590
8591MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8592 EVT VT, SDValue Op1) {
8593 SDVTList VTs = getVTList(VT);
8594 SDValue Ops[] = { Op1 };
8595 return getMachineNode(Opcode, dl, VTs, Ops);
8596}
8597
8598MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8599 EVT VT, SDValue Op1, SDValue Op2) {
8600 SDVTList VTs = getVTList(VT);
8601 SDValue Ops[] = { Op1, Op2 };
8602 return getMachineNode(Opcode, dl, VTs, Ops);
8603}
8604
8605MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8606 EVT VT, SDValue Op1, SDValue Op2,
8607 SDValue Op3) {
8608 SDVTList VTs = getVTList(VT);
8609 SDValue Ops[] = { Op1, Op2, Op3 };
8610 return getMachineNode(Opcode, dl, VTs, Ops);
8611}
8612
8613MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8614 EVT VT, ArrayRef<SDValue> Ops) {
8615 SDVTList VTs = getVTList(VT);
8616 return getMachineNode(Opcode, dl, VTs, Ops);
8617}
8618
8619MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8620 EVT VT1, EVT VT2, SDValue Op1,
8621 SDValue Op2) {
8622 SDVTList VTs = getVTList(VT1, VT2);
8623 SDValue Ops[] = { Op1, Op2 };
8624 return getMachineNode(Opcode, dl, VTs, Ops);
8625}
8626
8627MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8628 EVT VT1, EVT VT2, SDValue Op1,
8629 SDValue Op2, SDValue Op3) {
8630 SDVTList VTs = getVTList(VT1, VT2);
8631 SDValue Ops[] = { Op1, Op2, Op3 };
8632 return getMachineNode(Opcode, dl, VTs, Ops);
8633}
8634
8635MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8636 EVT VT1, EVT VT2,
8637 ArrayRef<SDValue> Ops) {
8638 SDVTList VTs = getVTList(VT1, VT2);
8639 return getMachineNode(Opcode, dl, VTs, Ops);
8640}
8641
8642MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8643 EVT VT1, EVT VT2, EVT VT3,
8644 SDValue Op1, SDValue Op2) {
8645 SDVTList VTs = getVTList(VT1, VT2, VT3);
8646 SDValue Ops[] = { Op1, Op2 };
8647 return getMachineNode(Opcode, dl, VTs, Ops);
8648}
8649
8650MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8651 EVT VT1, EVT VT2, EVT VT3,
8652 SDValue Op1, SDValue Op2,
8653 SDValue Op3) {
8654 SDVTList VTs = getVTList(VT1, VT2, VT3);
8655 SDValue Ops[] = { Op1, Op2, Op3 };
8656 return getMachineNode(Opcode, dl, VTs, Ops);
8657}
8658
8659MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8660 EVT VT1, EVT VT2, EVT VT3,
8661 ArrayRef<SDValue> Ops) {
8662 SDVTList VTs = getVTList(VT1, VT2, VT3);
8663 return getMachineNode(Opcode, dl, VTs, Ops);
8664}
8665
8666MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8667 ArrayRef<EVT> ResultTys,
8668 ArrayRef<SDValue> Ops) {
8669 SDVTList VTs = getVTList(ResultTys);
8670 return getMachineNode(Opcode, dl, VTs, Ops);
8671}
8672
8673MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
8674 SDVTList VTs,
8675 ArrayRef<SDValue> Ops) {
8676 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
8677 MachineSDNode *N;
8678 void *IP = nullptr;
8679
8680 if (DoCSE) {
8681 FoldingSetNodeID ID;
8682 AddNodeIDNode(ID, ~Opcode, VTs, Ops);
8683 IP = nullptr;
8684 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
8685 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
8686 }
8687 }
8688
8689 // Allocate a new MachineSDNode.
8690 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
8691 createOperands(N, Ops);
8692
8693 if (DoCSE)
8694 CSEMap.InsertNode(N, IP);
8695
8696 InsertNode(N);
8697 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
8698 return N;
8699}
8700
8701/// getTargetExtractSubreg - A convenience function for creating
8702/// TargetOpcode::EXTRACT_SUBREG nodes.
8703SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8704 SDValue Operand) {
8705 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8706 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
8707 VT, Operand, SRIdxVal);
8708 return SDValue(Subreg, 0);
8709}
8710
8711/// getTargetInsertSubreg - A convenience function for creating
8712/// TargetOpcode::INSERT_SUBREG nodes.
8713SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8714 SDValue Operand, SDValue Subreg) {
8715 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8716 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
8717 VT, Operand, Subreg, SRIdxVal);
8718 return SDValue(Result, 0);
8719}
8720
8721/// getNodeIfExists - Get the specified node if it's already available, or
8722/// else return NULL.
8723SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
8724 ArrayRef<SDValue> Ops) {
8725 SDNodeFlags Flags;
8726 if (Inserter)
8727 Flags = Inserter->getFlags();
8728 return getNodeIfExists(Opcode, VTList, Ops, Flags);
8729}
8730
8731SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
8732 ArrayRef<SDValue> Ops,
8733 const SDNodeFlags Flags) {
8734 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
8735 FoldingSetNodeID ID;
8736 AddNodeIDNode(ID, Opcode, VTList, Ops);
8737 void *IP = nullptr;
8738 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
8739 E->intersectFlagsWith(Flags);
8740 return E;
8741 }
8742 }
8743 return nullptr;
8744}
8745
8746/// doesNodeExist - Check if a node exists without modifying its flags.
8747bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
8748 ArrayRef<SDValue> Ops) {
8749 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
8750 FoldingSetNodeID ID;
8751 AddNodeIDNode(ID, Opcode, VTList, Ops);
8752 void *IP = nullptr;
8753 if (FindNodeOrInsertPos(ID, SDLoc(), IP))
8754 return true;
8755 }
8756 return false;
8757}
8758
8759/// getDbgValue - Creates a SDDbgValue node.
8760///
8761/// SDNode
8762SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
8763 SDNode *N, unsigned R, bool IsIndirect,
8764 const DebugLoc &DL, unsigned O) {
8765 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&((void)0)
8766 "Expected inlined-at fields to agree")((void)0);
8767 return new (DbgInfo->getAlloc())
8768 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
8769 {}, IsIndirect, DL, O,
8770 /*IsVariadic=*/false);
8771}
8772
8773/// Constant
8774SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
8775 DIExpression *Expr,
8776 const Value *C,
8777 const DebugLoc &DL, unsigned O) {
8778 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&((void)0)
8779 "Expected inlined-at fields to agree")((void)0);
8780 return new (DbgInfo->getAlloc())
8781 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
8782 /*IsIndirect=*/false, DL, O,
8783 /*IsVariadic=*/false);
8784}
8785
8786/// FrameIndex
8787SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
8788 DIExpression *Expr, unsigned FI,
8789 bool IsIndirect,
8790 const DebugLoc &DL,
8791 unsigned O) {
8792 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&((void)0)
8793 "Expected inlined-at fields to agree")((void)0);
8794 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
8795}
8796
8797/// FrameIndex with dependencies
8798SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
8799 DIExpression *Expr, unsigned FI,
8800 ArrayRef<SDNode *> Dependencies,
8801 bool IsIndirect,
8802 const DebugLoc &DL,
8803 unsigned O) {
8804 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&((void)0)
8805 "Expected inlined-at fields to agree")((void)0);
8806 return new (DbgInfo->getAlloc())
8807 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
8808 Dependencies, IsIndirect, DL, O,
8809 /*IsVariadic=*/false);
8810}
8811
8812/// VReg
8813SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
8814 unsigned VReg, bool IsIndirect,
8815 const DebugLoc &DL, unsigned O) {
8816 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&((void)0)
8817 "Expected inlined-at fields to agree")((void)0);
8818 return new (DbgInfo->getAlloc())
8819 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
8820 {}, IsIndirect, DL, O,
8821 /*IsVariadic=*/false);
8822}
8823
8824SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
8825 ArrayRef<SDDbgOperand> Locs,
8826 ArrayRef<SDNode *> Dependencies,
8827 bool IsIndirect, const DebugLoc &DL,
8828 unsigned O, bool IsVariadic) {
8829 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&((void)0)
8830 "Expected inlined-at fields to agree")((void)0);
8831 return new (DbgInfo->getAlloc())
8832 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
8833 DL, O, IsVariadic);
8834}
8835
8836void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
8837 unsigned OffsetInBits, unsigned SizeInBits,
8838 bool InvalidateDbg) {
8839 SDNode *FromNode = From.getNode();
8840 SDNode *ToNode = To.getNode();
8841 assert(FromNode && ToNode && "Can't modify dbg values")((void)0);
8842
8843 // PR35338
8844 // TODO: assert(From != To && "Redundant dbg value transfer");
8845 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
8846 if (From == To || FromNode == ToNode)
8847 return;
8848
8849 if (!FromNode->getHasDebugValue())
8850 return;
8851
8852 SDDbgOperand FromLocOp =
8853 SDDbgOperand::fromNode(From.getNode(), From.getResNo());
8854 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo());
8855
8856 SmallVector<SDDbgValue *, 2> ClonedDVs;
8857 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
8858 if (Dbg->isInvalidated())
8859 continue;
8860
8861 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
8862
8863 // Create a new location ops vector that is equal to the old vector, but
8864 // with each instance of FromLocOp replaced with ToLocOp.
8865 bool Changed = false;
8866 auto NewLocOps = Dbg->copyLocationOps();
8867 std::replace_if(
8868 NewLocOps.begin(), NewLocOps.end(),
8869 [&Changed, FromLocOp](const SDDbgOperand &Op) {
8870 bool Match = Op == FromLocOp;
8871 Changed |= Match;
8872 return Match;
8873 },
8874 ToLocOp);
8875 // Ignore this SDDbgValue if we didn't find a matching location.
8876 if (!Changed)
8877 continue;
8878
8879 DIVariable *Var = Dbg->getVariable();
8880 auto *Expr = Dbg->getExpression();
8881 // If a fragment is requested, update the expression.
8882 if (SizeInBits) {
8883 // When splitting a larger (e.g., sign-extended) value whose
8884 // lower bits are described with an SDDbgValue, do not attempt
8885 // to transfer the SDDbgValue to the upper bits.
8886 if (auto FI = Expr->getFragmentInfo())
8887 if (OffsetInBits + SizeInBits > FI->SizeInBits)
8888 continue;
8889 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
8890 SizeInBits);
8891 if (!Fragment)
8892 continue;
8893 Expr = *Fragment;
8894 }
8895
8896 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
8897 // Clone the SDDbgValue and move it to To.
8898 SDDbgValue *Clone = getDbgValueList(
8899 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
8900 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
8901 Dbg->isVariadic());
8902 ClonedDVs.push_back(Clone);
8903
8904 if (InvalidateDbg) {
8905 // Invalidate value and indicate the SDDbgValue should not be emitted.
8906 Dbg->setIsInvalidated();
8907 Dbg->setIsEmitted();
8908 }
8909 }
8910
8911 for (SDDbgValue *Dbg : ClonedDVs) {
8912 assert(is_contained(Dbg->getSDNodes(), ToNode) &&((void)0)
8913 "Transferred DbgValues should depend on the new SDNode")((void)0);
8914 AddDbgValue(Dbg, false);
8915 }
8916}
8917
8918void SelectionDAG::salvageDebugInfo(SDNode &N) {
8919 if (!N.getHasDebugValue())
8920 return;
8921
8922 SmallVector<SDDbgValue *, 2> ClonedDVs;
8923 for (auto DV : GetDbgValues(&N)) {
8924 if (DV->isInvalidated())
8925 continue;
8926 switch (N.getOpcode()) {
8927 default:
8928 break;
8929 case ISD::ADD:
8930 SDValue N0 = N.getOperand(0);
8931 SDValue N1 = N.getOperand(1);
8932 if (!isConstantIntBuildVectorOrConstantInt(N0) &&
8933 isConstantIntBuildVectorOrConstantInt(N1)) {
8934 uint64_t Offset = N.getConstantOperandVal(1);
8935
8936 // Rewrite an ADD constant node into a DIExpression. Since we are
8937 // performing arithmetic to compute the variable's *value* in the
8938 // DIExpression, we need to mark the expression with a
8939 // DW_OP_stack_value.
8940 auto *DIExpr = DV->getExpression();
8941 auto NewLocOps = DV->copyLocationOps();
8942 bool Changed = false;
8943 for (size_t i = 0; i < NewLocOps.size(); ++i) {
8944 // We're not given a ResNo to compare against because the whole
8945 // node is going away. We know that any ISD::ADD only has one
8946 // result, so we can assume any node match is using the result.
8947 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
8948 NewLocOps[i].getSDNode() != &N)
8949 continue;
8950 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
8951 SmallVector<uint64_t, 3> ExprOps;
8952 DIExpression::appendOffset(ExprOps, Offset);
8953 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
8954 Changed = true;
8955 }
8956 (void)Changed;
8957 assert(Changed && "Salvage target doesn't use N")((void)0);
8958
8959 auto AdditionalDependencies = DV->getAdditionalDependencies();
8960 SDDbgValue *Clone = getDbgValueList(DV->getVariable(), DIExpr,
8961 NewLocOps, AdditionalDependencies,
8962 DV->isIndirect(), DV->getDebugLoc(),
8963 DV->getOrder(), DV->isVariadic());
8964 ClonedDVs.push_back(Clone);
8965 DV->setIsInvalidated();
8966 DV->setIsEmitted();
8967 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";do { } while (false)
8968 N0.getNode()->dumprFull(this);do { } while (false)
8969 dbgs() << " into " << *DIExpr << '\n')do { } while (false);
8970 }
8971 }
8972 }
8973
8974 for (SDDbgValue *Dbg : ClonedDVs) {
8975 assert(!Dbg->getSDNodes().empty() &&((void)0)
8976 "Salvaged DbgValue should depend on a new SDNode")((void)0);
8977 AddDbgValue(Dbg, false);
8978 }
8979}
8980
8981/// Creates a SDDbgLabel node.
8982SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
8983 const DebugLoc &DL, unsigned O) {
8984 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&((void)0)
8985 "Expected inlined-at fields to agree")((void)0);
8986 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
8987}
8988
8989namespace {
8990
8991/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
8992/// pointed to by a use iterator is deleted, increment the use iterator
8993/// so that it doesn't dangle.
8994///
8995class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
8996 SDNode::use_iterator &UI;
8997 SDNode::use_iterator &UE;
8998
8999 void NodeDeleted(SDNode *N, SDNode *E) override {
9000 // Increment the iterator as needed.
9001 while (UI != UE && N == *UI)
9002 ++UI;
9003 }
9004
9005public:
9006 RAUWUpdateListener(SelectionDAG &d,
9007 SDNode::use_iterator &ui,
9008 SDNode::use_iterator &ue)
9009 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
9010};
9011
9012} // end anonymous namespace
9013
9014/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
9015/// This can cause recursive merging of nodes in the DAG.
9016///
9017/// This version assumes From has a single result value.
9018///
9019void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
9020 SDNode *From = FromN.getNode();
9021 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&((void)0)
9022 "Cannot replace with this method!")((void)0);
9023 assert(From != To.getNode() && "Cannot replace uses of with self")((void)0);
9024
9025 // Preserve Debug Values
9026 transferDbgValues(FromN, To);
9027
9028 // Iterate over all the existing uses of From. New uses will be added
9029 // to the beginning of the use list, which we avoid visiting.
9030 // This specifically avoids visiting uses of From that arise while the
9031 // replacement is happening, because any such uses would be the result
9032 // of CSE: If an existing node looks like From after one of its operands
9033 // is replaced by To, we don't want to replace of all its users with To
9034 // too. See PR3018 for more info.
9035 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
9036 RAUWUpdateListener Listener(*this, UI, UE);
9037 while (UI != UE) {
9038 SDNode *User = *UI;
9039
9040 // This node is about to morph, remove its old self from the CSE maps.
9041 RemoveNodeFromCSEMaps(User);
9042
9043 // A user can appear in a use list multiple times, and when this
9044 // happens the uses are usually next to each other in the list.
9045 // To help reduce the number of CSE recomputations, process all
9046 // the uses of this user that we can find this way.
9047 do {
9048 SDUse &Use = UI.getUse();
9049 ++UI;
9050 Use.set(To);
9051 if (To->isDivergent() != From->isDivergent())
9052 updateDivergence(User);
9053 } while (UI != UE && *UI == User);
9054 // Now that we have modified User, add it back to the CSE maps. If it
9055 // already exists there, recursively merge the results together.
9056 AddModifiedNodeToCSEMaps(User);
9057 }
9058
9059 // If we just RAUW'd the root, take note.
9060 if (FromN == getRoot())
9061 setRoot(To);
9062}
9063
9064/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
9065/// This can cause recursive merging of nodes in the DAG.
9066///
9067/// This version assumes that for each value of From, there is a
9068/// corresponding value in To in the same position with the same type.
9069///
9070void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
9071#ifndef NDEBUG1
9072 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
9073 assert((!From->hasAnyUseOfValue(i) ||((void)0)
9074 From->getValueType(i) == To->getValueType(i)) &&((void)0)
9075 "Cannot use this version of ReplaceAllUsesWith!")((void)0);
9076#endif
9077
9078 // Handle the trivial case.
9079 if (From == To)
9080 return;
9081
9082 // Preserve Debug Info. Only do this if there's a use.
9083 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
9084 if (From->hasAnyUseOfValue(i)) {
9085 assert((i < To->getNumValues()) && "Invalid To location")((void)0);
9086 transferDbgValues(SDValue(From, i), SDValue(To, i));
9087 }
9088
9089 // Iterate over just the existing users of From. See the comments in
9090 // the ReplaceAllUsesWith above.
9091 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
9092 RAUWUpdateListener Listener(*this, UI, UE);
9093 while (UI != UE) {
9094 SDNode *User = *UI;
9095
9096 // This node is about to morph, remove its old self from the CSE maps.
9097 RemoveNodeFromCSEMaps(User);
9098
9099 // A user can appear in a use list multiple times, and when this
9100 // happens the uses are usually next to each other in the list.
9101 // To help reduce the number of CSE recomputations, process all
9102 // the uses of this user that we can find this way.
9103 do {
9104 SDUse &Use = UI.getUse();
9105 ++UI;
9106 Use.setNode(To);
9107 if (To->isDivergent() != From->isDivergent())
9108 updateDivergence(User);
9109 } while (UI != UE && *UI == User);
9110
9111 // Now that we have modified User, add it back to the CSE maps. If it
9112 // already exists there, recursively merge the results together.
9113 AddModifiedNodeToCSEMaps(User);
9114 }
9115
9116 // If we just RAUW'd the root, take note.
9117 if (From == getRoot().getNode())
9118 setRoot(SDValue(To, getRoot().getResNo()));
9119}
9120
9121/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
9122/// This can cause recursive merging of nodes in the DAG.
9123///
9124/// This version can replace From with any result values. To must match the
9125/// number and types of values returned by From.
9126void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
9127 if (From->getNumValues() == 1) // Handle the simple case efficiently.
9128 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
9129
9130 // Preserve Debug Info.
9131 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
9132 transferDbgValues(SDValue(From, i), To[i]);
9133
9134 // Iterate over just the existing users of From. See the comments in
9135 // the ReplaceAllUsesWith above.
9136 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
9137 RAUWUpdateListener Listener(*this, UI, UE);
9138 while (UI != UE) {
9139 SDNode *User = *UI;
9140
9141 // This node is about to morph, remove its old self from the CSE maps.
9142 RemoveNodeFromCSEMaps(User);
9143
9144 // A user can appear in a use list multiple times, and when this happens the
9145 // uses are usually next to each other in the list. To help reduce the
9146 // number of CSE and divergence recomputations, process all the uses of this
9147 // user that we can find this way.
9148 bool To_IsDivergent = false;
9149 do {
9150 SDUse &Use = UI.getUse();
9151 const SDValue &ToOp = To[Use.getResNo()];
9152 ++UI;
9153 Use.set(ToOp);
9154 To_IsDivergent |= ToOp->isDivergent();
9155 } while (UI != UE && *UI == User);
9156
9157 if (To_IsDivergent != From->isDivergent())
9158 updateDivergence(User);
9159
9160 // Now that we have modified User, add it back to the CSE maps. If it
9161 // already exists there, recursively merge the results together.
9162 AddModifiedNodeToCSEMaps(User);
9163 }
9164
9165 // If we just RAUW'd the root, take note.
9166 if (From == getRoot().getNode())
9167 setRoot(SDValue(To[getRoot().getResNo()]));
9168}
9169
9170/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
9171/// uses of other values produced by From.getNode() alone. The Deleted
9172/// vector is handled the same way as for ReplaceAllUsesWith.
9173void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
9174 // Handle the really simple, really trivial case efficiently.
9175 if (From == To) return;
9176
9177 // Handle the simple, trivial, case efficiently.
9178 if (From.getNode()->getNumValues() == 1) {
9179 ReplaceAllUsesWith(From, To);
9180 return;
9181 }
9182
9183 // Preserve Debug Info.
9184 transferDbgValues(From, To);
9185
9186 // Iterate over just the existing users of From. See the comments in
9187 // the ReplaceAllUsesWith above.
9188 SDNode::use_iterator UI = From.getNode()->use_begin(),
9189 UE = From.getNode()->use_end();
9190 RAUWUpdateListener Listener(*this, UI, UE);
9191 while (UI != UE) {
9192 SDNode *User = *UI;
9193 bool UserRemovedFromCSEMaps = false;
9194
9195 // A user can appear in a use list multiple times, and when this
9196 // happens the uses are usually next to each other in the list.
9197 // To help reduce the number of CSE recomputations, process all
9198 // the uses of this user that we can find this way.
9199 do {
9200 SDUse &Use = UI.getUse();
9201
9202 // Skip uses of different values from the same node.
9203 if (Use.getResNo() != From.getResNo()) {
9204 ++UI;
9205 continue;
9206 }
9207
9208 // If this node hasn't been modified yet, it's still in the CSE maps,
9209 // so remove its old self from the CSE maps.
9210 if (!UserRemovedFromCSEMaps) {
9211 RemoveNodeFromCSEMaps(User);
9212 UserRemovedFromCSEMaps = true;
9213 }
9214
9215 ++UI;
9216 Use.set(To);
9217 if (To->isDivergent() != From->isDivergent())
9218 updateDivergence(User);
9219 } while (UI != UE && *UI == User);
9220 // We are iterating over all uses of the From node, so if a use
9221 // doesn't use the specific value, no changes are made.
9222 if (!UserRemovedFromCSEMaps)
9223 continue;
9224
9225 // Now that we have modified User, add it back to the CSE maps. If it
9226 // already exists there, recursively merge the results together.
9227 AddModifiedNodeToCSEMaps(User);
9228 }
9229
9230 // If we just RAUW'd the root, take note.
9231 if (From == getRoot())
9232 setRoot(To);
9233}
9234
9235namespace {
9236
9237 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
9238 /// to record information about a use.
9239 struct UseMemo {
9240 SDNode *User;
9241 unsigned Index;
9242 SDUse *Use;
9243 };
9244
9245 /// operator< - Sort Memos by User.
9246 bool operator<(const UseMemo &L, const UseMemo &R) {
9247 return (intptr_t)L.User < (intptr_t)R.User;
9248 }
9249
9250} // end anonymous namespace
9251
9252bool SelectionDAG::calculateDivergence(SDNode *N) {
9253 if (TLI->isSDNodeAlwaysUniform(N)) {
9254 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) &&((void)0)
9255 "Conflicting divergence information!")((void)0);
9256 return false;
9257 }
9258 if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA))
9259 return true;
9260 for (auto &Op : N->ops()) {
9261 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
9262 return true;
9263 }
9264 return false;
9265}
9266
9267void SelectionDAG::updateDivergence(SDNode *N) {
9268 SmallVector<SDNode *, 16> Worklist(1, N);
9269 do {
9270 N = Worklist.pop_back_val();
9271 bool IsDivergent = calculateDivergence(N);
9272 if (N->SDNodeBits.IsDivergent != IsDivergent) {
9273 N->SDNodeBits.IsDivergent = IsDivergent;
9274 llvm::append_range(Worklist, N->uses());
9275 }
9276 } while (!Worklist.empty());
9277}
9278
9279void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
9280 DenseMap<SDNode *, unsigned> Degree;
9281 Order.reserve(AllNodes.size());
9282 for (auto &N : allnodes()) {
9283 unsigned NOps = N.getNumOperands();
9284 Degree[&N] = NOps;
9285 if (0 == NOps)
9286 Order.push_back(&N);
9287 }
9288 for (size_t I = 0; I != Order.size(); ++I) {
9289 SDNode *N = Order[I];
9290 for (auto U : N->uses()) {
9291 unsigned &UnsortedOps = Degree[U];
9292 if (0 == --UnsortedOps)
9293 Order.push_back(U);
9294 }
9295 }
9296}
9297
9298#ifndef NDEBUG1
9299void SelectionDAG::VerifyDAGDiverence() {
9300 std::vector<SDNode *> TopoOrder;
9301 CreateTopologicalOrder(TopoOrder);
9302 for (auto *N : TopoOrder) {
9303 assert(calculateDivergence(N) == N->isDivergent() &&((void)0)
9304 "Divergence bit inconsistency detected")((void)0);
9305 }
9306}
9307#endif
9308
9309/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
9310/// uses of other values produced by From.getNode() alone. The same value
9311/// may appear in both the From and To list. The Deleted vector is
9312/// handled the same way as for ReplaceAllUsesWith.
9313void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
9314 const SDValue *To,
9315 unsigned Num){
9316 // Handle the simple, trivial case efficiently.
9317 if (Num == 1)
9318 return ReplaceAllUsesOfValueWith(*From, *To);
9319
9320 transferDbgValues(*From, *To);
9321
9322 // Read up all the uses and make records of them. This helps
9323 // processing new uses that are introduced during the
9324 // replacement process.
9325 SmallVector<UseMemo, 4> Uses;
9326 for (unsigned i = 0; i != Num; ++i) {
9327 unsigned FromResNo = From[i].getResNo();
9328 SDNode *FromNode = From[i].getNode();
9329 for (SDNode::use_iterator UI = FromNode->use_begin(),
9330 E = FromNode->use_end(); UI != E; ++UI) {
9331 SDUse &Use = UI.getUse();
9332 if (Use.getResNo() == FromResNo) {
9333 UseMemo Memo = { *UI, i, &Use };
9334 Uses.push_back(Memo);
9335 }
9336 }
9337 }
9338
9339 // Sort the uses, so that all the uses from a given User are together.
9340 llvm::sort(Uses);
9341
9342 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
9343 UseIndex != UseIndexEnd; ) {
9344 // We know that this user uses some value of From. If it is the right
9345 // value, update it.
9346 SDNode *User = Uses[UseIndex].User;
9347
9348 // This node is about to morph, remove its old self from the CSE maps.
9349 RemoveNodeFromCSEMaps(User);
9350
9351 // The Uses array is sorted, so all the uses for a given User
9352 // are next to each other in the list.
9353 // To help reduce the number of CSE recomputations, process all
9354 // the uses of this user that we can find this way.
9355 do {
9356 unsigned i = Uses[UseIndex].Index;
9357 SDUse &Use = *Uses[UseIndex].Use;
9358 ++UseIndex;
9359
9360 Use.set(To[i]);
9361 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
9362
9363 // Now that we have modified User, add it back to the CSE maps. If it
9364 // already exists there, recursively merge the results together.
9365 AddModifiedNodeToCSEMaps(User);
9366 }
9367}
9368
9369/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
9370/// based on their topological order. It returns the maximum id and a vector
9371/// of the SDNodes* in assigned order by reference.
9372unsigned SelectionDAG::AssignTopologicalOrder() {
9373 unsigned DAGSize = 0;
9374
9375 // SortedPos tracks the progress of the algorithm. Nodes before it are
9376 // sorted, nodes after it are unsorted. When the algorithm completes
9377 // it is at the end of the list.
9378 allnodes_iterator SortedPos = allnodes_begin();
9379
9380 // Visit all the nodes. Move nodes with no operands to the front of
9381 // the list immediately. Annotate nodes that do have operands with their
9382 // operand count. Before we do this, the Node Id fields of the nodes
9383 // may contain arbitrary values. After, the Node Id fields for nodes
9384 // before SortedPos will contain the topological sort index, and the
9385 // Node Id fields for nodes At SortedPos and after will contain the
9386 // count of outstanding operands.
9387 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
9388 SDNode *N = &*I++;
9389 checkForCycles(N, this);
9390 unsigned Degree = N->getNumOperands();
9391 if (Degree == 0) {
9392 // A node with no uses, add it to the result array immediately.
9393 N->setNodeId(DAGSize++);
9394 allnodes_iterator Q(N);
9395 if (Q != SortedPos)
9396 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
9397 assert(SortedPos != AllNodes.end() && "Overran node list")((void)0);
9398 ++SortedPos;
9399 } else {
9400 // Temporarily use the Node Id as scratch space for the degree count.
9401 N->setNodeId(Degree);
9402 }
9403 }
9404
9405 // Visit all the nodes. As we iterate, move nodes into sorted order,
9406 // such that by the time the end is reached all nodes will be sorted.
9407 for (SDNode &Node : allnodes()) {
9408 SDNode *N = &Node;
9409 checkForCycles(N, this);
9410 // N is in sorted position, so all its uses have one less operand
9411 // that needs to be sorted.
9412 for (SDNode *P : N->uses()) {
9413 unsigned Degree = P->getNodeId();
9414 assert(Degree != 0 && "Invalid node degree")((void)0);
9415 --Degree;
9416 if (Degree == 0) {
9417 // All of P's operands are sorted, so P may sorted now.
9418 P->setNodeId(DAGSize++);
9419 if (P->getIterator() != SortedPos)
9420 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
9421 assert(SortedPos != AllNodes.end() && "Overran node list")((void)0);
9422 ++SortedPos;
9423 } else {
9424 // Update P's outstanding operand count.
9425 P->setNodeId(Degree);
9426 }
9427 }
9428 if (Node.getIterator() == SortedPos) {
9429#ifndef NDEBUG1
9430 allnodes_iterator I(N);
9431 SDNode *S = &*++I;
9432 dbgs() << "Overran sorted position:\n";
9433 S->dumprFull(this); dbgs() << "\n";
9434 dbgs() << "Checking if this is due to cycles\n";
9435 checkForCycles(this, true);
9436#endif
9437 llvm_unreachable(nullptr)__builtin_unreachable();
9438 }
9439 }
9440
9441 assert(SortedPos == AllNodes.end() &&((void)0)
9442 "Topological sort incomplete!")((void)0);
9443 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&((void)0)
9444 "First node in topological sort is not the entry token!")((void)0);
9445 assert(AllNodes.front().getNodeId() == 0 &&((void)0)
9446 "First node in topological sort has non-zero id!")((void)0);
9447 assert(AllNodes.front().getNumOperands() == 0 &&((void)0)
9448 "First node in topological sort has operands!")((void)0);
9449 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&((void)0)
9450 "Last node in topologic sort has unexpected id!")((void)0);
9451 assert(AllNodes.back().use_empty() &&((void)0)
9452 "Last node in topologic sort has users!")((void)0);
9453 assert(DAGSize == allnodes_size() && "Node count mismatch!")((void)0);
9454 return DAGSize;
9455}
9456
9457/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
9458/// value is produced by SD.
9459void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
9460 for (SDNode *SD : DB->getSDNodes()) {
9461 if (!SD)
9462 continue;
9463 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue())((void)0);
9464 SD->setHasDebugValue(true);
9465 }
9466 DbgInfo->add(DB, isParameter);
9467}
9468
9469void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
9470
9471SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
9472 SDValue NewMemOpChain) {
9473 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node")((void)0);
9474 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT")((void)0);
9475 // The new memory operation must have the same position as the old load in
9476 // terms of memory dependency. Create a TokenFactor for the old load and new
9477 // memory operation and update uses of the old load's output chain to use that
9478 // TokenFactor.
9479 if (OldChain == NewMemOpChain || OldChain.use_empty())
9480 return NewMemOpChain;
9481
9482 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
9483 OldChain, NewMemOpChain);
9484 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
9485 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
9486 return TokenFactor;
9487}
9488
9489SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
9490 SDValue NewMemOp) {
9491 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node")((void)0);
9492 SDValue OldChain = SDValue(OldLoad, 1);
9493 SDValue NewMemOpChain = NewMemOp.getValue(1);
9494 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
9495}
9496
9497SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
9498 Function **OutFunction) {
9499 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol")((void)0);
9500
9501 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
9502 auto *Module = MF->getFunction().getParent();
9503 auto *Function = Module->getFunction(Symbol);
9504
9505 if (OutFunction != nullptr)
9506 *OutFunction = Function;
9507
9508 if (Function != nullptr) {
9509 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
9510 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
9511 }
9512
9513 std::string ErrorStr;
9514 raw_string_ostream ErrorFormatter(ErrorStr);
9515
9516 ErrorFormatter << "Undefined external symbol ";
9517 ErrorFormatter << '"' << Symbol << '"';
9518 ErrorFormatter.flush();
9519
9520 report_fatal_error(ErrorStr);
9521}
9522
9523//===----------------------------------------------------------------------===//
9524// SDNode Class
9525//===----------------------------------------------------------------------===//
9526
9527bool llvm::isNullConstant(SDValue V) {
9528 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
9529 return Const != nullptr && Const->isNullValue();
9530}
9531
9532bool llvm::isNullFPConstant(SDValue V) {
9533 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
9534 return Const != nullptr && Const->isZero() && !Const->isNegative();
9535}
9536
9537bool llvm::isAllOnesConstant(SDValue V) {
9538 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
9539 return Const != nullptr && Const->isAllOnesValue();
9540}
9541
9542bool llvm::isOneConstant(SDValue V) {
9543 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
9544 return Const != nullptr && Const->isOne();
9545}
9546
9547SDValue llvm::peekThroughBitcasts(SDValue V) {
9548 while (V.getOpcode() == ISD::BITCAST)
9549 V = V.getOperand(0);
9550 return V;
9551}
9552
9553SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
9554 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
9555 V = V.getOperand(0);
9556 return V;
9557}
9558
9559SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
9560 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
9561 V = V.getOperand(0);
9562 return V;
9563}
9564
9565bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
9566 if (V.getOpcode() != ISD::XOR)
9567 return false;
9568 V = peekThroughBitcasts(V.getOperand(1));
9569 unsigned NumBits = V.getScalarValueSizeInBits();
9570 ConstantSDNode *C =
9571 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
9572 return C && (C->getAPIntValue().countTrailingOnes() >= NumBits);
9573}
9574
9575ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
9576 bool AllowTruncation) {
9577 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
9578 return CN;
9579
9580 // SplatVectors can truncate their operands. Ignore that case here unless
9581 // AllowTruncation is set.
9582 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
9583 EVT VecEltVT = N->getValueType(0).getVectorElementType();
9584 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
9585 EVT CVT = CN->getValueType(0);
9586 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension")((void)0);
9587 if (AllowTruncation || CVT == VecEltVT)
9588 return CN;
9589 }
9590 }
9591
9592 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9593 BitVector UndefElements;
9594 ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
9595
9596 // BuildVectors can truncate their operands. Ignore that case here unless
9597 // AllowTruncation is set.
9598 if (CN && (UndefElements.none() || AllowUndefs)) {
9599 EVT CVT = CN->getValueType(0);
9600 EVT NSVT = N.getValueType().getScalarType();
9601 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension")((void)0);
9602 if (AllowTruncation || (CVT == NSVT))
9603 return CN;
9604 }
9605 }
9606
9607 return nullptr;
9608}
9609
9610ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
9611 bool AllowUndefs,
9612 bool AllowTruncation) {
9613 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
9614 return CN;
9615
9616 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9617 BitVector UndefElements;
9618 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
9619
9620 // BuildVectors can truncate their operands. Ignore that case here unless
9621 // AllowTruncation is set.
9622 if (CN && (UndefElements.none() || AllowUndefs)) {
9623 EVT CVT = CN->getValueType(0);
9624 EVT NSVT = N.getValueType().getScalarType();
9625 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension")((void)0);
9626 if (AllowTruncation || (CVT == NSVT))
9627 return CN;
9628 }
9629 }
9630
9631 return nullptr;
9632}
9633
9634ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
9635 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
9636 return CN;
9637
9638 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9639 BitVector UndefElements;
9640 ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
9641 if (CN && (UndefElements.none() || AllowUndefs))
9642 return CN;
9643 }
9644
9645 if (N.getOpcode() == ISD::SPLAT_VECTOR)
9646 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
9647 return CN;
9648
9649 return nullptr;
9650}
9651
9652ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
9653 const APInt &DemandedElts,
9654 bool AllowUndefs) {
9655 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
9656 return CN;
9657
9658 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
9659 BitVector UndefElements;
9660 ConstantFPSDNode *CN =
9661 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
9662 if (CN && (UndefElements.none() || AllowUndefs))
9663 return CN;
9664 }
9665
9666 return nullptr;
9667}
9668
9669bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
9670 // TODO: may want to use peekThroughBitcast() here.
9671 ConstantSDNode *C =
9672 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
9673 return C && C->isNullValue();
9674}
9675
9676bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
9677 // TODO: may want to use peekThroughBitcast() here.
9678 unsigned BitWidth = N.getScalarValueSizeInBits();
9679 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
9680 return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth;
9681}
9682
9683bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
9684 N = peekThroughBitcasts(N);
9685 unsigned BitWidth = N.getScalarValueSizeInBits();
9686 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
9687 return C && C->isAllOnesValue() && C->getValueSizeInBits(0) == BitWidth;
9688}
9689
9690HandleSDNode::~HandleSDNode() {
9691 DropOperands();
9692}
9693
9694GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
9695 const DebugLoc &DL,
9696 const GlobalValue *GA, EVT VT,
9697 int64_t o, unsigned TF)
9698 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
9699 TheGlobal = GA;
9700}
9701
9702AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
9703 EVT VT, unsigned SrcAS,
9704 unsigned DestAS)
9705 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
9706 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
9707
9708MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
9709 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
9710 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
9711 MemSDNodeBits.IsVolatile = MMO->isVolatile();
9712 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
9713 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
9714 MemSDNodeBits.IsInvariant = MMO->isInvariant();
9715
9716 // We check here that the size of the memory operand fits within the size of
9717 // the MMO. This is because the MMO might indicate only a possible address
9718 // range instead of specifying the affected memory addresses precisely.
9719 // TODO: Make MachineMemOperands aware of scalable vectors.
9720 assert(memvt.getStoreSize().getKnownMinSize() <= MMO->getSize() &&((void)0)
9721 "Size mismatch!")((void)0);
9722}
9723
9724/// Profile - Gather unique data for the node.
9725///
9726void SDNode::Profile(FoldingSetNodeID &ID) const {
9727 AddNodeIDNode(ID, this);
9728}
9729
9730namespace {
9731
9732 struct EVTArray {
9733 std::vector<EVT> VTs;
9734
9735 EVTArray() {
9736 VTs.reserve(MVT::VALUETYPE_SIZE);
9737 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
9738 VTs.push_back(MVT((MVT::SimpleValueType)i));
9739 }
9740 };
9741
9742} // end anonymous namespace
9743
9744static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
9745static ManagedStatic<EVTArray> SimpleVTArray;
9746static ManagedStatic<sys::SmartMutex<true>> VTMutex;
9747
9748/// getValueTypeList - Return a pointer to the specified value type.
9749///
9750const EVT *SDNode::getValueTypeList(EVT VT) {
9751 if (VT.isExtended()) {
9752 sys::SmartScopedLock<true> Lock(*VTMutex);
9753 return &(*EVTs->insert(VT).first);
9754 }
9755 assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!")((void)0);
9756 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
9757}
9758
9759/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
9760/// indicated value. This method ignores uses of other values defined by this
9761/// operation.
9762bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
9763 assert(Value < getNumValues() && "Bad value!")((void)0);
9764
9765 // TODO: Only iterate over uses of a given value of the node
9766 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
9767 if (UI.getUse().getResNo() == Value) {
9768 if (NUses == 0)
9769 return false;
9770 --NUses;
9771 }
9772 }
9773
9774 // Found exactly the right number of uses?
9775 return NUses == 0;
9776}
9777
9778/// hasAnyUseOfValue - Return true if there are any use of the indicated
9779/// value. This method ignores uses of other values defined by this operation.
9780bool SDNode::hasAnyUseOfValue(unsigned Value) const {
9781 assert(Value < getNumValues() && "Bad value!")((void)0);
9782
9783 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
9784 if (UI.getUse().getResNo() == Value)
9785 return true;
9786
9787 return false;
9788}
9789
9790/// isOnlyUserOf - Return true if this node is the only use of N.
9791bool SDNode::isOnlyUserOf(const SDNode *N) const {
9792 bool Seen = false;
9793 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9794 SDNode *User = *I;
9795 if (User == this)
9796 Seen = true;
9797 else
9798 return false;
9799 }
9800
9801 return Seen;
9802}
9803
9804/// Return true if the only users of N are contained in Nodes.
9805bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
9806 bool Seen = false;
9807 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9808 SDNode *User = *I;
9809 if (llvm::is_contained(Nodes, User))
9810 Seen = true;
9811 else
9812 return false;
9813 }
9814
9815 return Seen;
9816}
9817
9818/// isOperand - Return true if this node is an operand of N.
9819bool SDValue::isOperandOf(const SDNode *N) const {
9820 return is_contained(N->op_values(), *this);
9821}
9822
9823bool SDNode::isOperandOf(const SDNode *N) const {
9824 return any_of(N->op_values(),
9825 [this](SDValue Op) { return this == Op.getNode(); });
9826}
9827
9828/// reachesChainWithoutSideEffects - Return true if this operand (which must
9829/// be a chain) reaches the specified operand without crossing any
9830/// side-effecting instructions on any chain path. In practice, this looks
9831/// through token factors and non-volatile loads. In order to remain efficient,
9832/// this only looks a couple of nodes in, it does not do an exhaustive search.
9833///
9834/// Note that we only need to examine chains when we're searching for
9835/// side-effects; SelectionDAG requires that all side-effects are represented
9836/// by chains, even if another operand would force a specific ordering. This
9837/// constraint is necessary to allow transformations like splitting loads.
9838bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
9839 unsigned Depth) const {
9840 if (*this == Dest) return true;
9841
9842 // Don't search too deeply, we just want to be able to see through
9843 // TokenFactor's etc.
9844 if (Depth == 0) return false;
9845
9846 // If this is a token factor, all inputs to the TF happen in parallel.
9847 if (getOpcode() == ISD::TokenFactor) {
9848 // First, try a shallow search.
9849 if (is_contained((*this)->ops(), Dest)) {
9850 // We found the chain we want as an operand of this TokenFactor.
9851 // Essentially, we reach the chain without side-effects if we could
9852 // serialize the TokenFactor into a simple chain of operations with
9853 // Dest as the last operation. This is automatically true if the
9854 // chain has one use: there are no other ordering constraints.
9855 // If the chain has more than one use, we give up: some other
9856 // use of Dest might force a side-effect between Dest and the current
9857 // node.
9858 if (Dest.hasOneUse())
9859 return true;
9860 }
9861 // Next, try a deep search: check whether every operand of the TokenFactor
9862 // reaches Dest.
9863 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
9864 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
9865 });
9866 }
9867
9868 // Loads don't have side effects, look through them.
9869 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
9870 if (Ld->isUnordered())
9871 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
9872 }
9873 return false;
9874}
9875
9876bool SDNode::hasPredecessor(const SDNode *N) const {
9877 SmallPtrSet<const SDNode *, 32> Visited;
9878 SmallVector<const SDNode *, 16> Worklist;
9879 Worklist.push_back(this);
9880 return hasPredecessorHelper(N, Visited, Worklist);
9881}
9882
9883void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
9884 this->Flags.intersectWith(Flags);
9885}
9886
9887SDValue
9888SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
9889 ArrayRef<ISD::NodeType> CandidateBinOps,
9890 bool AllowPartials) {
9891 // The pattern must end in an extract from index 0.
9892 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9893 !isNullConstant(Extract->getOperand(1)))
9894 return SDValue();
9895
9896 // Match against one of the candidate binary ops.
9897 SDValue Op = Extract->getOperand(0);
9898 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
9899 return Op.getOpcode() == unsigned(BinOp);
9900 }))
9901 return SDValue();
9902
9903 // Floating-point reductions may require relaxed constraints on the final step
9904 // of the reduction because they may reorder intermediate operations.
9905 unsigned CandidateBinOp = Op.getOpcode();
9906 if (Op.getValueType().isFloatingPoint()) {
9907 SDNodeFlags Flags = Op->getFlags();
9908 switch (CandidateBinOp) {
9909 case ISD::FADD:
9910 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
9911 return SDValue();
9912 break;
9913 default:
9914 llvm_unreachable("Unhandled FP opcode for binop reduction")__builtin_unreachable();
9915 }
9916 }
9917
9918 // Matching failed - attempt to see if we did enough stages that a partial
9919 // reduction from a subvector is possible.
9920 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
9921 if (!AllowPartials || !Op)
9922 return SDValue();
9923 EVT OpVT = Op.getValueType();
9924 EVT OpSVT = OpVT.getScalarType();
9925 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
9926 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
9927 return SDValue();
9928 BinOp = (ISD::NodeType)CandidateBinOp;
9929 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
9930 getVectorIdxConstant(0, SDLoc(Op)));
9931 };
9932
9933 // At each stage, we're looking for something that looks like:
9934 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
9935 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
9936 // i32 undef, i32 undef, i32 undef, i32 undef>
9937 // %a = binop <8 x i32> %op, %s
9938 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
9939 // we expect something like:
9940 // <4,5,6,7,u,u,u,u>
9941 // <2,3,u,u,u,u,u,u>
9942 // <1,u,u,u,u,u,u,u>
9943 // While a partial reduction match would be:
9944 // <2,3,u,u,u,u,u,u>
9945 // <1,u,u,u,u,u,u,u>
9946 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
9947 SDValue PrevOp;
9948 for (unsigned i = 0; i < Stages; ++i) {
9949 unsigned MaskEnd = (1 << i);
9950
9951 if (Op.getOpcode() != CandidateBinOp)
9952 return PartialReduction(PrevOp, MaskEnd);
9953
9954 SDValue Op0 = Op.getOperand(0);
9955 SDValue Op1 = Op.getOperand(1);
9956
9957 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
9958 if (Shuffle) {
9959 Op = Op1;
9960 } else {
9961 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
9962 Op = Op0;
9963 }
9964
9965 // The first operand of the shuffle should be the same as the other operand
9966 // of the binop.
9967 if (!Shuffle || Shuffle->getOperand(0) != Op)
9968 return PartialReduction(PrevOp, MaskEnd);
9969
9970 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
9971 for (int Index = 0; Index < (int)MaskEnd; ++Index)
9972 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
9973 return PartialReduction(PrevOp, MaskEnd);
9974
9975 PrevOp = Op;
9976 }
9977
9978 // Handle subvector reductions, which tend to appear after the shuffle
9979 // reduction stages.
9980 while (Op.getOpcode() == CandidateBinOp) {
9981 unsigned NumElts = Op.getValueType().getVectorNumElements();
9982 SDValue Op0 = Op.getOperand(0);
9983 SDValue Op1 = Op.getOperand(1);
9984 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
9985 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
9986 Op0.getOperand(0) != Op1.getOperand(0))
9987 break;
9988 SDValue Src = Op0.getOperand(0);
9989 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
9990 if (NumSrcElts != (2 * NumElts))
9991 break;
9992 if (!(Op0.getConstantOperandAPInt(1) == 0 &&
9993 Op1.getConstantOperandAPInt(1) == NumElts) &&
9994 !(Op1.getConstantOperandAPInt(1) == 0 &&
9995 Op0.getConstantOperandAPInt(1) == NumElts))
9996 break;
9997 Op = Src;
9998 }
9999
10000 BinOp = (ISD::NodeType)CandidateBinOp;
10001 return Op;
10002}
10003
10004SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
10005 assert(N->getNumValues() == 1 &&((void)0)
10006 "Can't unroll a vector with multiple results!")((void)0);
10007
10008 EVT VT = N->getValueType(0);
10009 unsigned NE = VT.getVectorNumElements();
10010 EVT EltVT = VT.getVectorElementType();
10011 SDLoc dl(N);
10012
10013 SmallVector<SDValue, 8> Scalars;
10014 SmallVector<SDValue, 4> Operands(N->getNumOperands());
10015
10016 // If ResNE is 0, fully unroll the vector op.
10017 if (ResNE == 0)
10018 ResNE = NE;
10019 else if (NE > ResNE)
10020 NE = ResNE;
10021
10022 unsigned i;
10023 for (i= 0; i != NE; ++i) {
10024 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
10025 SDValue Operand = N->getOperand(j);
10026 EVT OperandVT = Operand.getValueType();
10027 if (OperandVT.isVector()) {
10028 // A vector operand; extract a single element.
10029 EVT OperandEltVT = OperandVT.getVectorElementType();
10030 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
10031 Operand, getVectorIdxConstant(i, dl));
10032 } else {
10033 // A scalar operand; just use it as is.
10034 Operands[j] = Operand;
10035 }
10036 }
10037
10038 switch (N->getOpcode()) {
10039 default: {
10040 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
10041 N->getFlags()));
10042 break;
10043 }
10044 case ISD::VSELECT:
10045 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
10046 break;
10047 case ISD::SHL:
10048 case ISD::SRA:
10049 case ISD::SRL:
10050 case ISD::ROTL:
10051 case ISD::ROTR:
10052 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
10053 getShiftAmountOperand(Operands[0].getValueType(),
10054 Operands[1])));
10055 break;
10056 case ISD::SIGN_EXTEND_INREG: {
10057 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
10058 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
10059 Operands[0],
10060 getValueType(ExtVT)));
10061 }
10062 }
10063 }
10064
10065 for (; i < ResNE; ++i)
10066 Scalars.push_back(getUNDEF(EltVT));
10067
10068 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
10069 return getBuildVector(VecVT, dl, Scalars);
10070}
10071
10072std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
10073 SDNode *N, unsigned ResNE) {
10074 unsigned Opcode = N->getOpcode();
10075 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||((void)0)
10076 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||((void)0)
10077 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&((void)0)
10078 "Expected an overflow opcode")((void)0);
10079
10080 EVT ResVT = N->getValueType(0);
10081 EVT OvVT = N->getValueType(1);
10082 EVT ResEltVT = ResVT.getVectorElementType();
10083 EVT OvEltVT = OvVT.getVectorElementType();
10084 SDLoc dl(N);
10085
10086 // If ResNE is 0, fully unroll the vector op.
10087 unsigned NE = ResVT.getVectorNumElements();
10088 if (ResNE == 0)
10089 ResNE = NE;
10090 else if (NE > ResNE)
10091 NE = ResNE;
10092
10093 SmallVector<SDValue, 8> LHSScalars;
10094 SmallVector<SDValue, 8> RHSScalars;
10095 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
10096 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
10097
10098 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
10099 SDVTList VTs = getVTList(ResEltVT, SVT);
10100 SmallVector<SDValue, 8> ResScalars;
10101 SmallVector<SDValue, 8> OvScalars;
10102 for (unsigned i = 0; i < NE; ++i) {
10103 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
10104 SDValue Ov =
10105 getSelect(dl, OvEltVT, Res.getValue(1),
10106 getBoolConstant(true, dl, OvEltVT, ResVT),
10107 getConstant(0, dl, OvEltVT));
10108
10109 ResScalars.push_back(Res);
10110 OvScalars.push_back(Ov);
10111 }
10112
10113 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
10114 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
10115
10116 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
10117 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
10118 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
10119 getBuildVector(NewOvVT, dl, OvScalars));
10120}
10121
10122bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
10123 LoadSDNode *Base,
10124 unsigned Bytes,
10125 int Dist) const {
10126 if (LD->isVolatile() || Base->isVolatile())
10127 return false;
10128 // TODO: probably too restrictive for atomics, revisit
10129 if (!LD->isSimple())
10130 return false;
10131 if (LD->isIndexed() || Base->isIndexed())
10132 return false;
10133 if (LD->getChain() != Base->getChain())
10134 return false;
10135 EVT VT = LD->getValueType(0);
10136 if (VT.getSizeInBits() / 8 != Bytes)
10137 return false;
10138
10139 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
10140 auto LocDecomp = BaseIndexOffset::match(LD, *this);
10141
10142 int64_t Offset = 0;
10143 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
10144 return (Dist * Bytes == Offset);
10145 return false;
10146}
10147
10148/// InferPtrAlignment - Infer alignment of a load / store address. Return None
10149/// if it cannot be inferred.
10150MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
10151 // If this is a GlobalAddress + cst, return the alignment.
10152 const GlobalValue *GV = nullptr;
10153 int64_t GVOffset = 0;
10154 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
10155 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
10156 KnownBits Known(PtrWidth);
10157 llvm::computeKnownBits(GV, Known, getDataLayout());
10158 unsigned AlignBits = Known.countMinTrailingZeros();
10159 if (AlignBits)
10160 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
10161 }
10162
10163 // If this is a direct reference to a stack slot, use information about the
10164 // stack slot's alignment.
10165 int FrameIdx = INT_MIN(-2147483647 -1);
10166 int64_t FrameOffset = 0;
10167 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
10168 FrameIdx = FI->getIndex();
10169 } else if (isBaseWithConstantOffset(Ptr) &&
10170 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
10171 // Handle FI+Cst
10172 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
10173 FrameOffset = Ptr.getConstantOperandVal(1);
10174 }
10175
10176 if (FrameIdx != INT_MIN(-2147483647 -1)) {
10177 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
10178 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
10179 }
10180
10181 return None;
10182}
10183
10184/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
10185/// which is split (or expanded) into two not necessarily identical pieces.
10186std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
10187 // Currently all types are split in half.
10188 EVT LoVT, HiVT;
10189 if (!VT.isVector())
10190 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
10191 else
10192 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
10193
10194 return std::make_pair(LoVT, HiVT);
10195}
10196
10197/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
10198/// type, dependent on an enveloping VT that has been split into two identical
10199/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
10200std::pair<EVT, EVT>
10201SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
10202 bool *HiIsEmpty) const {
10203 EVT EltTp = VT.getVectorElementType();
10204 // Examples:
10205 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
10206 // custom VL=9 with enveloping VL=8/8 yields 8/1
10207 // custom VL=10 with enveloping VL=8/8 yields 8/2
10208 // etc.
10209 ElementCount VTNumElts = VT.getVectorElementCount();
10210 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
10211 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&((void)0)
10212 "Mixing fixed width and scalable vectors when enveloping a type")((void)0);
10213 EVT LoVT, HiVT;
10214 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
10215 LoVT = EnvVT;
10216 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
10217 *HiIsEmpty = false;
10218 } else {
10219 // Flag that hi type has zero storage size, but return split envelop type
10220 // (this would be easier if vector types with zero elements were allowed).
10221 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
10222 HiVT = EnvVT;
10223 *HiIsEmpty = true;
10224 }
10225 return std::make_pair(LoVT, HiVT);
10226}
10227
10228/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
10229/// low/high part.
10230std::pair<SDValue, SDValue>
10231SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
10232 const EVT &HiVT) {
10233 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&((void)0)
10234 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&((void)0)
10235 "Splitting vector with an invalid mixture of fixed and scalable "((void)0)
10236 "vector types")((void)0);
10237 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=((void)0)
10238 N.getValueType().getVectorMinNumElements() &&((void)0)
10239 "More vector elements requested than available!")((void)0);
10240 SDValue Lo, Hi;
10241 Lo =
10242 getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL));
10243 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
10244 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
10245 // IDX with the runtime scaling factor of the result vector type. For
10246 // fixed-width result vectors, that runtime scaling factor is 1.
10247 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
10248 getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
10249 return std::make_pair(Lo, Hi);
10250}
10251
10252/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
10253SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
10254 EVT VT = N.getValueType();
10255 EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
10256 NextPowerOf2(VT.getVectorNumElements()));
10257 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
10258 getVectorIdxConstant(0, DL));
10259}
10260
10261void SelectionDAG::ExtractVectorElements(SDValue Op,
10262 SmallVectorImpl<SDValue> &Args,
10263 unsigned Start, unsigned Count,
10264 EVT EltVT) {
10265 EVT VT = Op.getValueType();
10266 if (Count == 0)
10267 Count = VT.getVectorNumElements();
10268 if (EltVT == EVT())
10269 EltVT = VT.getVectorElementType();
10270 SDLoc SL(Op);
10271 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
10272 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
10273 getVectorIdxConstant(i, SL)));
10274 }
10275}
10276
10277// getAddressSpace - Return the address space this GlobalAddress belongs to.
10278unsigned GlobalAddressSDNode::getAddressSpace() const {
10279 return getGlobal()->getType()->getAddressSpace();
10280}
10281
10282Type *ConstantPoolSDNode::getType() const {
10283 if (isMachineConstantPoolEntry())
10284 return Val.MachineCPVal->getType();
10285 return Val.ConstVal->getType();
10286}
10287
10288bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
10289 unsigned &SplatBitSize,
10290 bool &HasAnyUndefs,
10291 unsigned MinSplatBits,
10292 bool IsBigEndian) const {
10293 EVT VT = getValueType(0);
10294 assert(VT.isVector() && "Expected a vector type")((void)0);
10295 unsigned VecWidth = VT.getSizeInBits();
10296 if (MinSplatBits > VecWidth)
10297 return false;
10298
10299 // FIXME: The widths are based on this node's type, but build vectors can
10300 // truncate their operands.
10301 SplatValue = APInt(VecWidth, 0);
10302 SplatUndef = APInt(VecWidth, 0);
10303
10304 // Get the bits. Bits with undefined values (when the corresponding element
10305 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
10306 // in SplatValue. If any of the values are not constant, give up and return
10307 // false.
10308 unsigned int NumOps = getNumOperands();
10309 assert(NumOps > 0 && "isConstantSplat has 0-size build vector")((void)0);
10310 unsigned EltWidth = VT.getScalarSizeInBits();
10311
10312 for (unsigned j = 0; j < NumOps; ++j) {
10313 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
10314 SDValue OpVal = getOperand(i);
10315 unsigned BitPos = j * EltWidth;
10316
10317 if (OpVal.isUndef())
10318 SplatUndef.setBits(BitPos, BitPos + EltWidth);
10319 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
10320 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
10321 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
10322 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
10323 else
10324 return false;
10325 }
10326
10327 // The build_vector is all constants or undefs. Find the smallest element
10328 // size that splats the vector.
10329 HasAnyUndefs = (SplatUndef != 0);
10330
10331 // FIXME: This does not work for vectors with elements less than 8 bits.
10332 while (VecWidth > 8) {
10333 unsigned HalfSize = VecWidth / 2;
10334 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
10335 APInt LowValue = SplatValue.extractBits(HalfSize, 0);
10336 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
10337 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
10338
10339 // If the two halves do not match (ignoring undef bits), stop here.
10340 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
10341 MinSplatBits > HalfSize)
10342 break;
10343
10344 SplatValue = HighValue | LowValue;
10345 SplatUndef = HighUndef & LowUndef;
10346
10347 VecWidth = HalfSize;
10348 }
10349
10350 SplatBitSize = VecWidth;
10351 return true;
10352}
10353
10354SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
10355 BitVector *UndefElements) const {
10356 unsigned NumOps = getNumOperands();
10357 if (UndefElements) {
10358 UndefElements->clear();
10359 UndefElements->resize(NumOps);
10360 }
10361 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size")((void)0);
10362 if (!DemandedElts)
10363 return SDValue();
10364 SDValue Splatted;
10365 for (unsigned i = 0; i != NumOps; ++i) {
10366 if (!DemandedElts[i])
10367 continue;
10368 SDValue Op = getOperand(i);
10369 if (Op.isUndef()) {
10370 if (UndefElements)
10371 (*UndefElements)[i] = true;
10372 } else if (!Splatted) {
10373 Splatted = Op;
10374 } else if (Splatted != Op) {
10375 return SDValue();
10376 }
10377 }
10378
10379 if (!Splatted) {
10380 unsigned FirstDemandedIdx = DemandedElts.countTrailingZeros();
10381 assert(getOperand(FirstDemandedIdx).isUndef() &&((void)0)
10382 "Can only have a splat without a constant for all undefs.")((void)0);
10383 return getOperand(FirstDemandedIdx);
10384 }
10385
10386 return Splatted;
10387}
10388
10389SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
10390 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands());
10391 return getSplatValue(DemandedElts, UndefElements);
10392}
10393
10394bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
10395 SmallVectorImpl<SDValue> &Sequence,
10396 BitVector *UndefElements) const {
10397 unsigned NumOps = getNumOperands();
10398 Sequence.clear();
10399 if (UndefElements) {
10400 UndefElements->clear();
10401 UndefElements->resize(NumOps);
10402 }
10403 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size")((void)0);
10404 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
10405 return false;
10406
10407 // Set the undefs even if we don't find a sequence (like getSplatValue).
10408 if (UndefElements)
10409 for (unsigned I = 0; I != NumOps; ++I)
10410 if (DemandedElts[I] && getOperand(I).isUndef())
10411 (*UndefElements)[I] = true;
10412
10413 // Iteratively widen the sequence length looking for repetitions.
10414 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
10415 Sequence.append(SeqLen, SDValue());
10416 for (unsigned I = 0; I != NumOps; ++I) {
10417 if (!DemandedElts[I])
10418 continue;
10419 SDValue &SeqOp = Sequence[I % SeqLen];
10420 SDValue Op = getOperand(I);
10421 if (Op.isUndef()) {
10422 if (!SeqOp)
10423 SeqOp = Op;
10424 continue;
10425 }
10426 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
10427 Sequence.clear();
10428 break;
10429 }
10430 SeqOp = Op;
10431 }
10432 if (!Sequence.empty())
10433 return true;
10434 }
10435
10436 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern")((void)0);
10437 return false;
10438}
10439
10440bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
10441 BitVector *UndefElements) const {
10442 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands());
10443 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
10444}
10445
10446ConstantSDNode *
10447BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
10448 BitVector *UndefElements) const {
10449 return dyn_cast_or_null<ConstantSDNode>(
10450 getSplatValue(DemandedElts, UndefElements));
10451}
10452
10453ConstantSDNode *
10454BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
10455 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
10456}
10457
10458ConstantFPSDNode *
10459BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
10460 BitVector *UndefElements) const {
10461 return dyn_cast_or_null<ConstantFPSDNode>(
10462 getSplatValue(DemandedElts, UndefElements));
10463}
10464
10465ConstantFPSDNode *
10466BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
10467 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
10468}
10469
10470int32_t
10471BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
10472 uint32_t BitWidth) const {
10473 if (ConstantFPSDNode *CN =
10474 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
10475 bool IsExact;
10476 APSInt IntVal(BitWidth);
10477 const APFloat &APF = CN->getValueAPF();
10478 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
10479 APFloat::opOK ||
10480 !IsExact)
10481 return -1;
10482
10483 return IntVal.exactLogBase2();
10484 }
10485 return -1;
10486}
10487
10488bool BuildVectorSDNode::isConstant() const {
10489 for (const SDValue &Op : op_values()) {
10490 unsigned Opc = Op.getOpcode();
10491 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
10492 return false;
10493 }
10494 return true;
10495}
10496
10497bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
10498 // Find the first non-undef value in the shuffle mask.
10499 unsigned i, e;
10500 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
10501 /* search */;
10502
10503 // If all elements are undefined, this shuffle can be considered a splat
10504 // (although it should eventually get simplified away completely).
10505 if (i == e)
10506 return true;
10507
10508 // Make sure all remaining elements are either undef or the same as the first
10509 // non-undef value.
10510 for (int Idx = Mask[i]; i != e; ++i)
10511 if (Mask[i] >= 0 && Mask[i] != Idx)
10512 return false;
10513 return true;
10514}
10515
10516// Returns the SDNode if it is a constant integer BuildVector
10517// or constant integer.
10518SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const {
10519 if (isa<ConstantSDNode>(N))
10520 return N.getNode();
10521 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
10522 return N.getNode();
10523 // Treat a GlobalAddress supporting constant offset folding as a
10524 // constant integer.
10525 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
10526 if (GA->getOpcode() == ISD::GlobalAddress &&
10527 TLI->isOffsetFoldingLegal(GA))
10528 return GA;
10529 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
10530 isa<ConstantSDNode>(N.getOperand(0)))
10531 return N.getNode();
10532 return nullptr;
10533}
10534
10535// Returns the SDNode if it is a constant float BuildVector
10536// or constant float.
10537SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
10538 if (isa<ConstantFPSDNode>(N))
10539 return N.getNode();
10540
10541 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
10542 return N.getNode();
10543
10544 return nullptr;
10545}
10546
10547void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
10548 assert(!Node->OperandList && "Node already has operands")((void)0);
10549 assert(SDNode::getMaxNumOperands() >= Vals.size() &&((void)0)
10550 "too many operands to fit into SDNode")((void)0);
10551 SDUse *Ops = OperandRecycler.allocate(
10552 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
10553
10554 bool IsDivergent = false;
10555 for (unsigned I = 0; I != Vals.size(); ++I) {
10556 Ops[I].setUser(Node);
10557 Ops[I].setInitial(Vals[I]);
10558 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
10559 IsDivergent |= Ops[I].getNode()->isDivergent();
10560 }
10561 Node->NumOperands = Vals.size();
10562 Node->OperandList = Ops;
10563 if (!TLI->isSDNodeAlwaysUniform(Node)) {
10564 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA);
10565 Node->SDNodeBits.IsDivergent = IsDivergent;
10566 }
10567 checkForCycles(Node);
10568}
10569
10570SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
10571 SmallVectorImpl<SDValue> &Vals) {
10572 size_t Limit = SDNode::getMaxNumOperands();
10573 while (Vals.size() > Limit) {
10574 unsigned SliceIdx = Vals.size() - Limit;
10575 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
10576 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
10577 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
10578 Vals.emplace_back(NewTF);
10579 }
10580 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
10581}
10582
10583SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
10584 EVT VT, SDNodeFlags Flags) {
10585 switch (Opcode) {
10586 default:
10587 return SDValue();
10588 case ISD::ADD:
10589 case ISD::OR:
10590 case ISD::XOR:
10591 case ISD::UMAX:
10592 return getConstant(0, DL, VT);
10593 case ISD::MUL:
10594 return getConstant(1, DL, VT);
10595 case ISD::AND:
10596 case ISD::UMIN:
10597 return getAllOnesConstant(DL, VT);
10598 case ISD::SMAX:
10599 return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT);
10600 case ISD::SMIN:
10601 return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT);
10602 case ISD::FADD:
10603 return getConstantFP(-0.0, DL, VT);
10604 case ISD::FMUL:
10605 return getConstantFP(1.0, DL, VT);
10606 case ISD::FMINNUM:
10607 case ISD::FMAXNUM: {
10608 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
10609 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
10610 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
10611 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
10612 APFloat::getLargest(Semantics);
10613 if (Opcode == ISD::FMAXNUM)
10614 NeutralAF.changeSign();
10615
10616 return getConstantFP(NeutralAF, DL, VT);
10617 }
10618 }
10619}
10620
10621#ifndef NDEBUG1
10622static void checkForCyclesHelper(const SDNode *N,
10623 SmallPtrSetImpl<const SDNode*> &Visited,
10624 SmallPtrSetImpl<const SDNode*> &Checked,
10625 const llvm::SelectionDAG *DAG) {
10626 // If this node has already been checked, don't check it again.
10627 if (Checked.count(N))
10628 return;
10629
10630 // If a node has already been visited on this depth-first walk, reject it as
10631 // a cycle.
10632 if (!Visited.insert(N).second) {
10633 errs() << "Detected cycle in SelectionDAG\n";
10634 dbgs() << "Offending node:\n";
10635 N->dumprFull(DAG); dbgs() << "\n";
10636 abort();
10637 }
10638
10639 for (const SDValue &Op : N->op_values())
10640 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
10641
10642 Checked.insert(N);
10643 Visited.erase(N);
10644}
10645#endif
10646
10647void llvm::checkForCycles(const llvm::SDNode *N,
10648 const llvm::SelectionDAG *DAG,
10649 bool force) {
10650#ifndef NDEBUG1
10651 bool check = force;
10652#ifdef EXPENSIVE_CHECKS
10653 check = true;
10654#endif // EXPENSIVE_CHECKS
10655 if (check) {
10656 assert(N && "Checking nonexistent SDNode")((void)0);
10657 SmallPtrSet<const SDNode*, 32> visited;
10658 SmallPtrSet<const SDNode*, 32> checked;
10659 checkForCyclesHelper(N, visited, checked, DAG);
10660 }
10661#endif // !NDEBUG
10662}
10663
10664void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
10665 checkForCycles(DAG->getRoot().getNode(), DAG, force);
10666}
←

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support/Casting.h

→
1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
10// and dyn_cast_or_null<X>() templates.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_CASTING_H
15#define LLVM_SUPPORT_CASTING_H
16
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/type_traits.h"
19#include <cassert>
20#include <memory>
21#include <type_traits>
22
23namespace llvm {
24
25//===----------------------------------------------------------------------===//
26// isa<x> Support Templates
27//===----------------------------------------------------------------------===//
28
29// Define a template that can be specialized by smart pointers to reflect the
30// fact that they are automatically dereferenced, and are not involved with the
31// template selection process... the default implementation is a noop.
32//
33template<typename From> struct simplify_type {
34 using SimpleType = From; // The real type this represents...
35
36 // An accessor to get the real value...
37 static SimpleType &getSimplifiedValue(From &Val) { return Val; }
38};
39
40template<typename From> struct simplify_type<const From> {
41 using NonConstSimpleType = typename simplify_type<From>::SimpleType;
42 using SimpleType =
43 typename add_const_past_pointer<NonConstSimpleType>::type;
44 using RetType =
45 typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
46
47 static RetType getSimplifiedValue(const From& Val) {
48 return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
49 }
50};
51
52// The core of the implementation of isa<X> is here; To and From should be
53// the names of classes. This template can be specialized to customize the
54// implementation of isa<> without rewriting it from scratch.
55template <typename To, typename From, typename Enabler = void>
56struct isa_impl {
57 static inline bool doit(const From &Val) {
58 return To::classof(&Val);
59 }
60};
61
62/// Always allow upcasts, and perform no dynamic check for them.
63template <typename To, typename From>
64struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> {
65 static inline bool doit(const From &) { return true; }
66};
67
68template <typename To, typename From> struct isa_impl_cl {
69 static inline bool doit(const From &Val) {
70 return isa_impl<To, From>::doit(Val);
71 }
72};
73
74template <typename To, typename From> struct isa_impl_cl<To, const From> {
75 static inline bool doit(const From &Val) {
76 return isa_impl<To, From>::doit(Val);
77 }
78};
79
80template <typename To, typename From>
81struct isa_impl_cl<To, const std::unique_ptr<From>> {
82 static inline bool doit(const std::unique_ptr<From> &Val) {
83 assert(Val && "isa<> used on a null pointer")((void)0);
84 return isa_impl_cl<To, From>::doit(*Val);
85 }
86};
87
88template <typename To, typename From> struct isa_impl_cl<To, From*> {
89 static inline bool doit(const From *Val) {
90 assert(Val && "isa<> used on a null pointer")((void)0);
91 return isa_impl<To, From>::doit(*Val);
92 }
93};
94
95template <typename To, typename From> struct isa_impl_cl<To, From*const> {
96 static inline bool doit(const From *Val) {
97 assert(Val && "isa<> used on a null pointer")((void)0);
98 return isa_impl<To, From>::doit(*Val);
99 }
100};
101
102template <typename To, typename From> struct isa_impl_cl<To, const From*> {
103 static inline bool doit(const From *Val) {
104 assert(Val && "isa<> used on a null pointer")((void)0);
105 return isa_impl<To, From>::doit(*Val);
106 }
107};
108
109template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
110 static inline bool doit(const From *Val) {
111 assert(Val && "isa<> used on a null pointer")((void)0);
112 return isa_impl<To, From>::doit(*Val);
113 }
114};
115
116template<typename To, typename From, typename SimpleFrom>
117struct isa_impl_wrap {
118 // When From != SimplifiedType, we can simplify the type some more by using
119 // the simplify_type template.
120 static bool doit(const From &Val) {
121 return isa_impl_wrap<To, SimpleFrom,
122 typename simplify_type<SimpleFrom>::SimpleType>::doit(
123 simplify_type<const From>::getSimplifiedValue(Val));
124 }
125};
126
127template<typename To, typename FromTy>
128struct isa_impl_wrap<To, FromTy, FromTy> {
129 // When From == SimpleType, we are as simple as we are going to get.
130 static bool doit(const FromTy &Val) {
131 return isa_impl_cl<To,FromTy>::doit(Val);
132 }
133};
134
135// isa<X> - Return true if the parameter to the template is an instance of one
136// of the template type arguments. Used like this:
137//
138// if (isa<Type>(myVal)) { ... }
139// if (isa<Type0, Type1, Type2>(myVal)) { ... }
140//
141template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
142 return isa_impl_wrap<X, const Y,
143 typename simplify_type<const Y>::SimpleType>::doit(Val);
144}
145
146template <typename First, typename Second, typename... Rest, typename Y>
147LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
148 return isa<First>(Val) || isa<Second, Rest...>(Val);
149}
150
151// isa_and_nonnull<X> - Functionally identical to isa, except that a null value
152// is accepted.
153//
154template <typename... X, class Y>
155LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa_and_nonnull(const Y &Val) {
156 if (!Val)
157 return false;
158 return isa<X...>(Val);
159}
160
161//===----------------------------------------------------------------------===//
162// cast<x> Support Templates
163//===----------------------------------------------------------------------===//
164
165template<class To, class From> struct cast_retty;
166
167// Calculate what type the 'cast' function should return, based on a requested
168// type of To and a source type of From.
169template<class To, class From> struct cast_retty_impl {
170 using ret_type = To &; // Normal case, return Ty&
171};
172template<class To, class From> struct cast_retty_impl<To, const From> {
173 using ret_type = const To &; // Normal case, return Ty&
174};
175
176template<class To, class From> struct cast_retty_impl<To, From*> {
177 using ret_type = To *; // Pointer arg case, return Ty*
178};
179
180template<class To, class From> struct cast_retty_impl<To, const From*> {
181 using ret_type = const To *; // Constant pointer arg case, return const Ty*
182};
183
184template<class To, class From> struct cast_retty_impl<To, const From*const> {
185 using ret_type = const To *; // Constant pointer arg case, return const Ty*
186};
187
188template <class To, class From>
189struct cast_retty_impl<To, std::unique_ptr<From>> {
190private:
191 using PointerType = typename cast_retty_impl<To, From *>::ret_type;
192 using ResultType = std::remove_pointer_t<PointerType>;
193
194public:
195 using ret_type = std::unique_ptr<ResultType>;
196};
197
198template<class To, class From, class SimpleFrom>
199struct cast_retty_wrap {
200 // When the simplified type and the from type are not the same, use the type
201 // simplifier to reduce the type, then reuse cast_retty_impl to get the
202 // resultant type.
203 using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
204};
205
206template<class To, class FromTy>
207struct cast_retty_wrap<To, FromTy, FromTy> {
208 // When the simplified type is equal to the from type, use it directly.
209 using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
210};
211
212template<class To, class From>
213struct cast_retty {
214 using ret_type = typename cast_retty_wrap<
215 To, From, typename simplify_type<From>::SimpleType>::ret_type;
216};
217
218// Ensure the non-simple values are converted using the simplify_type template
219// that may be specialized by smart pointers...
220//
221template<class To, class From, class SimpleFrom> struct cast_convert_val {
222 // This is not a simple type, use the template to simplify it...
223 static typename cast_retty<To, From>::ret_type doit(From &Val) {
224 return cast_convert_val<To, SimpleFrom,
11
←
Returning without writing to 'Val.Node'
→
225 typename simplify_type<SimpleFrom>::SimpleType>::doit(
226 simplify_type<From>::getSimplifiedValue(Val));
8
←
Calling 'simplify_type::getSimplifiedValue'
→
10
←
Returning from 'simplify_type::getSimplifiedValue'
→
227 }
228};
229
230template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
231 // This _is_ a simple type, just cast it.
232 static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
233 typename cast_retty<To, FromTy>::ret_type Res2
234 = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
235 return Res2;
236 }
237};
238
239template <class X> struct is_simple_type {
240 static const bool value =
241 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
242};
243
244// cast<X> - Return the argument parameter cast to the specified type. This
245// casting operator asserts that the type is correct, so it does not return null
246// on failure. It does not allow a null argument (use cast_or_null for that).
247// It is typically used like this:
248//
249// cast<Instruction>(myVal)->getParent()
250//
251template <class X, class Y>
252inline std::enable_if_t<!is_simple_type<Y>::value,
253 typename cast_retty<X, const Y>::ret_type>
254cast(const Y &Val) {
255 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((void)0);
256 return cast_convert_val<
257 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
258}
259
260template <class X, class Y>
261inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
262 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((void)0);
263 return cast_convert_val<X, Y,
7
←
Calling 'cast_convert_val::doit'
→
12
←
Returning from 'cast_convert_val::doit'
→
13
←
Returning without writing to 'Val.Node'
→
264 typename simplify_type<Y>::SimpleType>::doit(Val);
265}
266
267template <class X, class Y>
268inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
269 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((void)0);
270 return cast_convert_val<X, Y*,
271 typename simplify_type<Y*>::SimpleType>::doit(Val);
272}
273
274template <class X, class Y>
275inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
276cast(std::unique_ptr<Y> &&Val) {
277 assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")((void)0);
278 using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
279 return ret_type(
280 cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
281 Val.release()));
282}
283
284// cast_or_null<X> - Functionally identical to cast, except that a null value is
285// accepted.
286//
287template <class X, class Y>
288LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
289 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
290cast_or_null(const Y &Val) {
291 if (!Val)
292 return nullptr;
293 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((void)0);
294 return cast<X>(Val);
295}
296
297template <class X, class Y>
298LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
299 typename cast_retty<X, Y>::ret_type>
300cast_or_null(Y &Val) {
301 if (!Val)
302 return nullptr;
303 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((void)0);
304 return cast<X>(Val);
305}
306
307template <class X, class Y>
308LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
309cast_or_null(Y *Val) {
310 if (!Val) return nullptr;
311 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((void)0);
312 return cast<X>(Val);
313}
314
315template <class X, class Y>
316inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
317cast_or_null(std::unique_ptr<Y> &&Val) {
318 if (!Val)
319 return nullptr;
320 return cast<X>(std::move(Val));
321}
322
323// dyn_cast<X> - Return the argument parameter cast to the specified type. This
324// casting operator returns null if the argument is of the wrong type, so it can
325// be used to test for a type as well as cast if successful. This should be
326// used in the context of an if statement like this:
327//
328// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
329//
330
331template <class X, class Y>
332LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
333 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
334dyn_cast(const Y &Val) {
335 return isa<X>(Val) ? cast<X>(Val) : nullptr;
336}
337
338template <class X, class Y>
339LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
340 return isa<X>(Val) ? cast<X>(Val) : nullptr;
4
←
Assuming 'Val' is a 'ConstantSDNode'
→
5
←
'?' condition is true
→
6
←
Calling 'cast<llvm::ConstantSDNode, llvm::SDValue>'
→
14
←
Returning from 'cast<llvm::ConstantSDNode, llvm::SDValue>'
→
15
←
Returning without writing to 'Val.Node'
→
341}
342
343template <class X, class Y>
344LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
345 return isa<X>(Val) ? cast<X>(Val) : nullptr;
346}
347
348// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
349// value is accepted.
350//
351template <class X, class Y>
352LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
353 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
354dyn_cast_or_null(const Y &Val) {
355 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
356}
357
358template <class X, class Y>
359LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
360 typename cast_retty<X, Y>::ret_type>
361dyn_cast_or_null(Y &Val) {
362 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
363}
364
365template <class X, class Y>
366LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
367dyn_cast_or_null(Y *Val) {
368 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
369}
370
371// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
372// taking ownership of the input pointer iff isa<X>(Val) is true. If the
373// cast is successful, From refers to nullptr on exit and the casted value
374// is returned. If the cast is unsuccessful, the function returns nullptr
375// and From is unchanged.
376template <class X, class Y>
377LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
378 -> decltype(cast<X>(Val)) {
379 if (!isa<X>(Val))
380 return nullptr;
381 return cast<X>(std::move(Val));
382}
383
384template <class X, class Y>
385LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) {
386 return unique_dyn_cast<X, Y>(Val);
387}
388
389// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
390// a null value is accepted.
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
393 -> decltype(cast<X>(Val)) {
394 if (!Val)
395 return nullptr;
396 return unique_dyn_cast<X, Y>(Val);
397}
398
399template <class X, class Y>
400LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val) {
401 return unique_dyn_cast_or_null<X, Y>(Val);
402}
403
404} // end namespace llvm
405
406#endif // LLVM_SUPPORT_CASTING_H
←

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
9
←
Returning without writing to 'Val.Node'
→
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class MaskedLoadStoreSDNode;
513 friend class MaskedGatherScatterSDNode;
514
515 uint16_t : NumMemSDNodeBits;
516
517 // This storage is shared between disparate class hierarchies to hold an
518 // enumeration specific to the class hierarchy in use.
519 // LSBaseSDNode => enum ISD::MemIndexedMode
520 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
521 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
522 uint16_t AddressingMode : 3;
523 };
524 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
525
526 class LoadSDNodeBitfields {
527 friend class LoadSDNode;
528 friend class MaskedLoadSDNode;
529 friend class MaskedGatherSDNode;
530
531 uint16_t : NumLSBaseSDNodeBits;
532
533 uint16_t ExtTy : 2; // enum ISD::LoadExtType
534 uint16_t IsExpanding : 1;
535 };
536
537 class StoreSDNodeBitfields {
538 friend class StoreSDNode;
539 friend class MaskedStoreSDNode;
540 friend class MaskedScatterSDNode;
541
542 uint16_t : NumLSBaseSDNodeBits;
543
544 uint16_t IsTruncating : 1;
545 uint16_t IsCompressing : 1;
546 };
547
548 union {
549 char RawSDNodeBits[sizeof(uint16_t)];
550 SDNodeBitfields SDNodeBits;
551 ConstantSDNodeBitfields ConstantSDNodeBits;
552 MemSDNodeBitfields MemSDNodeBits;
553 LSBaseSDNodeBitfields LSBaseSDNodeBits;
554 LoadSDNodeBitfields LoadSDNodeBits;
555 StoreSDNodeBitfields StoreSDNodeBits;
556 };
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK
560
561 // RawSDNodeBits must cover the entirety of the union. This means that all of
562 // the union's members must have size <= RawSDNodeBits. We write the RHS as
563 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
564 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
565 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
566 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
567 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
568 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
569 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
570
571private:
572 friend class SelectionDAG;
573 // TODO: unfriend HandleSDNode once we fix its operand handling.
574 friend class HandleSDNode;
575
576 /// Unique id per SDNode in the DAG.
577 int NodeId = -1;
578
579 /// The values that are used by this operation.
580 SDUse *OperandList = nullptr;
581
582 /// The types of the values this node defines. SDNode's may
583 /// define multiple values simultaneously.
584 const EVT *ValueList;
585
586 /// List of uses for this SDNode.
587 SDUse *UseList = nullptr;
588
589 /// The number of entries in the Operand/Value list.
590 unsigned short NumOperands = 0;
591 unsigned short NumValues;
592
593 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
594 // original LLVM instructions.
595 // This is used for turning off scheduling, because we'll forgo
596 // the normal scheduling algorithms and output the instructions according to
597 // this ordering.
598 unsigned IROrder;
599
600 /// Source line information.
601 DebugLoc debugLoc;
602
603 /// Return a pointer to the specified value type.
604 static const EVT *getValueTypeList(EVT VT);
605
606 SDNodeFlags Flags;
607
608public:
609 /// Unique and persistent id per SDNode in the DAG.
610 /// Used for debug printing.
611 uint16_t PersistentId;
612
613 //===--------------------------------------------------------------------===//
614 // Accessors
615 //
616
617 /// Return the SelectionDAG opcode value for this node. For
618 /// pre-isel nodes (those for which isMachineOpcode returns false), these
619 /// are the opcode values in the ISD and <target>ISD namespaces. For
620 /// post-isel opcodes, see getMachineOpcode.
621 unsigned getOpcode() const { return (unsigned short)NodeType; }
622
623 /// Test if this node has a target-specific opcode (in the
624 /// \<target\>ISD namespace).
625 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
626
627 /// Test if this node has a target-specific opcode that may raise
628 /// FP exceptions (in the \<target\>ISD namespace and greater than
629 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
630 /// opcode are currently automatically considered to possibly raise
631 /// FP exceptions as well.
632 bool isTargetStrictFPOpcode() const {
633 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
634 }
635
636 /// Test if this node has a target-specific
637 /// memory-referencing opcode (in the \<target\>ISD namespace and
638 /// greater than FIRST_TARGET_MEMORY_OPCODE).
639 bool isTargetMemoryOpcode() const {
640 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
641 }
642
643 /// Return true if the type of the node type undefined.
644 bool isUndef() const { return NodeType == ISD::UNDEF; }
645
646 /// Test if this node is a memory intrinsic (with valid pointer information).
647 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
648 /// non-memory intrinsics (with chains) that are not really instances of
649 /// MemSDNode. For such nodes, we need some extra state to determine the
650 /// proper classof relationship.
651 bool isMemIntrinsic() const {
652 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
653 NodeType == ISD::INTRINSIC_VOID) &&
654 SDNodeBits.IsMemIntrinsic;
655 }
656
657 /// Test if this node is a strict floating point pseudo-op.
658 bool isStrictFPOpcode() {
659 switch (NodeType) {
660 default:
661 return false;
662 case ISD::STRICT_FP16_TO_FP:
663 case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
665 case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
667 return true;
668 }
669 }
670
671 /// Test if this node has a post-isel opcode, directly
672 /// corresponding to a MachineInstr opcode.
673 bool isMachineOpcode() const { return NodeType < 0; }
674
675 /// This may only be called if isMachineOpcode returns
676 /// true. It returns the MachineInstr opcode value that the node's opcode
677 /// corresponds to.
678 unsigned getMachineOpcode() const {
679 assert(isMachineOpcode() && "Not a MachineInstr opcode!")((void)0);
680 return ~NodeType;
681 }
682
683 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
684 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
685
686 bool isDivergent() const { return SDNodeBits.IsDivergent; }
687
688 /// Return true if there are no uses of this node.
689 bool use_empty() const { return UseList == nullptr; }
690
691 /// Return true if there is exactly one use of this node.
692 bool hasOneUse() const { return hasSingleElement(uses()); }
693
694 /// Return the number of uses of this node. This method takes
695 /// time proportional to the number of uses.
696 size_t use_size() const { return std::distance(use_begin(), use_end()); }
697
698 /// Return the unique node id.
699 int getNodeId() const { return NodeId; }
700
701 /// Set unique node id.
702 void setNodeId(int Id) { NodeId = Id; }
703
704 /// Return the node ordering.
705 unsigned getIROrder() const { return IROrder; }
706
707 /// Set the node ordering.
708 void setIROrder(unsigned Order) { IROrder = Order; }
709
710 /// Return the source location info.
711 const DebugLoc &getDebugLoc() const { return debugLoc; }
712
713 /// Set source location info. Try to avoid this, putting
714 /// it in the constructor is preferable.
715 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
716
717 /// This class provides iterator support for SDUse
718 /// operands that use a specific SDNode.
719 class use_iterator {
720 friend class SDNode;
721
722 SDUse *Op = nullptr;
723
724 explicit use_iterator(SDUse *op) : Op(op) {}
725
726 public:
727 using iterator_category = std::forward_iterator_tag;
728 using value_type = SDUse;
729 using difference_type = std::ptrdiff_t;
730 using pointer = value_type *;
731 using reference = value_type &;
732
733 use_iterator() = default;
734 use_iterator(const use_iterator &I) : Op(I.Op) {}
735
736 bool operator==(const use_iterator &x) const {
737 return Op == x.Op;
738 }
739 bool operator!=(const use_iterator &x) const {
740 return !operator==(x);
741 }
742
743 /// Return true if this iterator is at the end of uses list.
744 bool atEnd() const { return Op == nullptr; }
745
746 // Iterator traversal: forward iteration only.
747 use_iterator &operator++() { // Preincrement
748 assert(Op && "Cannot increment end iterator!")((void)0);
749 Op = Op->getNext();
750 return *this;
751 }
752
753 use_iterator operator++(int) { // Postincrement
754 use_iterator tmp = *this; ++*this; return tmp;
755 }
756
757 /// Retrieve a pointer to the current user node.
758 SDNode *operator*() const {
759 assert(Op && "Cannot dereference end iterator!")((void)0);
760 return Op->getUser();
761 }
762
763 SDNode *operator->() const { return operator*(); }
764
765 SDUse &getUse() const { return *Op; }
766
767 /// Retrieve the operand # of this use in its user.
768 unsigned getOperandNo() const {
769 assert(Op && "Cannot dereference end iterator!")((void)0);
770 return (unsigned)(Op - Op->getUser()->OperandList);
771 }
772 };
773
774 /// Provide iteration support to walk over all uses of an SDNode.
775 use_iterator use_begin() const {
776 return use_iterator(UseList);
777 }
778
779 static use_iterator use_end() { return use_iterator(nullptr); }
780
781 inline iterator_range<use_iterator> uses() {
782 return make_range(use_begin(), use_end());
783 }
784 inline iterator_range<use_iterator> uses() const {
785 return make_range(use_begin(), use_end());
786 }
787
788 /// Return true if there are exactly NUSES uses of the indicated value.
789 /// This method ignores uses of other values defined by this operation.
790 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
791
792 /// Return true if there are any use of the indicated value.
793 /// This method ignores uses of other values defined by this operation.
794 bool hasAnyUseOfValue(unsigned Value) const;
795
796 /// Return true if this node is the only use of N.
797 bool isOnlyUserOf(const SDNode *N) const;
798
799 /// Return true if this node is an operand of N.
800 bool isOperandOf(const SDNode *N) const;
801
802 /// Return true if this node is a predecessor of N.
803 /// NOTE: Implemented on top of hasPredecessor and every bit as
804 /// expensive. Use carefully.
805 bool isPredecessorOf(const SDNode *N) const {
806 return N->hasPredecessor(this);
807 }
808
809 /// Return true if N is a predecessor of this node.
810 /// N is either an operand of this node, or can be reached by recursively
811 /// traversing up the operands.
812 /// NOTE: This is an expensive method. Use it carefully.
813 bool hasPredecessor(const SDNode *N) const;
814
815 /// Returns true if N is a predecessor of any node in Worklist. This
816 /// helper keeps Visited and Worklist sets externally to allow unions
817 /// searches to be performed in parallel, caching of results across
818 /// queries and incremental addition to Worklist. Stops early if N is
819 /// found but will resume. Remember to clear Visited and Worklists
820 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
821 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
822 /// topologically ordered (Operands have strictly smaller node id) and search
823 /// can be pruned leveraging this.
824 static bool hasPredecessorHelper(const SDNode *N,
825 SmallPtrSetImpl<const SDNode *> &Visited,
826 SmallVectorImpl<const SDNode *> &Worklist,
827 unsigned int MaxSteps = 0,
828 bool TopologicalPrune = false) {
829 SmallVector<const SDNode *, 8> DeferredNodes;
830 if (Visited.count(N))
831 return true;
832
833 // Node Id's are assigned in three places: As a topological
834 // ordering (> 0), during legalization (results in values set to
835 // 0), new nodes (set to -1). If N has a topolgical id then we
836 // know that all nodes with ids smaller than it cannot be
837 // successors and we need not check them. Filter out all node
838 // that can't be matches. We add them to the worklist before exit
839 // in case of multiple calls. Note that during selection the topological id
840 // may be violated if a node's predecessor is selected before it. We mark
841 // this at selection negating the id of unselected successors and
842 // restricting topological pruning to positive ids.
843
844 int NId = N->getNodeId();
845 // If we Invalidated the Id, reconstruct original NId.
846 if (NId < -1)
847 NId = -(NId + 1);
848
849 bool Found = false;
850 while (!Worklist.empty()) {
851 const SDNode *M = Worklist.pop_back_val();
852 int MId = M->getNodeId();
853 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
854 (MId > 0) && (MId < NId)) {
855 DeferredNodes.push_back(M);
856 continue;
857 }
858 for (const SDValue &OpV : M->op_values()) {
859 SDNode *Op = OpV.getNode();
860 if (Visited.insert(Op).second)
861 Worklist.push_back(Op);
862 if (Op == N)
863 Found = true;
864 }
865 if (Found)
866 break;
867 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
868 break;
869 }
870 // Push deferred nodes back on worklist.
871 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
872 // If we bailed early, conservatively return found.
873 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
874 return true;
875 return Found;
876 }
877
878 /// Return true if all the users of N are contained in Nodes.
879 /// NOTE: Requires at least one match, but doesn't require them all.
880 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
881
882 /// Return the number of values used by this operation.
883 unsigned getNumOperands() const { return NumOperands; }
884
885 /// Return the maximum number of operands that a SDNode can hold.
886 static constexpr size_t getMaxNumOperands() {
887 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
888 }
889
890 /// Helper method returns the integer value of a ConstantSDNode operand.
891 inline uint64_t getConstantOperandVal(unsigned Num) const;
892
893 /// Helper method returns the APInt of a ConstantSDNode operand.
894 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
895
896 const SDValue &getOperand(unsigned Num) const {
897 assert(Num < NumOperands && "Invalid child # of SDNode!")((void)0);
898 return OperandList[Num];
899 }
900
901 using op_iterator = SDUse *;
902
903 op_iterator op_begin() const { return OperandList; }
904 op_iterator op_end() const { return OperandList+NumOperands; }
905 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
906
907 /// Iterator for directly iterating over the operand SDValue's.
908 struct value_op_iterator
909 : iterator_adaptor_base<value_op_iterator, op_iterator,
910 std::random_access_iterator_tag, SDValue,
911 ptrdiff_t, value_op_iterator *,
912 value_op_iterator *> {
913 explicit value_op_iterator(SDUse *U = nullptr)
914 : iterator_adaptor_base(U) {}
915
916 const SDValue &operator*() const { return I->get(); }
917 };
918
919 iterator_range<value_op_iterator> op_values() const {
920 return make_range(value_op_iterator(op_begin()),
921 value_op_iterator(op_end()));
922 }
923
924 SDVTList getVTList() const {
925 SDVTList X = { ValueList, NumValues };
926 return X;
927 }
928
929 /// If this node has a glue operand, return the node
930 /// to which the glue operand points. Otherwise return NULL.
931 SDNode *getGluedNode() const {
932 if (getNumOperands() != 0 &&
933 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
934 return getOperand(getNumOperands()-1).getNode();
935 return nullptr;
936 }
937
938 /// If this node has a glue value with a user, return
939 /// the user (there is at most one). Otherwise return NULL.
940 SDNode *getGluedUser() const {
941 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
942 if (UI.getUse().get().getValueType() == MVT::Glue)
943 return *UI;
944 return nullptr;
945 }
946
947 SDNodeFlags getFlags() const { return Flags; }
948 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
949
950 /// Clear any flags in this node that aren't also set in Flags.
951 /// If Flags is not in a defined state then this has no effect.
952 void intersectFlagsWith(const SDNodeFlags Flags);
953
954 /// Return the number of values defined/returned by this operator.
955 unsigned getNumValues() const { return NumValues; }
956
957 /// Return the type of a specified result.
958 EVT getValueType(unsigned ResNo) const {
959 assert(ResNo < NumValues && "Illegal result number!")((void)0);
960 return ValueList[ResNo];
961 }
962
963 /// Return the type of a specified result as a simple type.
964 MVT getSimpleValueType(unsigned ResNo) const {
965 return getValueType(ResNo).getSimpleVT();
966 }
967
968 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
969 ///
970 /// If the value type is a scalable vector type, the scalable property will
971 /// be set and the runtime size will be a positive integer multiple of the
972 /// base size.
973 TypeSize getValueSizeInBits(unsigned ResNo) const {
974 return getValueType(ResNo).getSizeInBits();
975 }
976
977 using value_iterator = const EVT *;
978
979 value_iterator value_begin() const { return ValueList; }
980 value_iterator value_end() const { return ValueList+NumValues; }
981 iterator_range<value_iterator> values() const {
982 return llvm::make_range(value_begin(), value_end());
983 }
984
985 /// Return the opcode of this operation for printing.
986 std::string getOperationName(const SelectionDAG *G = nullptr) const;
987 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
988 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
989 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
990 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
991 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
992
993 /// Print a SelectionDAG node and all children down to
994 /// the leaves. The given SelectionDAG allows target-specific nodes
995 /// to be printed in human-readable form. Unlike printr, this will
996 /// print the whole DAG, including children that appear multiple
997 /// times.
998 ///
999 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and children up to
1002 /// depth "depth." The given SelectionDAG allows target-specific
1003 /// nodes to be printed in human-readable form. Unlike printr, this
1004 /// will print children that appear multiple times wherever they are
1005 /// used.
1006 ///
1007 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1008 unsigned depth = 100) const;
1009
1010 /// Dump this node, for debugging.
1011 void dump() const;
1012
1013 /// Dump (recursively) this node and its use-def subgraph.
1014 void dumpr() const;
1015
1016 /// Dump this node, for debugging.
1017 /// The given SelectionDAG allows target-specific nodes to be printed
1018 /// in human-readable form.
1019 void dump(const SelectionDAG *G) const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 /// The given SelectionDAG allows target-specific nodes to be printed
1023 /// in human-readable form.
1024 void dumpr(const SelectionDAG *G) const;
1025
1026 /// printrFull to dbgs(). The given SelectionDAG allows
1027 /// target-specific nodes to be printed in human-readable form.
1028 /// Unlike dumpr, this will print the whole DAG, including children
1029 /// that appear multiple times.
1030 void dumprFull(const SelectionDAG *G = nullptr) const;
1031
1032 /// printrWithDepth to dbgs(). The given
1033 /// SelectionDAG allows target-specific nodes to be printed in
1034 /// human-readable form. Unlike dumpr, this will print children
1035 /// that appear multiple times wherever they are used.
1036 ///
1037 void dumprWithDepth(const SelectionDAG *G = nullptr,
1038 unsigned depth = 100) const;
1039
1040 /// Gather unique data for the node.
1041 void Profile(FoldingSetNodeID &ID) const;
1042
1043 /// This method should only be used by the SDUse class.
1044 void addUse(SDUse &U) { U.addToList(&UseList); }
1045
1046protected:
1047 static SDVTList getSDVTList(EVT VT) {
1048 SDVTList Ret = { getValueTypeList(VT), 1 };
1049 return Ret;
1050 }
1051
1052 /// Create an SDNode.
1053 ///
1054 /// SDNodes are created without any operands, and never own the operand
1055 /// storage. To add operands, see SelectionDAG::createOperands.
1056 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1057 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1058 IROrder(Order), debugLoc(std::move(dl)) {
1059 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1060 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((void)0);
1061 assert(NumValues == VTs.NumVTs &&((void)0)
1062 "NumValues wasn't wide enough for its operands!")((void)0);
1063 }
1064
1065 /// Release the operands and set this node to have zero operands.
1066 void DropOperands();
1067};
1068
1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
1080 DebugLoc DL;
1081 int IROrder = 0;
1082
1083public:
1084 SDLoc() = default;
1085 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1086 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1087 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1088 assert(Order >= 0 && "bad IROrder")((void)0);
1089 if (I)
1090 DL = I->getDebugLoc();
1091 }
1092
1093 unsigned getIROrder() const { return IROrder; }
1094 const DebugLoc &getDebugLoc() const { return DL; }
1095};
1096
1097// Define inline functions from the SDValue class.
1098
1099inline SDValue::SDValue(SDNode *node, unsigned resno)
1100 : Node(node), ResNo(resno) {
1101 // Explicitly check for !ResNo to avoid use-after-free, because there are
1102 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1103 // combines.
1104 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&((void)0)
1105 "Invalid result number for the given node!")((void)0);
1106 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((void)0);
1107}
1108
1109inline unsigned SDValue::getOpcode() const {
1110 return Node->getOpcode();
20
←
Called C++ object pointer is null
1111}
1112
1113inline EVT SDValue::getValueType() const {
1114 return Node->getValueType(ResNo);
1115}
1116
1117inline unsigned SDValue::getNumOperands() const {
1118 return Node->getNumOperands();
1119}
1120
1121inline const SDValue &SDValue::getOperand(unsigned i) const {
1122 return Node->getOperand(i);
1123}
1124
1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1126 return Node->getConstantOperandVal(i);
1127}
1128
1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1130 return Node->getConstantOperandAPInt(i);
1131}
1132
1133inline bool SDValue::isTargetOpcode() const {
1134 return Node->isTargetOpcode();
1135}
1136
1137inline bool SDValue::isTargetMemoryOpcode() const {
1138 return Node->isTargetMemoryOpcode();
1139}
1140
1141inline bool SDValue::isMachineOpcode() const {
1142 return Node->isMachineOpcode();
1143}
1144
1145inline unsigned SDValue::getMachineOpcode() const {
1146 return Node->getMachineOpcode();
1147}
1148
1149inline bool SDValue::isUndef() const {
1150 return Node->isUndef();
1151}
1152
1153inline bool SDValue::use_empty() const {
1154 return !Node->hasAnyUseOfValue(ResNo);
1155}
1156
1157inline bool SDValue::hasOneUse() const {
1158 return Node->hasNUsesOfValue(1, ResNo);
1159}
1160
1161inline const DebugLoc &SDValue::getDebugLoc() const {
1162 return Node->getDebugLoc();
1163}
1164
1165inline void SDValue::dump() const {
1166 return Node->dump();
1167}
1168
1169inline void SDValue::dump(const SelectionDAG *G) const {
1170 return Node->dump(G);
1171}
1172
1173inline void SDValue::dumpr() const {
1174 return Node->dumpr();
1175}
1176
1177inline void SDValue::dumpr(const SelectionDAG *G) const {
1178 return Node->dumpr(G);
1179}
1180
1181// Define inline functions from the SDUse class.
1182
1183inline void SDUse::set(const SDValue &V) {
1184 if (Val.getNode()) removeFromList();
1185 Val = V;
1186 if (V.getNode()) V.getNode()->addUse(*this);
1187}
1188
1189inline void SDUse::setInitial(const SDValue &V) {
1190 Val = V;
1191 V.getNode()->addUse(*this);
1192}
1193
1194inline void SDUse::setNode(SDNode *N) {
1195 if (Val.getNode()) removeFromList();
1196 Val.setNode(N);
1197 if (N) N->addUse(*this);
1198}
1199
1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand. This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
1205 SDUse Op;
1206
1207public:
1208 explicit HandleSDNode(SDValue X)
1209 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1210 // HandleSDNodes are never inserted into the DAG, so they won't be
1211 // auto-numbered. Use ID 65535 as a sentinel.
1212 PersistentId = 0xffff;
1213
1214 // Manually set up the operand list. This node type is special in that it's
1215 // always stack allocated and SelectionDAG does not manage its operands.
1216 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1217 // be so special.
1218 Op.setUser(this);
1219 Op.setInitial(X);
1220 NumOperands = 1;
1221 OperandList = &Op;
1222 }
1223 ~HandleSDNode();
1224
1225 const SDValue &getValue() const { return Op; }
1226};
1227
1228class AddrSpaceCastSDNode : public SDNode {
1229private:
1230 unsigned SrcAddrSpace;
1231 unsigned DestAddrSpace;
1232
1233public:
1234 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1235 unsigned SrcAS, unsigned DestAS);
1236
1237 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1238 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1239
1240 static bool classof(const SDNode *N) {
1241 return N->getOpcode() == ISD::ADDRSPACECAST;
1242 }
1243};
1244
1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
1248 // VT of in-memory value.
1249 EVT MemoryVT;
1250
1251protected:
1252 /// Memory reference information.
1253 MachineMemOperand *MMO;
1254
1255public:
1256 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1257 EVT memvt, MachineMemOperand *MMO);
1258
1259 bool readMem() const { return MMO->isLoad(); }
1260 bool writeMem() const { return MMO->isStore(); }
1261
1262 /// Returns alignment and volatility of the memory access
1263 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1264 Align getAlign() const { return MMO->getAlign(); }
1265 // FIXME: Remove once transition to getAlign is over.
1266 unsigned getAlignment() const { return MMO->getAlign().value(); }
1267
1268 /// Return the SubclassData value, without HasDebugValue. This contains an
1269 /// encoding of the volatile flag, as well as bits used by subclasses. This
1270 /// function should only be used to compute a FoldingSetNodeID value.
1271 /// The HasDebugValue bit is masked out because CSE map needs to match
1272 /// nodes with debug info with nodes without debug info. Same is about
1273 /// isDivergent bit.
1274 unsigned getRawSubclassData() const {
1275 uint16_t Data;
1276 union {
1277 char RawSDNodeBits[sizeof(uint16_t)];
1278 SDNodeBitfields SDNodeBits;
1279 };
1280 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1281 SDNodeBits.HasDebugValue = 0;
1282 SDNodeBits.IsDivergent = false;
1283 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1284 return Data;
1285 }
1286
1287 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1288 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1289 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1290 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1291
1292 // Returns the offset from the location of the access.
1293 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1294
1295 /// Returns the AA info that describes the dereference.
1296 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1297
1298 /// Returns the Ranges that describes the dereference.
1299 const MDNode *getRanges() const { return MMO->getRanges(); }
1300
1301 /// Returns the synchronization scope ID for this memory operation.
1302 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1303
1304 /// Return the atomic ordering requirements for this memory operation. For
1305 /// cmpxchg atomic operations, return the atomic ordering requirements when
1306 /// store occurs.
1307 AtomicOrdering getSuccessOrdering() const {
1308 return MMO->getSuccessOrdering();
1309 }
1310
1311 /// Return a single atomic ordering that is at least as strong as both the
1312 /// success and failure orderings for an atomic operation. (For operations
1313 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1314 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1315
1316 /// Return true if the memory operation ordering is Unordered or higher.
1317 bool isAtomic() const { return MMO->isAtomic(); }
1318
1319 /// Returns true if the memory operation doesn't imply any ordering
1320 /// constraints on surrounding memory operations beyond the normal memory
1321 /// aliasing rules.
1322 bool isUnordered() const { return MMO->isUnordered(); }
1323
1324 /// Returns true if the memory operation is neither atomic or volatile.
1325 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1326
1327 /// Return the type of the in-memory value.
1328 EVT getMemoryVT() const { return MemoryVT; }
1329
1330 /// Return a MachineMemOperand object describing the memory
1331 /// reference performed by operation.
1332 MachineMemOperand *getMemOperand() const { return MMO; }
1333
1334 const MachinePointerInfo &getPointerInfo() const {
1335 return MMO->getPointerInfo();
1336 }
1337
1338 /// Return the address space for the associated pointer
1339 unsigned getAddressSpace() const {
1340 return getPointerInfo().getAddrSpace();
1341 }
1342
1343 /// Update this MemSDNode's MachineMemOperand information
1344 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1345 /// This must only be used when the new alignment applies to all users of
1346 /// this MachineMemOperand.
1347 void refineAlignment(const MachineMemOperand *NewMMO) {
1348 MMO->refineAlignment(NewMMO);
1349 }
1350
1351 const SDValue &getChain() const { return getOperand(0); }
1352
1353 const SDValue &getBasePtr() const {
1354 switch (getOpcode()) {
1355 case ISD::STORE:
1356 case ISD::MSTORE:
1357 return getOperand(2);
1358 case ISD::MGATHER:
1359 case ISD::MSCATTER:
1360 return getOperand(3);
1361 default:
1362 return getOperand(1);
1363 }
1364 }
1365
1366 // Methods to support isa and dyn_cast
1367 static bool classof(const SDNode *N) {
1368 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1369 // with either an intrinsic or a target opcode.
1370 switch (N->getOpcode()) {
1371 case ISD::LOAD:
1372 case ISD::STORE:
1373 case ISD::PREFETCH:
1374 case ISD::ATOMIC_CMP_SWAP:
1375 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1376 case ISD::ATOMIC_SWAP:
1377 case ISD::ATOMIC_LOAD_ADD:
1378 case ISD::ATOMIC_LOAD_SUB:
1379 case ISD::ATOMIC_LOAD_AND:
1380 case ISD::ATOMIC_LOAD_CLR:
1381 case ISD::ATOMIC_LOAD_OR:
1382 case ISD::ATOMIC_LOAD_XOR:
1383 case ISD::ATOMIC_LOAD_NAND:
1384 case ISD::ATOMIC_LOAD_MIN:
1385 case ISD::ATOMIC_LOAD_MAX:
1386 case ISD::ATOMIC_LOAD_UMIN:
1387 case ISD::ATOMIC_LOAD_UMAX:
1388 case ISD::ATOMIC_LOAD_FADD:
1389 case ISD::ATOMIC_LOAD_FSUB:
1390 case ISD::ATOMIC_LOAD:
1391 case ISD::ATOMIC_STORE:
1392 case ISD::MLOAD:
1393 case ISD::MSTORE:
1394 case ISD::MGATHER:
1395 case ISD::MSCATTER:
1396 return true;
1397 default:
1398 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1399 }
1400 }
1401};
1402
1403/// This is an SDNode representing atomic operations.
1404class AtomicSDNode : public MemSDNode {
1405public:
1406 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1407 EVT MemVT, MachineMemOperand *MMO)
1408 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1409 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||((void)0)
1410 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")((void)0);
1411 }
1412
1413 const SDValue &getBasePtr() const { return getOperand(1); }
1414 const SDValue &getVal() const { return getOperand(2); }
1415
1416 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1417 /// otherwise.
1418 bool isCompareAndSwap() const {
1419 unsigned Op = getOpcode();
1420 return Op == ISD::ATOMIC_CMP_SWAP ||
1421 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1422 }
1423
1424 /// For cmpxchg atomic operations, return the atomic ordering requirements
1425 /// when store does not occur.
1426 AtomicOrdering getFailureOrdering() const {
1427 assert(isCompareAndSwap() && "Must be cmpxchg operation")((void)0);
1428 return MMO->getFailureOrdering();
1429 }
1430
1431 // Methods to support isa and dyn_cast
1432 static bool classof(const SDNode *N) {
1433 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1434 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1435 N->getOpcode() == ISD::ATOMIC_SWAP ||
1436 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1437 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1438 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1439 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1440 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1441 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1442 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1443 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1444 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1445 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1446 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1447 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1448 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1449 N->getOpcode() == ISD::ATOMIC_LOAD ||
1450 N->getOpcode() == ISD::ATOMIC_STORE;
1451 }
1452};
1453
1454/// This SDNode is used for target intrinsics that touch
1455/// memory and need an associated MachineMemOperand. Its opcode may be
1456/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1457/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1458class MemIntrinsicSDNode : public MemSDNode {
1459public:
1460 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1461 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1462 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1463 SDNodeBits.IsMemIntrinsic = true;
1464 }
1465
1466 // Methods to support isa and dyn_cast
1467 static bool classof(const SDNode *N) {
1468 // We lower some target intrinsics to their target opcode
1469 // early a node with a target opcode can be of this class
1470 return N->isMemIntrinsic() ||
1471 N->getOpcode() == ISD::PREFETCH ||
1472 N->isTargetMemoryOpcode();
1473 }
1474};
1475
1476/// This SDNode is used to implement the code generator
1477/// support for the llvm IR shufflevector instruction. It combines elements
1478/// from two input vectors into a new input vector, with the selection and
1479/// ordering of elements determined by an array of integers, referred to as
1480/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1481/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1482/// An index of -1 is treated as undef, such that the code generator may put
1483/// any value in the corresponding element of the result.
1484class ShuffleVectorSDNode : public SDNode {
1485 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1486 // is freed when the SelectionDAG object is destroyed.
1487 const int *Mask;
1488
1489protected:
1490 friend class SelectionDAG;
1491
1492 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1493 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1494
1495public:
1496 ArrayRef<int> getMask() const {
1497 EVT VT = getValueType(0);
1498 return makeArrayRef(Mask, VT.getVectorNumElements());
1499 }
1500
1501 int getMaskElt(unsigned Idx) const {
1502 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")((void)0);
1503 return Mask[Idx];
1504 }
1505
1506 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1507
1508 int getSplatIndex() const {
1509 assert(isSplat() && "Cannot get splat index for non-splat!")((void)0);
1510 EVT VT = getValueType(0);
1511 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1512 if (Mask[i] >= 0)
1513 return Mask[i];
1514
1515 // We can choose any index value here and be correct because all elements
1516 // are undefined. Return 0 for better potential for callers to simplify.
1517 return 0;
1518 }
1519
1520 static bool isSplatMask(const int *Mask, EVT VT);
1521
1522 /// Change values in a shuffle permute mask assuming
1523 /// the two vector operands have swapped position.
1524 static void commuteMask(MutableArrayRef<int> Mask) {
1525 unsigned NumElems = Mask.size();
1526 for (unsigned i = 0; i != NumElems; ++i) {
1527 int idx = Mask[i];
1528 if (idx < 0)
1529 continue;
1530 else if (idx < (int)NumElems)
1531 Mask[i] = idx + NumElems;
1532 else
1533 Mask[i] = idx - NumElems;
1534 }
1535 }
1536
1537 static bool classof(const SDNode *N) {
1538 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1539 }
1540};
1541
1542class ConstantSDNode : public SDNode {
1543 friend class SelectionDAG;
1544
1545 const ConstantInt *Value;
1546
1547 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1548 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1549 getSDVTList(VT)),
1550 Value(val) {
1551 ConstantSDNodeBits.IsOpaque = isOpaque;
1552 }
1553
1554public:
1555 const ConstantInt *getConstantIntValue() const { return Value; }
1556 const APInt &getAPIntValue() const { return Value->getValue(); }
1557 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1558 int64_t getSExtValue() const { return Value->getSExtValue(); }
1559 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX0xffffffffffffffffULL) {
1560 return Value->getLimitedValue(Limit);
1561 }
1562 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1563 Align getAlignValue() const { return Value->getAlignValue(); }
1564
1565 bool isOne() const { return Value->isOne(); }
1566 bool isNullValue() const { return Value->isZero(); }
1567 bool isAllOnesValue() const { return Value->isMinusOne(); }
1568 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1569 bool isMinSignedValue() const { return Value->isMinValue(true); }
1570
1571 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1572
1573 static bool classof(const SDNode *N) {
1574 return N->getOpcode() == ISD::Constant ||
1575 N->getOpcode() == ISD::TargetConstant;
1576 }
1577};
1578
1579uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1580 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1581}
1582
1583const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1584 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1585}
1586
1587class ConstantFPSDNode : public SDNode {
1588 friend class SelectionDAG;
1589
1590 const ConstantFP *Value;
1591
1592 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1593 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1594 DebugLoc(), getSDVTList(VT)),
1595 Value(val) {}
1596
1597public:
1598 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1599 const ConstantFP *getConstantFPValue() const { return Value; }
1600
1601 /// Return true if the value is positive or negative zero.
1602 bool isZero() const { return Value->isZero(); }
1603
1604 /// Return true if the value is a NaN.
1605 bool isNaN() const { return Value->isNaN(); }
1606
1607 /// Return true if the value is an infinity
1608 bool isInfinity() const { return Value->isInfinity(); }
1609
1610 /// Return true if the value is negative.
1611 bool isNegative() const { return Value->isNegative(); }
1612
1613 /// We don't rely on operator== working on double values, as
1614 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1615 /// As such, this method can be used to do an exact bit-for-bit comparison of
1616 /// two floating point values.
1617
1618 /// We leave the version with the double argument here because it's just so
1619 /// convenient to write "2.0" and the like. Without this function we'd
1620 /// have to duplicate its logic everywhere it's called.
1621 bool isExactlyValue(double V) const {
1622 return Value->getValueAPF().isExactlyValue(V);
1623 }
1624 bool isExactlyValue(const APFloat& V) const;
1625
1626 static bool isValueValidForType(EVT VT, const APFloat& Val);
1627
1628 static bool classof(const SDNode *N) {
1629 return N->getOpcode() == ISD::ConstantFP ||
1630 N->getOpcode() == ISD::TargetConstantFP;
1631 }
1632};
1633
1634/// Returns true if \p V is a constant integer zero.
1635bool isNullConstant(SDValue V);
1636
1637/// Returns true if \p V is an FP constant with a value of positive zero.
1638bool isNullFPConstant(SDValue V);
1639
1640/// Returns true if \p V is an integer constant with all bits set.
1641bool isAllOnesConstant(SDValue V);
1642
1643/// Returns true if \p V is a constant integer one.
1644bool isOneConstant(SDValue V);
1645
1646/// Return the non-bitcasted source operand of \p V if it exists.
1647/// If \p V is not a bitcasted value, it is returned as-is.
1648SDValue peekThroughBitcasts(SDValue V);
1649
1650/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1651/// If \p V is not a bitcasted one-use value, it is returned as-is.
1652SDValue peekThroughOneUseBitcasts(SDValue V);
1653
1654/// Return the non-extracted vector source operand of \p V if it exists.
1655/// If \p V is not an extracted subvector, it is returned as-is.
1656SDValue peekThroughExtractSubvectors(SDValue V);
1657
1658/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1659/// constant is canonicalized to be operand 1.
1660bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1661
1662/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1663ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1664 bool AllowTruncation = false);
1665
1666/// Returns the SDNode if it is a demanded constant splat BuildVector or
1667/// constant int.
1668ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1669 bool AllowUndefs = false,
1670 bool AllowTruncation = false);
1671
1672/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1673ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1674
1675/// Returns the SDNode if it is a demanded constant splat BuildVector or
1676/// constant float.
1677ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1678 bool AllowUndefs = false);
1679
1680/// Return true if the value is a constant 0 integer or a splatted vector of
1681/// a constant 0 integer (with no undefs by default).
1682/// Build vector implicit truncation is not an issue for null values.
1683bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1684
1685/// Return true if the value is a constant 1 integer or a splatted vector of a
1686/// constant 1 integer (with no undefs).
1687/// Does not permit build vector implicit truncation.
1688bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1689
1690/// Return true if the value is a constant -1 integer or a splatted vector of a
1691/// constant -1 integer (with no undefs).
1692/// Does not permit build vector implicit truncation.
1693bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1694
1695/// Return true if \p V is either a integer or FP constant.
1696inline bool isIntOrFPConstant(SDValue V) {
1697 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1698}
1699
1700class GlobalAddressSDNode : public SDNode {
1701 friend class SelectionDAG;
1702
1703 const GlobalValue *TheGlobal;
1704 int64_t Offset;
1705 unsigned TargetFlags;
1706
1707 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1708 const GlobalValue *GA, EVT VT, int64_t o,
1709 unsigned TF);
1710
1711public:
1712 const GlobalValue *getGlobal() const { return TheGlobal; }
1713 int64_t getOffset() const { return Offset; }
1714 unsigned getTargetFlags() const { return TargetFlags; }
1715 // Return the address space this GlobalAddress belongs to.
1716 unsigned getAddressSpace() const;
1717
1718 static bool classof(const SDNode *N) {
1719 return N->getOpcode() == ISD::GlobalAddress ||
1720 N->getOpcode() == ISD::TargetGlobalAddress ||
1721 N->getOpcode() == ISD::GlobalTLSAddress ||
1722 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1723 }
1724};
1725
1726class FrameIndexSDNode : public SDNode {
1727 friend class SelectionDAG;
1728
1729 int FI;
1730
1731 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1732 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1733 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1734 }
1735
1736public:
1737 int getIndex() const { return FI; }
1738
1739 static bool classof(const SDNode *N) {
1740 return N->getOpcode() == ISD::FrameIndex ||
1741 N->getOpcode() == ISD::TargetFrameIndex;
1742 }
1743};
1744
1745/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1746/// the offet and size that are started/ended in the underlying FrameIndex.
1747class LifetimeSDNode : public SDNode {
1748 friend class SelectionDAG;
1749 int64_t Size;
1750 int64_t Offset; // -1 if offset is unknown.
1751
1752 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1753 SDVTList VTs, int64_t Size, int64_t Offset)
1754 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1755public:
1756 int64_t getFrameIndex() const {
1757 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1758 }
1759
1760 bool hasOffset() const { return Offset >= 0; }
1761 int64_t getOffset() const {
1762 assert(hasOffset() && "offset is unknown")((void)0);
1763 return Offset;
1764 }
1765 int64_t getSize() const {
1766 assert(hasOffset() && "offset is unknown")((void)0);
1767 return Size;
1768 }
1769
1770 // Methods to support isa and dyn_cast
1771 static bool classof(const SDNode *N) {
1772 return N->getOpcode() == ISD::LIFETIME_START ||
1773 N->getOpcode() == ISD::LIFETIME_END;
1774 }
1775};
1776
1777/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1778/// the index of the basic block being probed. A pseudo probe serves as a place
1779/// holder and will be removed at the end of compilation. It does not have any
1780/// operand because we do not want the instruction selection to deal with any.
1781class PseudoProbeSDNode : public SDNode {
1782 friend class SelectionDAG;
1783 uint64_t Guid;
1784 uint64_t Index;
1785 uint32_t Attributes;
1786
1787 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1788 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1789 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1790 Attributes(Attr) {}
1791
1792public:
1793 uint64_t getGuid() const { return Guid; }
1794 uint64_t getIndex() const { return Index; }
1795 uint32_t getAttributes() const { return Attributes; }
1796
1797 // Methods to support isa and dyn_cast
1798 static bool classof(const SDNode *N) {
1799 return N->getOpcode() == ISD::PSEUDO_PROBE;
1800 }
1801};
1802
1803class JumpTableSDNode : public SDNode {
1804 friend class SelectionDAG;
1805
1806 int JTI;
1807 unsigned TargetFlags;
1808
1809 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1810 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1811 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1812 }
1813
1814public:
1815 int getIndex() const { return JTI; }
1816 unsigned getTargetFlags() const { return TargetFlags; }
1817
1818 static bool classof(const SDNode *N) {
1819 return N->getOpcode() == ISD::JumpTable ||
1820 N->getOpcode() == ISD::TargetJumpTable;
1821 }
1822};
1823
1824class ConstantPoolSDNode : public SDNode {
1825 friend class SelectionDAG;
1826
1827 union {
1828 const Constant *ConstVal;
1829 MachineConstantPoolValue *MachineCPVal;
1830 } Val;
1831 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1832 Align Alignment; // Minimum alignment requirement of CP.
1833 unsigned TargetFlags;
1834
1835 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1836 Align Alignment, unsigned TF)
1837 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1838 DebugLoc(), getSDVTList(VT)),
1839 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1840 assert(Offset >= 0 && "Offset is too large")((void)0);
1841 Val.ConstVal = c;
1842 }
1843
1844 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1845 Align Alignment, unsigned TF)
1846 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1847 DebugLoc(), getSDVTList(VT)),
1848 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1849 assert(Offset >= 0 && "Offset is too large")((void)0);
1850 Val.MachineCPVal = v;
1851 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1852 }
1853
1854public:
1855 bool isMachineConstantPoolEntry() const {
1856 return Offset < 0;
1857 }
1858
1859 const Constant *getConstVal() const {
1860 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")((void)0);
1861 return Val.ConstVal;
1862 }
1863
1864 MachineConstantPoolValue *getMachineCPVal() const {
1865 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")((void)0);
1866 return Val.MachineCPVal;
1867 }
1868
1869 int getOffset() const {
1870 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1871 }
1872
1873 // Return the alignment of this constant pool object, which is either 0 (for
1874 // default alignment) or the desired value.
1875 Align getAlign() const { return Alignment; }
1876 unsigned getTargetFlags() const { return TargetFlags; }
1877
1878 Type *getType() const;
1879
1880 static bool classof(const SDNode *N) {
1881 return N->getOpcode() == ISD::ConstantPool ||
1882 N->getOpcode() == ISD::TargetConstantPool;
1883 }
1884};
1885
1886/// Completely target-dependent object reference.
1887class TargetIndexSDNode : public SDNode {
1888 friend class SelectionDAG;
1889
1890 unsigned TargetFlags;
1891 int Index;
1892 int64_t Offset;
1893
1894public:
1895 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1896 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1897 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1898
1899 unsigned getTargetFlags() const { return TargetFlags; }
1900 int getIndex() const { return Index; }
1901 int64_t getOffset() const { return Offset; }
1902
1903 static bool classof(const SDNode *N) {
1904 return N->getOpcode() == ISD::TargetIndex;
1905 }
1906};
1907
1908class BasicBlockSDNode : public SDNode {
1909 friend class SelectionDAG;
1910
1911 MachineBasicBlock *MBB;
1912
1913 /// Debug info is meaningful and potentially useful here, but we create
1914 /// blocks out of order when they're jumped to, which makes it a bit
1915 /// harder. Let's see if we need it first.
1916 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1917 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1918 {}
1919
1920public:
1921 MachineBasicBlock *getBasicBlock() const { return MBB; }
1922
1923 static bool classof(const SDNode *N) {
1924 return N->getOpcode() == ISD::BasicBlock;
1925 }
1926};
1927
1928/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1929class BuildVectorSDNode : public SDNode {
1930public:
1931 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1932 explicit BuildVectorSDNode() = delete;
1933
1934 /// Check if this is a constant splat, and if so, find the
1935 /// smallest element size that splats the vector. If MinSplatBits is
1936 /// nonzero, the element size must be at least that large. Note that the
1937 /// splat element may be the entire vector (i.e., a one element vector).
1938 /// Returns the splat element value in SplatValue. Any undefined bits in
1939 /// that value are zero, and the corresponding bits in the SplatUndef mask
1940 /// are set. The SplatBitSize value is set to the splat element size in
1941 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1942 /// undefined. isBigEndian describes the endianness of the target.
1943 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1944 unsigned &SplatBitSize, bool &HasAnyUndefs,
1945 unsigned MinSplatBits = 0,
1946 bool isBigEndian = false) const;
1947
1948 /// Returns the demanded splatted value or a null value if this is not a
1949 /// splat.
1950 ///
1951 /// The DemandedElts mask indicates the elements that must be in the splat.
1952 /// If passed a non-null UndefElements bitvector, it will resize it to match
1953 /// the vector width and set the bits where elements are undef.
1954 SDValue getSplatValue(const APInt &DemandedElts,
1955 BitVector *UndefElements = nullptr) const;
1956
1957 /// Returns the splatted value or a null value if this is not a splat.
1958 ///
1959 /// If passed a non-null UndefElements bitvector, it will resize it to match
1960 /// the vector width and set the bits where elements are undef.
1961 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1962
1963 /// Find the shortest repeating sequence of values in the build vector.
1964 ///
1965 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1966 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1967 ///
1968 /// Currently this must be a power-of-2 build vector.
1969 /// The DemandedElts mask indicates the elements that must be present,
1970 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1971 /// non-null UndefElements bitvector, it will resize it to match the original
1972 /// vector width and set the bits where elements are undef. If result is
1973 /// false, Sequence will be empty.
1974 bool getRepeatedSequence(const APInt &DemandedElts,
1975 SmallVectorImpl<SDValue> &Sequence,
1976 BitVector *UndefElements = nullptr) const;
1977
1978 /// Find the shortest repeating sequence of values in the build vector.
1979 ///
1980 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1981 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1982 ///
1983 /// Currently this must be a power-of-2 build vector.
1984 /// If passed a non-null UndefElements bitvector, it will resize it to match
1985 /// the original vector width and set the bits where elements are undef.
1986 /// If result is false, Sequence will be empty.
1987 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
1988 BitVector *UndefElements = nullptr) const;
1989
1990 /// Returns the demanded splatted constant or null if this is not a constant
1991 /// splat.
1992 ///
1993 /// The DemandedElts mask indicates the elements that must be in the splat.
1994 /// If passed a non-null UndefElements bitvector, it will resize it to match
1995 /// the vector width and set the bits where elements are undef.
1996 ConstantSDNode *
1997 getConstantSplatNode(const APInt &DemandedElts,
1998 BitVector *UndefElements = nullptr) const;
1999
2000 /// Returns the splatted constant or null if this is not a constant
2001 /// splat.
2002 ///
2003 /// If passed a non-null UndefElements bitvector, it will resize it to match
2004 /// the vector width and set the bits where elements are undef.
2005 ConstantSDNode *
2006 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2007
2008 /// Returns the demanded splatted constant FP or null if this is not a
2009 /// constant FP splat.
2010 ///
2011 /// The DemandedElts mask indicates the elements that must be in the splat.
2012 /// If passed a non-null UndefElements bitvector, it will resize it to match
2013 /// the vector width and set the bits where elements are undef.
2014 ConstantFPSDNode *
2015 getConstantFPSplatNode(const APInt &DemandedElts,
2016 BitVector *UndefElements = nullptr) const;
2017
2018 /// Returns the splatted constant FP or null if this is not a constant
2019 /// FP splat.
2020 ///
2021 /// If passed a non-null UndefElements bitvector, it will resize it to match
2022 /// the vector width and set the bits where elements are undef.
2023 ConstantFPSDNode *
2024 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2025
2026 /// If this is a constant FP splat and the splatted constant FP is an
2027 /// exact power or 2, return the log base 2 integer value. Otherwise,
2028 /// return -1.
2029 ///
2030 /// The BitWidth specifies the necessary bit precision.
2031 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2032 uint32_t BitWidth) const;
2033
2034 bool isConstant() const;
2035
2036 static bool classof(const SDNode *N) {
2037 return N->getOpcode() == ISD::BUILD_VECTOR;
2038 }
2039};
2040
2041/// An SDNode that holds an arbitrary LLVM IR Value. This is
2042/// used when the SelectionDAG needs to make a simple reference to something
2043/// in the LLVM IR representation.
2044///
2045class SrcValueSDNode : public SDNode {
2046 friend class SelectionDAG;
2047
2048 const Value *V;
2049
2050 /// Create a SrcValue for a general value.
2051 explicit SrcValueSDNode(const Value *v)
2052 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2053
2054public:
2055 /// Return the contained Value.
2056 const Value *getValue() const { return V; }
2057
2058 static bool classof(const SDNode *N) {
2059 return N->getOpcode() == ISD::SRCVALUE;
2060 }
2061};
2062
2063class MDNodeSDNode : public SDNode {
2064 friend class SelectionDAG;
2065
2066 const MDNode *MD;
2067
2068 explicit MDNodeSDNode(const MDNode *md)
2069 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2070 {}
2071
2072public:
2073 const MDNode *getMD() const { return MD; }
2074
2075 static bool classof(const SDNode *N) {
2076 return N->getOpcode() == ISD::MDNODE_SDNODE;
2077 }
2078};
2079
2080class RegisterSDNode : public SDNode {
2081 friend class SelectionDAG;
2082
2083 Register Reg;
2084
2085 RegisterSDNode(Register reg, EVT VT)
2086 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2087
2088public:
2089 Register getReg() const { return Reg; }
2090
2091 static bool classof(const SDNode *N) {
2092 return N->getOpcode() == ISD::Register;
2093 }
2094};
2095
2096class RegisterMaskSDNode : public SDNode {
2097 friend class SelectionDAG;
2098
2099 // The memory for RegMask is not owned by the node.
2100 const uint32_t *RegMask;
2101
2102 RegisterMaskSDNode(const uint32_t *mask)
2103 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2104 RegMask(mask) {}
2105
2106public:
2107 const uint32_t *getRegMask() const { return RegMask; }
2108
2109 static bool classof(const SDNode *N) {
2110 return N->getOpcode() == ISD::RegisterMask;
2111 }
2112};
2113
2114class BlockAddressSDNode : public SDNode {
2115 friend class SelectionDAG;
2116
2117 const BlockAddress *BA;
2118 int64_t Offset;
2119 unsigned TargetFlags;
2120
2121 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2122 int64_t o, unsigned Flags)
2123 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2124 BA(ba), Offset(o), TargetFlags(Flags) {}
2125
2126public:
2127 const BlockAddress *getBlockAddress() const { return BA; }
2128 int64_t getOffset() const { return Offset; }
2129 unsigned getTargetFlags() const { return TargetFlags; }
2130
2131 static bool classof(const SDNode *N) {
2132 return N->getOpcode() == ISD::BlockAddress ||
2133 N->getOpcode() == ISD::TargetBlockAddress;
2134 }
2135};
2136
2137class LabelSDNode : public SDNode {
2138 friend class SelectionDAG;
2139
2140 MCSymbol *Label;
2141
2142 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2143 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2144 assert(LabelSDNode::classof(this) && "not a label opcode")((void)0);
2145 }
2146
2147public:
2148 MCSymbol *getLabel() const { return Label; }
2149
2150 static bool classof(const SDNode *N) {
2151 return N->getOpcode() == ISD::EH_LABEL ||
2152 N->getOpcode() == ISD::ANNOTATION_LABEL;
2153 }
2154};
2155
2156class ExternalSymbolSDNode : public SDNode {
2157 friend class SelectionDAG;
2158
2159 const char *Symbol;
2160 unsigned TargetFlags;
2161
2162 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2163 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2164 DebugLoc(), getSDVTList(VT)),
2165 Symbol(Sym), TargetFlags(TF) {}
2166
2167public:
2168 const char *getSymbol() const { return Symbol; }
2169 unsigned getTargetFlags() const { return TargetFlags; }
2170
2171 static bool classof(const SDNode *N) {
2172 return N->getOpcode() == ISD::ExternalSymbol ||
2173 N->getOpcode() == ISD::TargetExternalSymbol;
2174 }
2175};
2176
2177class MCSymbolSDNode : public SDNode {
2178 friend class SelectionDAG;
2179
2180 MCSymbol *Symbol;
2181
2182 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2183 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2184
2185public:
2186 MCSymbol *getMCSymbol() const { return Symbol; }
2187
2188 static bool classof(const SDNode *N) {
2189 return N->getOpcode() == ISD::MCSymbol;
2190 }
2191};
2192
2193class CondCodeSDNode : public SDNode {
2194 friend class SelectionDAG;
2195
2196 ISD::CondCode Condition;
2197
2198 explicit CondCodeSDNode(ISD::CondCode Cond)
2199 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2200 Condition(Cond) {}
2201
2202public:
2203 ISD::CondCode get() const { return Condition; }
2204
2205 static bool classof(const SDNode *N) {
2206 return N->getOpcode() == ISD::CONDCODE;
2207 }
2208};
2209
2210/// This class is used to represent EVT's, which are used
2211/// to parameterize some operations.
2212class VTSDNode : public SDNode {
2213 friend class SelectionDAG;
2214
2215 EVT ValueType;
2216
2217 explicit VTSDNode(EVT VT)
2218 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2219 ValueType(VT) {}
2220
2221public:
2222 EVT getVT() const { return ValueType; }
2223
2224 static bool classof(const SDNode *N) {
2225 return N->getOpcode() == ISD::VALUETYPE;
2226 }
2227};
2228
2229/// Base class for LoadSDNode and StoreSDNode
2230class LSBaseSDNode : public MemSDNode {
2231public:
2232 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2233 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2234 MachineMemOperand *MMO)
2235 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2236 LSBaseSDNodeBits.AddressingMode = AM;
2237 assert(getAddressingMode() == AM && "Value truncated")((void)0);
2238 }
2239
2240 const SDValue &getOffset() const {
2241 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2242 }
2243
2244 /// Return the addressing mode for this load or store:
2245 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2246 ISD::MemIndexedMode getAddressingMode() const {
2247 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2248 }
2249
2250 /// Return true if this is a pre/post inc/dec load/store.
2251 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2252
2253 /// Return true if this is NOT a pre/post inc/dec load/store.
2254 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2255
2256 static bool classof(const SDNode *N) {
2257 return N->getOpcode() == ISD::LOAD ||
2258 N->getOpcode() == ISD::STORE;
2259 }
2260};
2261
2262/// This class is used to represent ISD::LOAD nodes.
2263class LoadSDNode : public LSBaseSDNode {
2264 friend class SelectionDAG;
2265
2266 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2267 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2268 MachineMemOperand *MMO)
2269 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2270 LoadSDNodeBits.ExtTy = ETy;
2271 assert(readMem() && "Load MachineMemOperand is not a load!")((void)0);
2272 assert(!writeMem() && "Load MachineMemOperand is a store!")((void)0);
2273 }
2274
2275public:
2276 /// Return whether this is a plain node,
2277 /// or one of the varieties of value-extending loads.
2278 ISD::LoadExtType getExtensionType() const {
2279 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2280 }
2281
2282 const SDValue &getBasePtr() const { return getOperand(1); }
2283 const SDValue &getOffset() const { return getOperand(2); }
2284
2285 static bool classof(const SDNode *N) {
2286 return N->getOpcode() == ISD::LOAD;
2287 }
2288};
2289
2290/// This class is used to represent ISD::STORE nodes.
2291class StoreSDNode : public LSBaseSDNode {
2292 friend class SelectionDAG;
2293
2294 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2295 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2296 MachineMemOperand *MMO)
2297 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2298 StoreSDNodeBits.IsTruncating = isTrunc;
2299 assert(!readMem() && "Store MachineMemOperand is a load!")((void)0);
2300 assert(writeMem() && "Store MachineMemOperand is not a store!")((void)0);
2301 }
2302
2303public:
2304 /// Return true if the op does a truncation before store.
2305 /// For integers this is the same as doing a TRUNCATE and storing the result.
2306 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2307 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2308 void setTruncatingStore(bool Truncating) {
2309 StoreSDNodeBits.IsTruncating = Truncating;
2310 }
2311
2312 const SDValue &getValue() const { return getOperand(1); }
2313 const SDValue &getBasePtr() const { return getOperand(2); }
2314 const SDValue &getOffset() const { return getOperand(3); }
2315
2316 static bool classof(const SDNode *N) {
2317 return N->getOpcode() == ISD::STORE;
2318 }
2319};
2320
2321/// This base class is used to represent MLOAD and MSTORE nodes
2322class MaskedLoadStoreSDNode : public MemSDNode {
2323public:
2324 friend class SelectionDAG;
2325
2326 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2327 const DebugLoc &dl, SDVTList VTs,
2328 ISD::MemIndexedMode AM, EVT MemVT,
2329 MachineMemOperand *MMO)
2330 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2331 LSBaseSDNodeBits.AddressingMode = AM;
2332 assert(getAddressingMode() == AM && "Value truncated")((void)0);
2333 }
2334
2335 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2336 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2337 // Mask is a vector of i1 elements
2338 const SDValue &getOffset() const {
2339 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2340 }
2341 const SDValue &getMask() const {
2342 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2343 }
2344
2345 /// Return the addressing mode for this load or store:
2346 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2347 ISD::MemIndexedMode getAddressingMode() const {
2348 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2349 }
2350
2351 /// Return true if this is a pre/post inc/dec load/store.
2352 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2353
2354 /// Return true if this is NOT a pre/post inc/dec load/store.
2355 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2356
2357 static bool classof(const SDNode *N) {
2358 return N->getOpcode() == ISD::MLOAD ||
2359 N->getOpcode() == ISD::MSTORE;
2360 }
2361};
2362
2363/// This class is used to represent an MLOAD node
2364class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2365public:
2366 friend class SelectionDAG;
2367
2368 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2369 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2370 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2371 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2372 LoadSDNodeBits.ExtTy = ETy;
2373 LoadSDNodeBits.IsExpanding = IsExpanding;
2374 }
2375
2376 ISD::LoadExtType getExtensionType() const {
2377 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2378 }
2379
2380 const SDValue &getBasePtr() const { return getOperand(1); }
2381 const SDValue &getOffset() const { return getOperand(2); }
2382 const SDValue &getMask() const { return getOperand(3); }
2383 const SDValue &getPassThru() const { return getOperand(4); }
2384
2385 static bool classof(const SDNode *N) {
2386 return N->getOpcode() == ISD::MLOAD;
2387 }
2388
2389 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2390};
2391
2392/// This class is used to represent an MSTORE node
2393class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2394public:
2395 friend class SelectionDAG;
2396
2397 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2398 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2399 EVT MemVT, MachineMemOperand *MMO)
2400 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2401 StoreSDNodeBits.IsTruncating = isTrunc;
2402 StoreSDNodeBits.IsCompressing = isCompressing;
2403 }
2404
2405 /// Return true if the op does a truncation before store.
2406 /// For integers this is the same as doing a TRUNCATE and storing the result.
2407 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2408 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2409
2410 /// Returns true if the op does a compression to the vector before storing.
2411 /// The node contiguously stores the active elements (integers or floats)
2412 /// in src (those with their respective bit set in writemask k) to unaligned
2413 /// memory at base_addr.
2414 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2415
2416 const SDValue &getValue() const { return getOperand(1); }
2417 const SDValue &getBasePtr() const { return getOperand(2); }
2418 const SDValue &getOffset() const { return getOperand(3); }
2419 const SDValue &getMask() const { return getOperand(4); }
2420
2421 static bool classof(const SDNode *N) {
2422 return N->getOpcode() == ISD::MSTORE;
2423 }
2424};
2425
2426/// This is a base class used to represent
2427/// MGATHER and MSCATTER nodes
2428///
2429class MaskedGatherScatterSDNode : public MemSDNode {
2430public:
2431 friend class SelectionDAG;
2432
2433 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2434 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2435 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2436 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2437 LSBaseSDNodeBits.AddressingMode = IndexType;
2438 assert(getIndexType() == IndexType && "Value truncated")((void)0);
2439 }
2440
2441 /// How is Index applied to BasePtr when computing addresses.
2442 ISD::MemIndexType getIndexType() const {
2443 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2444 }
2445 void setIndexType(ISD::MemIndexType IndexType) {
2446 LSBaseSDNodeBits.AddressingMode = IndexType;
2447 }
2448 bool isIndexScaled() const {
2449 return (getIndexType() == ISD::SIGNED_SCALED) ||
2450 (getIndexType() == ISD::UNSIGNED_SCALED);
2451 }
2452 bool isIndexSigned() const {
2453 return (getIndexType() == ISD::SIGNED_SCALED) ||
2454 (getIndexType() == ISD::SIGNED_UNSCALED);
2455 }
2456
2457 // In the both nodes address is Op1, mask is Op2:
2458 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2459 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2460 // Mask is a vector of i1 elements
2461 const SDValue &getBasePtr() const { return getOperand(3); }
2462 const SDValue &getIndex() const { return getOperand(4); }
2463 const SDValue &getMask() const { return getOperand(2); }
2464 const SDValue &getScale() const { return getOperand(5); }
2465
2466 static bool classof(const SDNode *N) {
2467 return N->getOpcode() == ISD::MGATHER ||
2468 N->getOpcode() == ISD::MSCATTER;
2469 }
2470};
2471
2472/// This class is used to represent an MGATHER node
2473///
2474class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2475public:
2476 friend class SelectionDAG;
2477
2478 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2479 EVT MemVT, MachineMemOperand *MMO,
2480 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2481 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2482 IndexType) {
2483 LoadSDNodeBits.ExtTy = ETy;
2484 }
2485
2486 const SDValue &getPassThru() const { return getOperand(1); }
2487
2488 ISD::LoadExtType getExtensionType() const {
2489 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2490 }
2491
2492 static bool classof(const SDNode *N) {
2493 return N->getOpcode() == ISD::MGATHER;
2494 }
2495};
2496
2497/// This class is used to represent an MSCATTER node
2498///
2499class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2500public:
2501 friend class SelectionDAG;
2502
2503 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2504 EVT MemVT, MachineMemOperand *MMO,
2505 ISD::MemIndexType IndexType, bool IsTrunc)
2506 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2507 IndexType) {
2508 StoreSDNodeBits.IsTruncating = IsTrunc;
2509 }
2510
2511 /// Return true if the op does a truncation before store.
2512 /// For integers this is the same as doing a TRUNCATE and storing the result.
2513 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2514 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2515
2516 const SDValue &getValue() const { return getOperand(1); }
2517
2518 static bool classof(const SDNode *N) {
2519 return N->getOpcode() == ISD::MSCATTER;
2520 }
2521};
2522
2523/// An SDNode that represents everything that will be needed
2524/// to construct a MachineInstr. These nodes are created during the
2525/// instruction selection proper phase.
2526///
2527/// Note that the only supported way to set the `memoperands` is by calling the
2528/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2529/// inside the DAG rather than in the node.
2530class MachineSDNode : public SDNode {
2531private:
2532 friend class SelectionDAG;
2533
2534 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2535 : SDNode(Opc, Order, DL, VTs) {}
2536
2537 // We use a pointer union between a single `MachineMemOperand` pointer and
2538 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2539 // the number of these is zero, the single pointer variant used when the
2540 // number is one, and the array is used for larger numbers.
2541 //
2542 // The array is allocated via the `SelectionDAG`'s allocator and so will
2543 // always live until the DAG is cleaned up and doesn't require ownership here.
2544 //
2545 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2546 // subclasses aren't managed in a conforming C++ manner. See the comments on
2547 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2548 // constraint here is that these don't manage memory with their constructor or
2549 // destructor and can be initialized to a good state even if they start off
2550 // uninitialized.
2551 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2552
2553 // Note that this could be folded into the above `MemRefs` member if doing so
2554 // is advantageous at some point. We don't need to store this in most cases.
2555 // However, at the moment this doesn't appear to make the allocation any
2556 // smaller and makes the code somewhat simpler to read.
2557 int NumMemRefs = 0;
2558
2559public:
2560 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2561
2562 ArrayRef<MachineMemOperand *> memoperands() const {
2563 // Special case the common cases.
2564 if (NumMemRefs == 0)
2565 return {};
2566 if (NumMemRefs == 1)
2567 return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);
2568
2569 // Otherwise we have an actual array.
2570 return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
2571 }
2572 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2573 mmo_iterator memoperands_end() const { return memoperands().end(); }
2574 bool memoperands_empty() const { return memoperands().empty(); }
2575
2576 /// Clear out the memory reference descriptor list.
2577 void clearMemRefs() {
2578 MemRefs = nullptr;
2579 NumMemRefs = 0;
2580 }
2581
2582 static bool classof(const SDNode *N) {
2583 return N->isMachineOpcode();
2584 }
2585};
2586
2587/// An SDNode that records if a register contains a value that is guaranteed to
2588/// be aligned accordingly.
2589class AssertAlignSDNode : public SDNode {
2590 Align Alignment;
2591
2592public:
2593 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
2594 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
2595
2596 Align getAlign() const { return Alignment; }
2597
2598 static bool classof(const SDNode *N) {
2599 return N->getOpcode() == ISD::AssertAlign;
2600 }
2601};
2602
2603class SDNodeIterator {
2604 const SDNode *Node;
2605 unsigned Operand;
2606
2607 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
2608
2609public:
2610 using iterator_category = std::forward_iterator_tag;
2611 using value_type = SDNode;
2612 using difference_type = std::ptrdiff_t;
2613 using pointer = value_type *;
2614 using reference = value_type &;
2615
2616 bool operator==(const SDNodeIterator& x) const {
2617 return Operand == x.Operand;
2618 }
2619 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
2620
2621 pointer operator*() const {
2622 return Node->getOperand(Operand).getNode();
2623 }
2624 pointer operator->() const { return operator*(); }
2625
2626 SDNodeIterator& operator++() { // Preincrement
2627 ++Operand;
2628 return *this;
2629 }
2630 SDNodeIterator operator++(int) { // Postincrement
2631 SDNodeIterator tmp = *this; ++*this; return tmp;
2632 }
2633 size_t operator-(SDNodeIterator Other) const {
2634 assert(Node == Other.Node &&((void)0)
2635 "Cannot compare iterators of two different nodes!")((void)0);
2636 return Operand - Other.Operand;
2637 }
2638
2639 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
2640 static SDNodeIterator end (const SDNode *N) {
2641 return SDNodeIterator(N, N->getNumOperands());
2642 }
2643
2644 unsigned getOperand() const { return Operand; }
2645 const SDNode *getNode() const { return Node; }
2646};
2647
2648template <> struct GraphTraits<SDNode*> {
2649 using NodeRef = SDNode *;
2650 using ChildIteratorType = SDNodeIterator;
2651
2652 static NodeRef getEntryNode(SDNode *N) { return N; }
2653
2654 static ChildIteratorType child_begin(NodeRef N) {
2655 return SDNodeIterator::begin(N);
2656 }
2657
2658 static ChildIteratorType child_end(NodeRef N) {
2659 return SDNodeIterator::end(N);
2660 }
2661};
2662
2663/// A representation of the largest SDNode, for use in sizeof().
2664///
2665/// This needs to be a union because the largest node differs on 32 bit systems
2666/// with 4 and 8 byte pointer alignment, respectively.
2667using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
2668 BlockAddressSDNode,
2669 GlobalAddressSDNode,
2670 PseudoProbeSDNode>;
2671
2672/// The SDNode class with the greatest alignment requirement.
2673using MostAlignedSDNode = GlobalAddressSDNode;
2674
2675namespace ISD {
2676
2677 /// Returns true if the specified node is a non-extending and unindexed load.
2678 inline bool isNormalLoad(const SDNode *N) {
2679 const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
2680 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
2681 Ld->getAddressingMode() == ISD::UNINDEXED;
2682 }
2683
2684 /// Returns true if the specified node is a non-extending load.
2685 inline bool isNON_EXTLoad(const SDNode *N) {
2686 return isa<LoadSDNode>(N) &&
2687 cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
2688 }
2689
2690 /// Returns true if the specified node is a EXTLOAD.
2691 inline bool isEXTLoad(const SDNode *N) {
2692 return isa<LoadSDNode>(N) &&
2693 cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
2694 }
2695
2696 /// Returns true if the specified node is a SEXTLOAD.
2697 inline bool isSEXTLoad(const SDNode *N) {
2698 return isa<LoadSDNode>(N) &&
2699 cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
2700 }
2701
2702 /// Returns true if the specified node is a ZEXTLOAD.
2703 inline bool isZEXTLoad(const SDNode *N) {
2704 return isa<LoadSDNode>(N) &&
2705 cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
2706 }
2707
2708 /// Returns true if the specified node is an unindexed load.
2709 inline bool isUNINDEXEDLoad(const SDNode *N) {
2710 return isa<LoadSDNode>(N) &&
2711 cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2712 }
2713
2714 /// Returns true if the specified node is a non-truncating
2715 /// and unindexed store.
2716 inline bool isNormalStore(const SDNode *N) {
2717 const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
2718 return St && !St->isTruncatingStore() &&
2719 St->getAddressingMode() == ISD::UNINDEXED;
2720 }
2721
2722 /// Returns true if the specified node is an unindexed store.
2723 inline bool isUNINDEXEDStore(const SDNode *N) {
2724 return isa<StoreSDNode>(N) &&
2725 cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2726 }
2727
2728 /// Attempt to match a unary predicate against a scalar/splat constant or
2729 /// every element of a constant BUILD_VECTOR.
2730 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2731 bool matchUnaryPredicate(SDValue Op,
2732 std::function<bool(ConstantSDNode *)> Match,
2733 bool AllowUndefs = false);
2734
2735 /// Attempt to match a binary predicate against a pair of scalar/splat
2736 /// constants or every element of a pair of constant BUILD_VECTORs.
2737 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2738 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
2739 bool matchBinaryPredicate(
2740 SDValue LHS, SDValue RHS,
2741 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
2742 bool AllowUndefs = false, bool AllowTypeMismatch = false);
2743
2744 /// Returns true if the specified value is the overflow result from one
2745 /// of the overflow intrinsic nodes.
2746 inline bool isOverflowIntrOpRes(SDValue Op) {
2747 unsigned Opc = Op.getOpcode();
2748 return (Op.getResNo() == 1 &&
2749 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2750 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2751 }
2752
2753} // end namespace ISD
2754
2755} // end namespace llvm
2756
2757#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H