Bug Summary

File:src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1110, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

→
1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Target/TargetLoweringObjectFile.h"
33#include "llvm/Target/TargetMachine.h"
34#include <cctype>
35using namespace llvm;
36
37/// NOTE: The TargetMachine owns TLOF.
38TargetLowering::TargetLowering(const TargetMachine &tm)
39 : TargetLoweringBase(tm) {}
40
41const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
42 return nullptr;
43}
44
45bool TargetLowering::isPositionIndependent() const {
46 return getTargetMachine().isPositionIndependent();
47}
48
49/// Check whether a given call node is in tail position within its function. If
50/// so, it sets Chain to the input chain of the tail call.
51bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
52 SDValue &Chain) const {
53 const Function &F = DAG.getMachineFunction().getFunction();
54
55 // First, check if tail calls have been disabled in this function.
56 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
57 return false;
58
59 // Conservatively require the attributes of the call to match those of
60 // the return. Ignore following attributes because they don't affect the
61 // call sequence.
62 AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
63 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
64 Attribute::DereferenceableOrNull, Attribute::NoAlias,
65 Attribute::NonNull})
66 CallerAttrs.removeAttribute(Attr);
67
68 if (CallerAttrs.hasAttributes())
69 return false;
70
71 // It's not safe to eliminate the sign / zero extension of the return value.
72 if (CallerAttrs.contains(Attribute::ZExt) ||
73 CallerAttrs.contains(Attribute::SExt))
74 return false;
75
76 // Check if the only use is a function return node.
77 return isUsedByReturnOnly(Node, Chain);
78}
79
80bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
81 const uint32_t *CallerPreservedMask,
82 const SmallVectorImpl<CCValAssign> &ArgLocs,
83 const SmallVectorImpl<SDValue> &OutVals) const {
84 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
85 const CCValAssign &ArgLoc = ArgLocs[I];
86 if (!ArgLoc.isRegLoc())
87 continue;
88 MCRegister Reg = ArgLoc.getLocReg();
89 // Only look at callee saved registers.
90 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
91 continue;
92 // Check that we pass the value used for the caller.
93 // (We look for a CopyFromReg reading a virtual register that is used
94 // for the function live-in value of register Reg)
95 SDValue Value = OutVals[I];
96 if (Value->getOpcode() != ISD::CopyFromReg)
97 return false;
98 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
99 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
100 return false;
101 }
102 return true;
103}
104
105/// Set CallLoweringInfo attribute flags based on a call instruction
106/// and called function attributes.
107void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
108 unsigned ArgIdx) {
109 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
110 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
111 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
112 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
113 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
114 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
115 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
116 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
117 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
118 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
119 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
120 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
121 Alignment = Call->getParamStackAlign(ArgIdx);
122 IndirectType = nullptr;
123 assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&((void)0)
124 "multiple ABI attributes?")((void)0);
125 if (IsByVal) {
126 IndirectType = Call->getParamByValType(ArgIdx);
127 if (!Alignment)
128 Alignment = Call->getParamAlign(ArgIdx);
129 }
130 if (IsPreallocated)
131 IndirectType = Call->getParamPreallocatedType(ArgIdx);
132 if (IsInAlloca)
133 IndirectType = Call->getParamInAllocaType(ArgIdx);
134}
135
136/// Generate a libcall taking the given operands as arguments and returning a
137/// result of type RetVT.
138std::pair<SDValue, SDValue>
139TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
140 ArrayRef<SDValue> Ops,
141 MakeLibCallOptions CallOptions,
142 const SDLoc &dl,
143 SDValue InChain) const {
144 if (!InChain)
145 InChain = DAG.getEntryNode();
146
147 TargetLowering::ArgListTy Args;
148 Args.reserve(Ops.size());
149
150 TargetLowering::ArgListEntry Entry;
151 for (unsigned i = 0; i < Ops.size(); ++i) {
152 SDValue NewOp = Ops[i];
153 Entry.Node = NewOp;
154 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
155 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
156 CallOptions.IsSExt);
157 Entry.IsZExt = !Entry.IsSExt;
158
159 if (CallOptions.IsSoften &&
160 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
161 Entry.IsSExt = Entry.IsZExt = false;
162 }
163 Args.push_back(Entry);
164 }
165
166 if (LC == RTLIB::UNKNOWN_LIBCALL)
167 report_fatal_error("Unsupported library call operation!");
168 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
169 getPointerTy(DAG.getDataLayout()));
170
171 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
172 TargetLowering::CallLoweringInfo CLI(DAG);
173 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
174 bool zeroExtend = !signExtend;
175
176 if (CallOptions.IsSoften &&
177 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
178 signExtend = zeroExtend = false;
179 }
180
181 CLI.setDebugLoc(dl)
182 .setChain(InChain)
183 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
184 .setNoReturn(CallOptions.DoesNotReturn)
185 .setDiscardResult(!CallOptions.IsReturnValueUsed)
186 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
187 .setSExtResult(signExtend)
188 .setZExtResult(zeroExtend);
189 return LowerCallTo(CLI);
190}
191
192bool TargetLowering::findOptimalMemOpLowering(
193 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
194 unsigned SrcAS, const AttributeList &FuncAttributes) const {
195 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
196 return false;
197
198 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
199
200 if (VT == MVT::Other) {
201 // Use the largest integer type whose alignment constraints are satisfied.
202 // We only need to check DstAlign here as SrcAlign is always greater or
203 // equal to DstAlign (or zero).
204 VT = MVT::i64;
205 if (Op.isFixedDstAlign())
206 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
207 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
208 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
209 assert(VT.isInteger())((void)0);
210
211 // Find the largest legal integer type.
212 MVT LVT = MVT::i64;
213 while (!isTypeLegal(LVT))
214 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
215 assert(LVT.isInteger())((void)0);
216
217 // If the type we've chosen is larger than the largest legal integer type
218 // then use that instead.
219 if (VT.bitsGT(LVT))
220 VT = LVT;
221 }
222
223 unsigned NumMemOps = 0;
224 uint64_t Size = Op.size();
225 while (Size) {
226 unsigned VTSize = VT.getSizeInBits() / 8;
227 while (VTSize > Size) {
228 // For now, only use non-vector load / store's for the left-over pieces.
229 EVT NewVT = VT;
230 unsigned NewVTSize;
231
232 bool Found = false;
233 if (VT.isVector() || VT.isFloatingPoint()) {
234 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
235 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
236 isSafeMemOpType(NewVT.getSimpleVT()))
237 Found = true;
238 else if (NewVT == MVT::i64 &&
239 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
240 isSafeMemOpType(MVT::f64)) {
241 // i64 is usually not legal on 32-bit targets, but f64 may be.
242 NewVT = MVT::f64;
243 Found = true;
244 }
245 }
246
247 if (!Found) {
248 do {
249 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
250 if (NewVT == MVT::i8)
251 break;
252 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
253 }
254 NewVTSize = NewVT.getSizeInBits() / 8;
255
256 // If the new VT cannot cover all of the remaining bits, then consider
257 // issuing a (or a pair of) unaligned and overlapping load / store.
258 bool Fast;
259 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
260 allowsMisalignedMemoryAccesses(
261 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
262 MachineMemOperand::MONone, &Fast) &&
263 Fast)
264 VTSize = Size;
265 else {
266 VT = NewVT;
267 VTSize = NewVTSize;
268 }
269 }
270
271 if (++NumMemOps > Limit)
272 return false;
273
274 MemOps.push_back(VT);
275 Size -= VTSize;
276 }
277
278 return true;
279}
280
281/// Soften the operands of a comparison. This code is shared among BR_CC,
282/// SELECT_CC, and SETCC handlers.
283void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
284 SDValue &NewLHS, SDValue &NewRHS,
285 ISD::CondCode &CCCode,
286 const SDLoc &dl, const SDValue OldLHS,
287 const SDValue OldRHS) const {
288 SDValue Chain;
289 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
290 OldRHS, Chain);
291}
292
293void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
294 SDValue &NewLHS, SDValue &NewRHS,
295 ISD::CondCode &CCCode,
296 const SDLoc &dl, const SDValue OldLHS,
297 const SDValue OldRHS,
298 SDValue &Chain,
299 bool IsSignaling) const {
300 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
301 // not supporting it. We can update this code when libgcc provides such
302 // functions.
303
304 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)((void)0)
305 && "Unsupported setcc type!")((void)0);
306
307 // Expand into one or more soft-fp libcall(s).
308 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
309 bool ShouldInvertCC = false;
310 switch (CCCode) {
311 case ISD::SETEQ:
312 case ISD::SETOEQ:
313 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
314 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
315 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
316 break;
317 case ISD::SETNE:
318 case ISD::SETUNE:
319 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
320 (VT == MVT::f64) ? RTLIB::UNE_F64 :
321 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
322 break;
323 case ISD::SETGE:
324 case ISD::SETOGE:
325 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
326 (VT == MVT::f64) ? RTLIB::OGE_F64 :
327 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
328 break;
329 case ISD::SETLT:
330 case ISD::SETOLT:
331 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
332 (VT == MVT::f64) ? RTLIB::OLT_F64 :
333 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
334 break;
335 case ISD::SETLE:
336 case ISD::SETOLE:
337 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
338 (VT == MVT::f64) ? RTLIB::OLE_F64 :
339 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
340 break;
341 case ISD::SETGT:
342 case ISD::SETOGT:
343 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
344 (VT == MVT::f64) ? RTLIB::OGT_F64 :
345 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
346 break;
347 case ISD::SETO:
348 ShouldInvertCC = true;
349 LLVM_FALLTHROUGH[[gnu::fallthrough]];
350 case ISD::SETUO:
351 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
352 (VT == MVT::f64) ? RTLIB::UO_F64 :
353 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
354 break;
355 case ISD::SETONE:
356 // SETONE = O && UNE
357 ShouldInvertCC = true;
358 LLVM_FALLTHROUGH[[gnu::fallthrough]];
359 case ISD::SETUEQ:
360 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
361 (VT == MVT::f64) ? RTLIB::UO_F64 :
362 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
364 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
365 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
366 break;
367 default:
368 // Invert CC for unordered comparisons
369 ShouldInvertCC = true;
370 switch (CCCode) {
371 case ISD::SETULT:
372 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
373 (VT == MVT::f64) ? RTLIB::OGE_F64 :
374 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
375 break;
376 case ISD::SETULE:
377 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
378 (VT == MVT::f64) ? RTLIB::OGT_F64 :
379 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
380 break;
381 case ISD::SETUGT:
382 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
383 (VT == MVT::f64) ? RTLIB::OLE_F64 :
384 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
385 break;
386 case ISD::SETUGE:
387 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
388 (VT == MVT::f64) ? RTLIB::OLT_F64 :
389 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
390 break;
391 default: llvm_unreachable("Do not know how to soften this setcc!")__builtin_unreachable();
392 }
393 }
394
395 // Use the target specific return value for comparions lib calls.
396 EVT RetVT = getCmpLibcallReturnType();
397 SDValue Ops[2] = {NewLHS, NewRHS};
398 TargetLowering::MakeLibCallOptions CallOptions;
399 EVT OpsVT[2] = { OldLHS.getValueType(),
400 OldRHS.getValueType() };
401 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
402 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
403 NewLHS = Call.first;
404 NewRHS = DAG.getConstant(0, dl, RetVT);
405
406 CCCode = getCmpLibcallCC(LC1);
407 if (ShouldInvertCC) {
408 assert(RetVT.isInteger())((void)0);
409 CCCode = getSetCCInverse(CCCode, RetVT);
410 }
411
412 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
413 // Update Chain.
414 Chain = Call.second;
415 } else {
416 EVT SetCCVT =
417 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
418 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
419 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
420 CCCode = getCmpLibcallCC(LC2);
421 if (ShouldInvertCC)
422 CCCode = getSetCCInverse(CCCode, RetVT);
423 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
424 if (Chain)
425 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
426 Call2.second);
427 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
428 Tmp.getValueType(), Tmp, NewLHS);
429 NewRHS = SDValue();
430 }
431}
432
433/// Return the entry encoding for a jump table in the current function. The
434/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
435unsigned TargetLowering::getJumpTableEncoding() const {
436 // In non-pic modes, just use the address of a block.
437 if (!isPositionIndependent())
438 return MachineJumpTableInfo::EK_BlockAddress;
439
440 // In PIC mode, if the target supports a GPRel32 directive, use it.
441 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
442 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
443
444 // Otherwise, use a label difference.
445 return MachineJumpTableInfo::EK_LabelDifference32;
446}
447
448SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
449 SelectionDAG &DAG) const {
450 // If our PIC model is GP relative, use the global offset table as the base.
451 unsigned JTEncoding = getJumpTableEncoding();
452
453 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
454 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
455 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
456
457 return Table;
458}
459
460/// This returns the relocation base for the given PIC jumptable, the same as
461/// getPICJumpTableRelocBase, but as an MCExpr.
462const MCExpr *
463TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
464 unsigned JTI,MCContext &Ctx) const{
465 // The normal PIC reloc base is the label at the start of the jump table.
466 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
467}
468
469bool
470TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
471 const TargetMachine &TM = getTargetMachine();
472 const GlobalValue *GV = GA->getGlobal();
473
474 // If the address is not even local to this DSO we will have to load it from
475 // a got and then add the offset.
476 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
477 return false;
478
479 // If the code is position independent we will have to add a base register.
480 if (isPositionIndependent())
481 return false;
482
483 // Otherwise we can do it.
484 return true;
485}
486
487//===----------------------------------------------------------------------===//
488// Optimization Methods
489//===----------------------------------------------------------------------===//
490
491/// If the specified instruction has a constant integer operand and there are
492/// bits set in that constant that are not demanded, then clear those bits and
493/// return true.
494bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
495 const APInt &DemandedBits,
496 const APInt &DemandedElts,
497 TargetLoweringOpt &TLO) const {
498 SDLoc DL(Op);
499 unsigned Opcode = Op.getOpcode();
500
501 // Do target-specific constant optimization.
502 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
503 return TLO.New.getNode();
504
505 // FIXME: ISD::SELECT, ISD::SELECT_CC
506 switch (Opcode) {
507 default:
508 break;
509 case ISD::XOR:
510 case ISD::AND:
511 case ISD::OR: {
512 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
513 if (!Op1C || Op1C->isOpaque())
514 return false;
515
516 // If this is a 'not' op, don't touch it because that's a canonical form.
517 const APInt &C = Op1C->getAPIntValue();
518 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
519 return false;
520
521 if (!C.isSubsetOf(DemandedBits)) {
522 EVT VT = Op.getValueType();
523 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
524 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
525 return TLO.CombineTo(Op, NewOp);
526 }
527
528 break;
529 }
530 }
531
532 return false;
533}
534
535bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
536 const APInt &DemandedBits,
537 TargetLoweringOpt &TLO) const {
538 EVT VT = Op.getValueType();
539 APInt DemandedElts = VT.isVector()
540 ? APInt::getAllOnesValue(VT.getVectorNumElements())
541 : APInt(1, 1);
542 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
543}
544
545/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
546/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
547/// generalized for targets with other types of implicit widening casts.
548bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
549 const APInt &Demanded,
550 TargetLoweringOpt &TLO) const {
551 assert(Op.getNumOperands() == 2 &&((void)0)
552 "ShrinkDemandedOp only supports binary operators!")((void)0);
553 assert(Op.getNode()->getNumValues() == 1 &&((void)0)
554 "ShrinkDemandedOp only supports nodes with one result!")((void)0);
555
556 SelectionDAG &DAG = TLO.DAG;
557 SDLoc dl(Op);
558
559 // Early return, as this function cannot handle vector types.
560 if (Op.getValueType().isVector())
561 return false;
562
563 // Don't do this if the node has another user, which may require the
564 // full value.
565 if (!Op.getNode()->hasOneUse())
566 return false;
567
568 // Search for the smallest integer type with free casts to and from
569 // Op's type. For expedience, just check power-of-2 integer types.
570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
571 unsigned DemandedSize = Demanded.getActiveBits();
572 unsigned SmallVTBits = DemandedSize;
573 if (!isPowerOf2_32(SmallVTBits))
574 SmallVTBits = NextPowerOf2(SmallVTBits);
575 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
576 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
577 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
578 TLI.isZExtFree(SmallVT, Op.getValueType())) {
579 // We found a type with free casts.
580 SDValue X = DAG.getNode(
581 Op.getOpcode(), dl, SmallVT,
582 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
583 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
584 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")((void)0);
585 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
586 return TLO.CombineTo(Op, Z);
587 }
588 }
589 return false;
590}
591
592bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
593 DAGCombinerInfo &DCI) const {
594 SelectionDAG &DAG = DCI.DAG;
595 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
596 !DCI.isBeforeLegalizeOps());
597 KnownBits Known;
598
599 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
600 if (Simplified) {
601 DCI.AddToWorklist(Op.getNode());
602 DCI.CommitTargetLoweringOpt(TLO);
603 }
604 return Simplified;
605}
606
607bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
608 KnownBits &Known,
609 TargetLoweringOpt &TLO,
610 unsigned Depth,
611 bool AssumeSingleUse) const {
612 EVT VT = Op.getValueType();
613
614 // TODO: We can probably do more work on calculating the known bits and
615 // simplifying the operations for scalable vectors, but for now we just
616 // bail out.
617 if (VT.isScalableVector()) {
618 // Pretend we don't know anything for now.
619 Known = KnownBits(DemandedBits.getBitWidth());
620 return false;
621 }
622
623 APInt DemandedElts = VT.isVector()
624 ? APInt::getAllOnesValue(VT.getVectorNumElements())
625 : APInt(1, 1);
626 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
627 AssumeSingleUse);
628}
629
630// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
631// TODO: Under what circumstances can we create nodes? Constant folding?
632SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
633 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
634 SelectionDAG &DAG, unsigned Depth) const {
635 // Limit search depth.
636 if (Depth >= SelectionDAG::MaxRecursionDepth)
637 return SDValue();
638
639 // Ignore UNDEFs.
640 if (Op.isUndef())
641 return SDValue();
642
643 // Not demanding any bits/elts from Op.
644 if (DemandedBits == 0 || DemandedElts == 0)
645 return DAG.getUNDEF(Op.getValueType());
646
647 unsigned NumElts = DemandedElts.getBitWidth();
648 unsigned BitWidth = DemandedBits.getBitWidth();
649 KnownBits LHSKnown, RHSKnown;
650 switch (Op.getOpcode()) {
651 case ISD::BITCAST: {
652 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
653 EVT SrcVT = Src.getValueType();
654 EVT DstVT = Op.getValueType();
655 if (SrcVT == DstVT)
656 return Src;
657
658 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
659 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
660 if (NumSrcEltBits == NumDstEltBits)
661 if (SDValue V = SimplifyMultipleUseDemandedBits(
662 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
663 return DAG.getBitcast(DstVT, V);
664
665 // TODO - bigendian once we have test coverage.
666 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
667 DAG.getDataLayout().isLittleEndian()) {
668 unsigned Scale = NumDstEltBits / NumSrcEltBits;
669 unsigned NumSrcElts = SrcVT.getVectorNumElements();
670 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
671 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
672 for (unsigned i = 0; i != Scale; ++i) {
673 unsigned Offset = i * NumSrcEltBits;
674 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
675 if (!Sub.isNullValue()) {
676 DemandedSrcBits |= Sub;
677 for (unsigned j = 0; j != NumElts; ++j)
678 if (DemandedElts[j])
679 DemandedSrcElts.setBit((j * Scale) + i);
680 }
681 }
682
683 if (SDValue V = SimplifyMultipleUseDemandedBits(
684 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
685 return DAG.getBitcast(DstVT, V);
686 }
687
688 // TODO - bigendian once we have test coverage.
689 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
690 DAG.getDataLayout().isLittleEndian()) {
691 unsigned Scale = NumSrcEltBits / NumDstEltBits;
692 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
693 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
694 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
695 for (unsigned i = 0; i != NumElts; ++i)
696 if (DemandedElts[i]) {
697 unsigned Offset = (i % Scale) * NumDstEltBits;
698 DemandedSrcBits.insertBits(DemandedBits, Offset);
699 DemandedSrcElts.setBit(i / Scale);
700 }
701
702 if (SDValue V = SimplifyMultipleUseDemandedBits(
703 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
704 return DAG.getBitcast(DstVT, V);
705 }
706
707 break;
708 }
709 case ISD::AND: {
710 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
711 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
712
713 // If all of the demanded bits are known 1 on one side, return the other.
714 // These bits cannot contribute to the result of the 'and' in this
715 // context.
716 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
717 return Op.getOperand(0);
718 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
719 return Op.getOperand(1);
720 break;
721 }
722 case ISD::OR: {
723 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
724 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
725
726 // If all of the demanded bits are known zero on one side, return the
727 // other. These bits cannot contribute to the result of the 'or' in this
728 // context.
729 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
730 return Op.getOperand(0);
731 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
732 return Op.getOperand(1);
733 break;
734 }
735 case ISD::XOR: {
736 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
737 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
738
739 // If all of the demanded bits are known zero on one side, return the
740 // other.
741 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
742 return Op.getOperand(0);
743 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
744 return Op.getOperand(1);
745 break;
746 }
747 case ISD::SHL: {
748 // If we are only demanding sign bits then we can use the shift source
749 // directly.
750 if (const APInt *MaxSA =
751 DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
752 SDValue Op0 = Op.getOperand(0);
753 unsigned ShAmt = MaxSA->getZExtValue();
754 unsigned NumSignBits =
755 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
756 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
757 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
758 return Op0;
759 }
760 break;
761 }
762 case ISD::SETCC: {
763 SDValue Op0 = Op.getOperand(0);
764 SDValue Op1 = Op.getOperand(1);
765 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
766 // If (1) we only need the sign-bit, (2) the setcc operands are the same
767 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
768 // -1, we may be able to bypass the setcc.
769 if (DemandedBits.isSignMask() &&
770 Op0.getScalarValueSizeInBits() == BitWidth &&
771 getBooleanContents(Op0.getValueType()) ==
772 BooleanContent::ZeroOrNegativeOneBooleanContent) {
773 // If we're testing X < 0, then this compare isn't needed - just use X!
774 // FIXME: We're limiting to integer types here, but this should also work
775 // if we don't care about FP signed-zero. The use of SETLT with FP means
776 // that we don't care about NaNs.
777 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
778 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
779 return Op0;
780 }
781 break;
782 }
783 case ISD::SIGN_EXTEND_INREG: {
784 // If none of the extended bits are demanded, eliminate the sextinreg.
785 SDValue Op0 = Op.getOperand(0);
786 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
787 unsigned ExBits = ExVT.getScalarSizeInBits();
788 if (DemandedBits.getActiveBits() <= ExBits)
789 return Op0;
790 // If the input is already sign extended, just drop the extension.
791 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
792 if (NumSignBits >= (BitWidth - ExBits + 1))
793 return Op0;
794 break;
795 }
796 case ISD::ANY_EXTEND_VECTOR_INREG:
797 case ISD::SIGN_EXTEND_VECTOR_INREG:
798 case ISD::ZERO_EXTEND_VECTOR_INREG: {
799 // If we only want the lowest element and none of extended bits, then we can
800 // return the bitcasted source vector.
801 SDValue Src = Op.getOperand(0);
802 EVT SrcVT = Src.getValueType();
803 EVT DstVT = Op.getValueType();
804 if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
805 DAG.getDataLayout().isLittleEndian() &&
806 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
807 return DAG.getBitcast(DstVT, Src);
808 }
809 break;
810 }
811 case ISD::INSERT_VECTOR_ELT: {
812 // If we don't demand the inserted element, return the base vector.
813 SDValue Vec = Op.getOperand(0);
814 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
815 EVT VecVT = Vec.getValueType();
816 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
817 !DemandedElts[CIdx->getZExtValue()])
818 return Vec;
819 break;
820 }
821 case ISD::INSERT_SUBVECTOR: {
822 // If we don't demand the inserted subvector, return the base vector.
823 SDValue Vec = Op.getOperand(0);
824 SDValue Sub = Op.getOperand(1);
825 uint64_t Idx = Op.getConstantOperandVal(2);
826 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
827 if (DemandedElts.extractBits(NumSubElts, Idx) == 0)
828 return Vec;
829 break;
830 }
831 case ISD::VECTOR_SHUFFLE: {
832 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
833
834 // If all the demanded elts are from one operand and are inline,
835 // then we can use the operand directly.
836 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
837 for (unsigned i = 0; i != NumElts; ++i) {
838 int M = ShuffleMask[i];
839 if (M < 0 || !DemandedElts[i])
840 continue;
841 AllUndef = false;
842 IdentityLHS &= (M == (int)i);
843 IdentityRHS &= ((M - NumElts) == i);
844 }
845
846 if (AllUndef)
847 return DAG.getUNDEF(Op.getValueType());
848 if (IdentityLHS)
849 return Op.getOperand(0);
850 if (IdentityRHS)
851 return Op.getOperand(1);
852 break;
853 }
854 default:
855 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
856 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
857 Op, DemandedBits, DemandedElts, DAG, Depth))
858 return V;
859 break;
860 }
861 return SDValue();
862}
863
864SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
865 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
866 unsigned Depth) const {
867 EVT VT = Op.getValueType();
868 APInt DemandedElts = VT.isVector()
869 ? APInt::getAllOnesValue(VT.getVectorNumElements())
870 : APInt(1, 1);
871 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
872 Depth);
873}
874
875SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
876 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
877 unsigned Depth) const {
878 APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());
879 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
880 Depth);
881}
882
883/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
884/// result of Op are ever used downstream. If we can use this information to
885/// simplify Op, create a new simplified DAG node and return true, returning the
886/// original and new nodes in Old and New. Otherwise, analyze the expression and
887/// return a mask of Known bits for the expression (used to simplify the
888/// caller). The Known bits may only be accurate for those bits in the
889/// OriginalDemandedBits and OriginalDemandedElts.
890bool TargetLowering::SimplifyDemandedBits(
891 SDValue Op, const APInt &OriginalDemandedBits,
892 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
893 unsigned Depth, bool AssumeSingleUse) const {
894 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
895 assert(Op.getScalarValueSizeInBits() == BitWidth &&((void)0)
896 "Mask size mismatches value type size!")((void)0);
897
898 // Don't know anything.
899 Known = KnownBits(BitWidth);
900
901 // TODO: We can probably do more work on calculating the known bits and
902 // simplifying the operations for scalable vectors, but for now we just
903 // bail out.
904 if (Op.getValueType().isScalableVector())
905 return false;
906
907 unsigned NumElts = OriginalDemandedElts.getBitWidth();
908 assert((!Op.getValueType().isVector() ||((void)0)
909 NumElts == Op.getValueType().getVectorNumElements()) &&((void)0)
910 "Unexpected vector size")((void)0);
911
912 APInt DemandedBits = OriginalDemandedBits;
913 APInt DemandedElts = OriginalDemandedElts;
914 SDLoc dl(Op);
915 auto &DL = TLO.DAG.getDataLayout();
916
917 // Undef operand.
918 if (Op.isUndef())
919 return false;
920
921 if (Op.getOpcode() == ISD::Constant) {
922 // We know all of the bits for a constant!
923 Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
924 return false;
925 }
926
927 if (Op.getOpcode() == ISD::ConstantFP) {
928 // We know all of the bits for a floating point constant!
929 Known = KnownBits::makeConstant(
930 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
931 return false;
932 }
933
934 // Other users may use these bits.
935 EVT VT = Op.getValueType();
936 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
937 if (Depth != 0) {
938 // If not at the root, Just compute the Known bits to
939 // simplify things downstream.
940 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
941 return false;
942 }
943 // If this is the root being simplified, allow it to have multiple uses,
944 // just set the DemandedBits/Elts to all bits.
945 DemandedBits = APInt::getAllOnesValue(BitWidth);
946 DemandedElts = APInt::getAllOnesValue(NumElts);
947 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
948 // Not demanding any bits/elts from Op.
949 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
950 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
951 // Limit search depth.
952 return false;
953 }
954
955 KnownBits Known2;
956 switch (Op.getOpcode()) {
957 case ISD::TargetConstant:
958 llvm_unreachable("Can't simplify this node")__builtin_unreachable();
959 case ISD::SCALAR_TO_VECTOR: {
960 if (!DemandedElts[0])
961 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
962
963 KnownBits SrcKnown;
964 SDValue Src = Op.getOperand(0);
965 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
966 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
967 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
968 return true;
969
970 // Upper elements are undef, so only get the knownbits if we just demand
971 // the bottom element.
972 if (DemandedElts == 1)
973 Known = SrcKnown.anyextOrTrunc(BitWidth);
974 break;
975 }
976 case ISD::BUILD_VECTOR:
977 // Collect the known bits that are shared by every demanded element.
978 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
979 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
980 return false; // Don't fall through, will infinitely loop.
981 case ISD::LOAD: {
982 auto *LD = cast<LoadSDNode>(Op);
983 if (getTargetConstantFromLoad(LD)) {
984 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
985 return false; // Don't fall through, will infinitely loop.
986 }
987 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
988 // If this is a ZEXTLoad and we are looking at the loaded value.
989 EVT MemVT = LD->getMemoryVT();
990 unsigned MemBits = MemVT.getScalarSizeInBits();
991 Known.Zero.setBitsFrom(MemBits);
992 return false; // Don't fall through, will infinitely loop.
993 }
994 break;
995 }
996 case ISD::INSERT_VECTOR_ELT: {
997 SDValue Vec = Op.getOperand(0);
998 SDValue Scl = Op.getOperand(1);
999 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1000 EVT VecVT = Vec.getValueType();
1001
1002 // If index isn't constant, assume we need all vector elements AND the
1003 // inserted element.
1004 APInt DemandedVecElts(DemandedElts);
1005 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1006 unsigned Idx = CIdx->getZExtValue();
1007 DemandedVecElts.clearBit(Idx);
1008
1009 // Inserted element is not required.
1010 if (!DemandedElts[Idx])
1011 return TLO.CombineTo(Op, Vec);
1012 }
1013
1014 KnownBits KnownScl;
1015 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1016 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1017 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1018 return true;
1019
1020 Known = KnownScl.anyextOrTrunc(BitWidth);
1021
1022 KnownBits KnownVec;
1023 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1024 Depth + 1))
1025 return true;
1026
1027 if (!!DemandedVecElts)
1028 Known = KnownBits::commonBits(Known, KnownVec);
1029
1030 return false;
1031 }
1032 case ISD::INSERT_SUBVECTOR: {
1033 // Demand any elements from the subvector and the remainder from the src its
1034 // inserted into.
1035 SDValue Src = Op.getOperand(0);
1036 SDValue Sub = Op.getOperand(1);
1037 uint64_t Idx = Op.getConstantOperandVal(2);
1038 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1039 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1040 APInt DemandedSrcElts = DemandedElts;
1041 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
1042
1043 KnownBits KnownSub, KnownSrc;
1044 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1045 Depth + 1))
1046 return true;
1047 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1048 Depth + 1))
1049 return true;
1050
1051 Known.Zero.setAllBits();
1052 Known.One.setAllBits();
1053 if (!!DemandedSubElts)
1054 Known = KnownBits::commonBits(Known, KnownSub);
1055 if (!!DemandedSrcElts)
1056 Known = KnownBits::commonBits(Known, KnownSrc);
1057
1058 // Attempt to avoid multi-use src if we don't need anything from it.
1059 if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||
1060 !DemandedSrcElts.isAllOnesValue()) {
1061 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1062 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1063 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1064 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1065 if (NewSub || NewSrc) {
1066 NewSub = NewSub ? NewSub : Sub;
1067 NewSrc = NewSrc ? NewSrc : Src;
1068 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1069 Op.getOperand(2));
1070 return TLO.CombineTo(Op, NewOp);
1071 }
1072 }
1073 break;
1074 }
1075 case ISD::EXTRACT_SUBVECTOR: {
1076 // Offset the demanded elts by the subvector index.
1077 SDValue Src = Op.getOperand(0);
1078 if (Src.getValueType().isScalableVector())
1079 break;
1080 uint64_t Idx = Op.getConstantOperandVal(1);
1081 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1082 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1083
1084 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1085 Depth + 1))
1086 return true;
1087
1088 // Attempt to avoid multi-use src if we don't need anything from it.
1089 if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {
1090 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1091 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1092 if (DemandedSrc) {
1093 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1094 Op.getOperand(1));
1095 return TLO.CombineTo(Op, NewOp);
1096 }
1097 }
1098 break;
1099 }
1100 case ISD::CONCAT_VECTORS: {
1101 Known.Zero.setAllBits();
1102 Known.One.setAllBits();
1103 EVT SubVT = Op.getOperand(0).getValueType();
1104 unsigned NumSubVecs = Op.getNumOperands();
1105 unsigned NumSubElts = SubVT.getVectorNumElements();
1106 for (unsigned i = 0; i != NumSubVecs; ++i) {
1107 APInt DemandedSubElts =
1108 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1109 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1110 Known2, TLO, Depth + 1))
1111 return true;
1112 // Known bits are shared by every demanded subvector element.
1113 if (!!DemandedSubElts)
1114 Known = KnownBits::commonBits(Known, Known2);
1115 }
1116 break;
1117 }
1118 case ISD::VECTOR_SHUFFLE: {
1119 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1120
1121 // Collect demanded elements from shuffle operands..
1122 APInt DemandedLHS(NumElts, 0);
1123 APInt DemandedRHS(NumElts, 0);
1124 for (unsigned i = 0; i != NumElts; ++i) {
1125 if (!DemandedElts[i])
1126 continue;
1127 int M = ShuffleMask[i];
1128 if (M < 0) {
1129 // For UNDEF elements, we don't know anything about the common state of
1130 // the shuffle result.
1131 DemandedLHS.clearAllBits();
1132 DemandedRHS.clearAllBits();
1133 break;
1134 }
1135 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((void)0);
1136 if (M < (int)NumElts)
1137 DemandedLHS.setBit(M);
1138 else
1139 DemandedRHS.setBit(M - NumElts);
1140 }
1141
1142 if (!!DemandedLHS || !!DemandedRHS) {
1143 SDValue Op0 = Op.getOperand(0);
1144 SDValue Op1 = Op.getOperand(1);
1145
1146 Known.Zero.setAllBits();
1147 Known.One.setAllBits();
1148 if (!!DemandedLHS) {
1149 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1150 Depth + 1))
1151 return true;
1152 Known = KnownBits::commonBits(Known, Known2);
1153 }
1154 if (!!DemandedRHS) {
1155 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1156 Depth + 1))
1157 return true;
1158 Known = KnownBits::commonBits(Known, Known2);
1159 }
1160
1161 // Attempt to avoid multi-use ops if we don't need anything from them.
1162 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1163 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1164 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1165 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1166 if (DemandedOp0 || DemandedOp1) {
1167 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1168 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1169 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1170 return TLO.CombineTo(Op, NewOp);
1171 }
1172 }
1173 break;
1174 }
1175 case ISD::AND: {
1176 SDValue Op0 = Op.getOperand(0);
1177 SDValue Op1 = Op.getOperand(1);
1178
1179 // If the RHS is a constant, check to see if the LHS would be zero without
1180 // using the bits from the RHS. Below, we use knowledge about the RHS to
1181 // simplify the LHS, here we're using information from the LHS to simplify
1182 // the RHS.
1183 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1184 // Do not increment Depth here; that can cause an infinite loop.
1185 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1186 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1187 if ((LHSKnown.Zero & DemandedBits) ==
1188 (~RHSC->getAPIntValue() & DemandedBits))
1189 return TLO.CombineTo(Op, Op0);
1190
1191 // If any of the set bits in the RHS are known zero on the LHS, shrink
1192 // the constant.
1193 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1194 DemandedElts, TLO))
1195 return true;
1196
1197 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1198 // constant, but if this 'and' is only clearing bits that were just set by
1199 // the xor, then this 'and' can be eliminated by shrinking the mask of
1200 // the xor. For example, for a 32-bit X:
1201 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1202 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1203 LHSKnown.One == ~RHSC->getAPIntValue()) {
1204 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1205 return TLO.CombineTo(Op, Xor);
1206 }
1207 }
1208
1209 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1210 Depth + 1))
1211 return true;
1212 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1213 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1214 Known2, TLO, Depth + 1))
1215 return true;
1216 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1217
1218 // Attempt to avoid multi-use ops if we don't need anything from them.
1219 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1220 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1221 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1222 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1223 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1224 if (DemandedOp0 || DemandedOp1) {
1225 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1226 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1227 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1228 return TLO.CombineTo(Op, NewOp);
1229 }
1230 }
1231
1232 // If all of the demanded bits are known one on one side, return the other.
1233 // These bits cannot contribute to the result of the 'and'.
1234 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1235 return TLO.CombineTo(Op, Op0);
1236 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1237 return TLO.CombineTo(Op, Op1);
1238 // If all of the demanded bits in the inputs are known zeros, return zero.
1239 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1240 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1241 // If the RHS is a constant, see if we can simplify it.
1242 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1243 TLO))
1244 return true;
1245 // If the operation can be done in a smaller type, do so.
1246 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1247 return true;
1248
1249 Known &= Known2;
1250 break;
1251 }
1252 case ISD::OR: {
1253 SDValue Op0 = Op.getOperand(0);
1254 SDValue Op1 = Op.getOperand(1);
1255
1256 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1257 Depth + 1))
1258 return true;
1259 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1260 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1261 Known2, TLO, Depth + 1))
1262 return true;
1263 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1264
1265 // Attempt to avoid multi-use ops if we don't need anything from them.
1266 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1267 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1268 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1269 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1270 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1271 if (DemandedOp0 || DemandedOp1) {
1272 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1273 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1274 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1275 return TLO.CombineTo(Op, NewOp);
1276 }
1277 }
1278
1279 // If all of the demanded bits are known zero on one side, return the other.
1280 // These bits cannot contribute to the result of the 'or'.
1281 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1282 return TLO.CombineTo(Op, Op0);
1283 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1284 return TLO.CombineTo(Op, Op1);
1285 // If the RHS is a constant, see if we can simplify it.
1286 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1287 return true;
1288 // If the operation can be done in a smaller type, do so.
1289 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1290 return true;
1291
1292 Known |= Known2;
1293 break;
1294 }
1295 case ISD::XOR: {
1296 SDValue Op0 = Op.getOperand(0);
1297 SDValue Op1 = Op.getOperand(1);
1298
1299 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1300 Depth + 1))
1301 return true;
1302 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1303 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1304 Depth + 1))
1305 return true;
1306 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1307
1308 // Attempt to avoid multi-use ops if we don't need anything from them.
1309 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1310 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1311 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1312 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1313 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1314 if (DemandedOp0 || DemandedOp1) {
1315 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1316 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1317 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1318 return TLO.CombineTo(Op, NewOp);
1319 }
1320 }
1321
1322 // If all of the demanded bits are known zero on one side, return the other.
1323 // These bits cannot contribute to the result of the 'xor'.
1324 if (DemandedBits.isSubsetOf(Known.Zero))
1325 return TLO.CombineTo(Op, Op0);
1326 if (DemandedBits.isSubsetOf(Known2.Zero))
1327 return TLO.CombineTo(Op, Op1);
1328 // If the operation can be done in a smaller type, do so.
1329 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1330 return true;
1331
1332 // If all of the unknown bits are known to be zero on one side or the other
1333 // turn this into an *inclusive* or.
1334 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1335 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1336 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1337
1338 ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1339 if (C) {
1340 // If one side is a constant, and all of the set bits in the constant are
1341 // also known set on the other side, turn this into an AND, as we know
1342 // the bits will be cleared.
1343 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1344 // NB: it is okay if more bits are known than are requested
1345 if (C->getAPIntValue() == Known2.One) {
1346 SDValue ANDC =
1347 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1348 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1349 }
1350
1351 // If the RHS is a constant, see if we can change it. Don't alter a -1
1352 // constant because that's a 'not' op, and that is better for combining
1353 // and codegen.
1354 if (!C->isAllOnesValue() &&
1355 DemandedBits.isSubsetOf(C->getAPIntValue())) {
1356 // We're flipping all demanded bits. Flip the undemanded bits too.
1357 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1358 return TLO.CombineTo(Op, New);
1359 }
1360 }
1361
1362 // If we can't turn this into a 'not', try to shrink the constant.
1363 if (!C || !C->isAllOnesValue())
1364 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1365 return true;
1366
1367 Known ^= Known2;
1368 break;
1369 }
1370 case ISD::SELECT:
1371 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1372 Depth + 1))
1373 return true;
1374 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1375 Depth + 1))
1376 return true;
1377 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1378 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1379
1380 // If the operands are constants, see if we can simplify them.
1381 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1382 return true;
1383
1384 // Only known if known in both the LHS and RHS.
1385 Known = KnownBits::commonBits(Known, Known2);
1386 break;
1387 case ISD::SELECT_CC:
1388 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1389 Depth + 1))
1390 return true;
1391 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1392 Depth + 1))
1393 return true;
1394 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1395 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1396
1397 // If the operands are constants, see if we can simplify them.
1398 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1399 return true;
1400
1401 // Only known if known in both the LHS and RHS.
1402 Known = KnownBits::commonBits(Known, Known2);
1403 break;
1404 case ISD::SETCC: {
1405 SDValue Op0 = Op.getOperand(0);
1406 SDValue Op1 = Op.getOperand(1);
1407 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1408 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1409 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1410 // -1, we may be able to bypass the setcc.
1411 if (DemandedBits.isSignMask() &&
1412 Op0.getScalarValueSizeInBits() == BitWidth &&
1413 getBooleanContents(Op0.getValueType()) ==
1414 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1415 // If we're testing X < 0, then this compare isn't needed - just use X!
1416 // FIXME: We're limiting to integer types here, but this should also work
1417 // if we don't care about FP signed-zero. The use of SETLT with FP means
1418 // that we don't care about NaNs.
1419 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1420 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1421 return TLO.CombineTo(Op, Op0);
1422
1423 // TODO: Should we check for other forms of sign-bit comparisons?
1424 // Examples: X <= -1, X >= 0
1425 }
1426 if (getBooleanContents(Op0.getValueType()) ==
1427 TargetLowering::ZeroOrOneBooleanContent &&
1428 BitWidth > 1)
1429 Known.Zero.setBitsFrom(1);
1430 break;
1431 }
1432 case ISD::SHL: {
1433 SDValue Op0 = Op.getOperand(0);
1434 SDValue Op1 = Op.getOperand(1);
1435 EVT ShiftVT = Op1.getValueType();
1436
1437 if (const APInt *SA =
1438 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1439 unsigned ShAmt = SA->getZExtValue();
1440 if (ShAmt == 0)
1441 return TLO.CombineTo(Op, Op0);
1442
1443 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1444 // single shift. We can do this if the bottom bits (which are shifted
1445 // out) are never demanded.
1446 // TODO - support non-uniform vector amounts.
1447 if (Op0.getOpcode() == ISD::SRL) {
1448 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1449 if (const APInt *SA2 =
1450 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1451 unsigned C1 = SA2->getZExtValue();
1452 unsigned Opc = ISD::SHL;
1453 int Diff = ShAmt - C1;
1454 if (Diff < 0) {
1455 Diff = -Diff;
1456 Opc = ISD::SRL;
1457 }
1458 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1459 return TLO.CombineTo(
1460 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1461 }
1462 }
1463 }
1464
1465 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1466 // are not demanded. This will likely allow the anyext to be folded away.
1467 // TODO - support non-uniform vector amounts.
1468 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1469 SDValue InnerOp = Op0.getOperand(0);
1470 EVT InnerVT = InnerOp.getValueType();
1471 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1472 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1473 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1474 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1475 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1476 ShTy = InnerVT;
1477 SDValue NarrowShl =
1478 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1479 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1480 return TLO.CombineTo(
1481 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1482 }
1483
1484 // Repeat the SHL optimization above in cases where an extension
1485 // intervenes: (shl (anyext (shr x, c1)), c2) to
1486 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1487 // aren't demanded (as above) and that the shifted upper c1 bits of
1488 // x aren't demanded.
1489 // TODO - support non-uniform vector amounts.
1490 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1491 InnerOp.hasOneUse()) {
1492 if (const APInt *SA2 =
1493 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1494 unsigned InnerShAmt = SA2->getZExtValue();
1495 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1496 DemandedBits.getActiveBits() <=
1497 (InnerBits - InnerShAmt + ShAmt) &&
1498 DemandedBits.countTrailingZeros() >= ShAmt) {
1499 SDValue NewSA =
1500 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1501 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1502 InnerOp.getOperand(0));
1503 return TLO.CombineTo(
1504 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1505 }
1506 }
1507 }
1508 }
1509
1510 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1511 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1512 Depth + 1))
1513 return true;
1514 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1515 Known.Zero <<= ShAmt;
1516 Known.One <<= ShAmt;
1517 // low bits known zero.
1518 Known.Zero.setLowBits(ShAmt);
1519
1520 // Try shrinking the operation as long as the shift amount will still be
1521 // in range.
1522 if ((ShAmt < DemandedBits.getActiveBits()) &&
1523 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1524 return true;
1525 }
1526
1527 // If we are only demanding sign bits then we can use the shift source
1528 // directly.
1529 if (const APInt *MaxSA =
1530 TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1531 unsigned ShAmt = MaxSA->getZExtValue();
1532 unsigned NumSignBits =
1533 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1534 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1535 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1536 return TLO.CombineTo(Op, Op0);
1537 }
1538 break;
1539 }
1540 case ISD::SRL: {
1541 SDValue Op0 = Op.getOperand(0);
1542 SDValue Op1 = Op.getOperand(1);
1543 EVT ShiftVT = Op1.getValueType();
1544
1545 if (const APInt *SA =
1546 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1547 unsigned ShAmt = SA->getZExtValue();
1548 if (ShAmt == 0)
1549 return TLO.CombineTo(Op, Op0);
1550
1551 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1552 // single shift. We can do this if the top bits (which are shifted out)
1553 // are never demanded.
1554 // TODO - support non-uniform vector amounts.
1555 if (Op0.getOpcode() == ISD::SHL) {
1556 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1557 if (const APInt *SA2 =
1558 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1559 unsigned C1 = SA2->getZExtValue();
1560 unsigned Opc = ISD::SRL;
1561 int Diff = ShAmt - C1;
1562 if (Diff < 0) {
1563 Diff = -Diff;
1564 Opc = ISD::SHL;
1565 }
1566 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1567 return TLO.CombineTo(
1568 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1569 }
1570 }
1571 }
1572
1573 APInt InDemandedMask = (DemandedBits << ShAmt);
1574
1575 // If the shift is exact, then it does demand the low bits (and knows that
1576 // they are zero).
1577 if (Op->getFlags().hasExact())
1578 InDemandedMask.setLowBits(ShAmt);
1579
1580 // Compute the new bits that are at the top now.
1581 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1582 Depth + 1))
1583 return true;
1584 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1585 Known.Zero.lshrInPlace(ShAmt);
1586 Known.One.lshrInPlace(ShAmt);
1587 // High bits known zero.
1588 Known.Zero.setHighBits(ShAmt);
1589 }
1590 break;
1591 }
1592 case ISD::SRA: {
1593 SDValue Op0 = Op.getOperand(0);
1594 SDValue Op1 = Op.getOperand(1);
1595 EVT ShiftVT = Op1.getValueType();
1596
1597 // If we only want bits that already match the signbit then we don't need
1598 // to shift.
1599 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1600 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1601 NumHiDemandedBits)
1602 return TLO.CombineTo(Op, Op0);
1603
1604 // If this is an arithmetic shift right and only the low-bit is set, we can
1605 // always convert this into a logical shr, even if the shift amount is
1606 // variable. The low bit of the shift cannot be an input sign bit unless
1607 // the shift amount is >= the size of the datatype, which is undefined.
1608 if (DemandedBits.isOneValue())
1609 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1610
1611 if (const APInt *SA =
1612 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1613 unsigned ShAmt = SA->getZExtValue();
1614 if (ShAmt == 0)
1615 return TLO.CombineTo(Op, Op0);
1616
1617 APInt InDemandedMask = (DemandedBits << ShAmt);
1618
1619 // If the shift is exact, then it does demand the low bits (and knows that
1620 // they are zero).
1621 if (Op->getFlags().hasExact())
1622 InDemandedMask.setLowBits(ShAmt);
1623
1624 // If any of the demanded bits are produced by the sign extension, we also
1625 // demand the input sign bit.
1626 if (DemandedBits.countLeadingZeros() < ShAmt)
1627 InDemandedMask.setSignBit();
1628
1629 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1630 Depth + 1))
1631 return true;
1632 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1633 Known.Zero.lshrInPlace(ShAmt);
1634 Known.One.lshrInPlace(ShAmt);
1635
1636 // If the input sign bit is known to be zero, or if none of the top bits
1637 // are demanded, turn this into an unsigned shift right.
1638 if (Known.Zero[BitWidth - ShAmt - 1] ||
1639 DemandedBits.countLeadingZeros() >= ShAmt) {
1640 SDNodeFlags Flags;
1641 Flags.setExact(Op->getFlags().hasExact());
1642 return TLO.CombineTo(
1643 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1644 }
1645
1646 int Log2 = DemandedBits.exactLogBase2();
1647 if (Log2 >= 0) {
1648 // The bit must come from the sign.
1649 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1650 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1651 }
1652
1653 if (Known.One[BitWidth - ShAmt - 1])
1654 // New bits are known one.
1655 Known.One.setHighBits(ShAmt);
1656
1657 // Attempt to avoid multi-use ops if we don't need anything from them.
1658 if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1659 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1660 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1661 if (DemandedOp0) {
1662 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1663 return TLO.CombineTo(Op, NewOp);
1664 }
1665 }
1666 }
1667 break;
1668 }
1669 case ISD::FSHL:
1670 case ISD::FSHR: {
1671 SDValue Op0 = Op.getOperand(0);
1672 SDValue Op1 = Op.getOperand(1);
1673 SDValue Op2 = Op.getOperand(2);
1674 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1675
1676 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1677 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1678
1679 // For fshl, 0-shift returns the 1st arg.
1680 // For fshr, 0-shift returns the 2nd arg.
1681 if (Amt == 0) {
1682 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1683 Known, TLO, Depth + 1))
1684 return true;
1685 break;
1686 }
1687
1688 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1689 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1690 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1691 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1692 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1693 Depth + 1))
1694 return true;
1695 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1696 Depth + 1))
1697 return true;
1698
1699 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1700 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1701 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1702 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1703 Known.One |= Known2.One;
1704 Known.Zero |= Known2.Zero;
1705 }
1706
1707 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1708 if (isPowerOf2_32(BitWidth)) {
1709 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1710 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1711 Known2, TLO, Depth + 1))
1712 return true;
1713 }
1714 break;
1715 }
1716 case ISD::ROTL:
1717 case ISD::ROTR: {
1718 SDValue Op0 = Op.getOperand(0);
1719 SDValue Op1 = Op.getOperand(1);
1720
1721 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1722 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1723 return TLO.CombineTo(Op, Op0);
1724
1725 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1726 if (isPowerOf2_32(BitWidth)) {
1727 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1728 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1729 Depth + 1))
1730 return true;
1731 }
1732 break;
1733 }
1734 case ISD::UMIN: {
1735 // Check if one arg is always less than (or equal) to the other arg.
1736 SDValue Op0 = Op.getOperand(0);
1737 SDValue Op1 = Op.getOperand(1);
1738 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1739 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1740 Known = KnownBits::umin(Known0, Known1);
1741 if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1742 return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1743 if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1744 return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1745 break;
1746 }
1747 case ISD::UMAX: {
1748 // Check if one arg is always greater than (or equal) to the other arg.
1749 SDValue Op0 = Op.getOperand(0);
1750 SDValue Op1 = Op.getOperand(1);
1751 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1752 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1753 Known = KnownBits::umax(Known0, Known1);
1754 if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1755 return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1756 if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1757 return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1758 break;
1759 }
1760 case ISD::BITREVERSE: {
1761 SDValue Src = Op.getOperand(0);
1762 APInt DemandedSrcBits = DemandedBits.reverseBits();
1763 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1764 Depth + 1))
1765 return true;
1766 Known.One = Known2.One.reverseBits();
1767 Known.Zero = Known2.Zero.reverseBits();
1768 break;
1769 }
1770 case ISD::BSWAP: {
1771 SDValue Src = Op.getOperand(0);
1772 APInt DemandedSrcBits = DemandedBits.byteSwap();
1773 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1774 Depth + 1))
1775 return true;
1776 Known.One = Known2.One.byteSwap();
1777 Known.Zero = Known2.Zero.byteSwap();
1778 break;
1779 }
1780 case ISD::CTPOP: {
1781 // If only 1 bit is demanded, replace with PARITY as long as we're before
1782 // op legalization.
1783 // FIXME: Limit to scalars for now.
1784 if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())
1785 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1786 Op.getOperand(0)));
1787
1788 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1789 break;
1790 }
1791 case ISD::SIGN_EXTEND_INREG: {
1792 SDValue Op0 = Op.getOperand(0);
1793 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1794 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1795
1796 // If we only care about the highest bit, don't bother shifting right.
1797 if (DemandedBits.isSignMask()) {
1798 unsigned NumSignBits =
1799 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1800 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1801 // However if the input is already sign extended we expect the sign
1802 // extension to be dropped altogether later and do not simplify.
1803 if (!AlreadySignExtended) {
1804 // Compute the correct shift amount type, which must be getShiftAmountTy
1805 // for scalar types after legalization.
1806 EVT ShiftAmtTy = VT;
1807 if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1808 ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1809
1810 SDValue ShiftAmt =
1811 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1812 return TLO.CombineTo(Op,
1813 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1814 }
1815 }
1816
1817 // If none of the extended bits are demanded, eliminate the sextinreg.
1818 if (DemandedBits.getActiveBits() <= ExVTBits)
1819 return TLO.CombineTo(Op, Op0);
1820
1821 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1822
1823 // Since the sign extended bits are demanded, we know that the sign
1824 // bit is demanded.
1825 InputDemandedBits.setBit(ExVTBits - 1);
1826
1827 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1828 return true;
1829 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1830
1831 // If the sign bit of the input is known set or clear, then we know the
1832 // top bits of the result.
1833
1834 // If the input sign bit is known zero, convert this into a zero extension.
1835 if (Known.Zero[ExVTBits - 1])
1836 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1837
1838 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1839 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1840 Known.One.setBitsFrom(ExVTBits);
1841 Known.Zero &= Mask;
1842 } else { // Input sign bit unknown
1843 Known.Zero &= Mask;
1844 Known.One &= Mask;
1845 }
1846 break;
1847 }
1848 case ISD::BUILD_PAIR: {
1849 EVT HalfVT = Op.getOperand(0).getValueType();
1850 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1851
1852 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1853 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1854
1855 KnownBits KnownLo, KnownHi;
1856
1857 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1858 return true;
1859
1860 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1861 return true;
1862
1863 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1864 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1865
1866 Known.One = KnownLo.One.zext(BitWidth) |
1867 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1868 break;
1869 }
1870 case ISD::ZERO_EXTEND:
1871 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1872 SDValue Src = Op.getOperand(0);
1873 EVT SrcVT = Src.getValueType();
1874 unsigned InBits = SrcVT.getScalarSizeInBits();
1875 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1876 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1877
1878 // If none of the top bits are demanded, convert this into an any_extend.
1879 if (DemandedBits.getActiveBits() <= InBits) {
1880 // If we only need the non-extended bits of the bottom element
1881 // then we can just bitcast to the result.
1882 if (IsVecInReg && DemandedElts == 1 &&
1883 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1884 TLO.DAG.getDataLayout().isLittleEndian())
1885 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1886
1887 unsigned Opc =
1888 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1889 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1890 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1891 }
1892
1893 APInt InDemandedBits = DemandedBits.trunc(InBits);
1894 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1895 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1896 Depth + 1))
1897 return true;
1898 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1899 assert(Known.getBitWidth() == InBits && "Src width has changed?")((void)0);
1900 Known = Known.zext(BitWidth);
1901
1902 // Attempt to avoid multi-use ops if we don't need anything from them.
1903 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1904 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1905 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1906 break;
1907 }
1908 case ISD::SIGN_EXTEND:
1909 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1910 SDValue Src = Op.getOperand(0);
1911 EVT SrcVT = Src.getValueType();
1912 unsigned InBits = SrcVT.getScalarSizeInBits();
1913 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1914 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1915
1916 // If none of the top bits are demanded, convert this into an any_extend.
1917 if (DemandedBits.getActiveBits() <= InBits) {
1918 // If we only need the non-extended bits of the bottom element
1919 // then we can just bitcast to the result.
1920 if (IsVecInReg && DemandedElts == 1 &&
1921 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1922 TLO.DAG.getDataLayout().isLittleEndian())
1923 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1924
1925 unsigned Opc =
1926 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1927 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1928 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1929 }
1930
1931 APInt InDemandedBits = DemandedBits.trunc(InBits);
1932 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1933
1934 // Since some of the sign extended bits are demanded, we know that the sign
1935 // bit is demanded.
1936 InDemandedBits.setBit(InBits - 1);
1937
1938 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1939 Depth + 1))
1940 return true;
1941 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1942 assert(Known.getBitWidth() == InBits && "Src width has changed?")((void)0);
1943
1944 // If the sign bit is known one, the top bits match.
1945 Known = Known.sext(BitWidth);
1946
1947 // If the sign bit is known zero, convert this to a zero extend.
1948 if (Known.isNonNegative()) {
1949 unsigned Opc =
1950 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1951 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1952 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1953 }
1954
1955 // Attempt to avoid multi-use ops if we don't need anything from them.
1956 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1957 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1958 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1959 break;
1960 }
1961 case ISD::ANY_EXTEND:
1962 case ISD::ANY_EXTEND_VECTOR_INREG: {
1963 SDValue Src = Op.getOperand(0);
1964 EVT SrcVT = Src.getValueType();
1965 unsigned InBits = SrcVT.getScalarSizeInBits();
1966 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1967 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1968
1969 // If we only need the bottom element then we can just bitcast.
1970 // TODO: Handle ANY_EXTEND?
1971 if (IsVecInReg && DemandedElts == 1 &&
1972 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1973 TLO.DAG.getDataLayout().isLittleEndian())
1974 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1975
1976 APInt InDemandedBits = DemandedBits.trunc(InBits);
1977 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1978 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1979 Depth + 1))
1980 return true;
1981 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1982 assert(Known.getBitWidth() == InBits && "Src width has changed?")((void)0);
1983 Known = Known.anyext(BitWidth);
1984
1985 // Attempt to avoid multi-use ops if we don't need anything from them.
1986 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1987 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1988 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1989 break;
1990 }
1991 case ISD::TRUNCATE: {
1992 SDValue Src = Op.getOperand(0);
1993
1994 // Simplify the input, using demanded bit information, and compute the known
1995 // zero/one bits live out.
1996 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1997 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1998 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
1999 Depth + 1))
2000 return true;
2001 Known = Known.trunc(BitWidth);
2002
2003 // Attempt to avoid multi-use ops if we don't need anything from them.
2004 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2005 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2006 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2007
2008 // If the input is only used by this truncate, see if we can shrink it based
2009 // on the known demanded bits.
2010 if (Src.getNode()->hasOneUse()) {
2011 switch (Src.getOpcode()) {
2012 default:
2013 break;
2014 case ISD::SRL:
2015 // Shrink SRL by a constant if none of the high bits shifted in are
2016 // demanded.
2017 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2018 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2019 // undesirable.
2020 break;
2021
2022 const APInt *ShAmtC =
2023 TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2024 if (!ShAmtC || ShAmtC->uge(BitWidth))
2025 break;
2026 uint64_t ShVal = ShAmtC->getZExtValue();
2027
2028 APInt HighBits =
2029 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2030 HighBits.lshrInPlace(ShVal);
2031 HighBits = HighBits.trunc(BitWidth);
2032
2033 if (!(HighBits & DemandedBits)) {
2034 // None of the shifted in bits are needed. Add a truncate of the
2035 // shift input, then shift it.
2036 SDValue NewShAmt = TLO.DAG.getConstant(
2037 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2038 SDValue NewTrunc =
2039 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2040 return TLO.CombineTo(
2041 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2042 }
2043 break;
2044 }
2045 }
2046
2047 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
2048 break;
2049 }
2050 case ISD::AssertZext: {
2051 // AssertZext demands all of the high bits, plus any of the low bits
2052 // demanded by its users.
2053 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2054 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
2055 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2056 TLO, Depth + 1))
2057 return true;
2058 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
2059
2060 Known.Zero |= ~InMask;
2061 break;
2062 }
2063 case ISD::EXTRACT_VECTOR_ELT: {
2064 SDValue Src = Op.getOperand(0);
2065 SDValue Idx = Op.getOperand(1);
2066 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2067 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2068
2069 if (SrcEltCnt.isScalable())
2070 return false;
2071
2072 // Demand the bits from every vector element without a constant index.
2073 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2074 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
2075 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2076 if (CIdx->getAPIntValue().ult(NumSrcElts))
2077 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2078
2079 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2080 // anything about the extended bits.
2081 APInt DemandedSrcBits = DemandedBits;
2082 if (BitWidth > EltBitWidth)
2083 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2084
2085 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2086 Depth + 1))
2087 return true;
2088
2089 // Attempt to avoid multi-use ops if we don't need anything from them.
2090 if (!DemandedSrcBits.isAllOnesValue() ||
2091 !DemandedSrcElts.isAllOnesValue()) {
2092 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2093 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2094 SDValue NewOp =
2095 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2096 return TLO.CombineTo(Op, NewOp);
2097 }
2098 }
2099
2100 Known = Known2;
2101 if (BitWidth > EltBitWidth)
2102 Known = Known.anyext(BitWidth);
2103 break;
2104 }
2105 case ISD::BITCAST: {
2106 SDValue Src = Op.getOperand(0);
2107 EVT SrcVT = Src.getValueType();
2108 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2109
2110 // If this is an FP->Int bitcast and if the sign bit is the only
2111 // thing demanded, turn this into a FGETSIGN.
2112 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2113 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2114 SrcVT.isFloatingPoint()) {
2115 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2116 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2117 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2118 SrcVT != MVT::f128) {
2119 // Cannot eliminate/lower SHL for f128 yet.
2120 EVT Ty = OpVTLegal ? VT : MVT::i32;
2121 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2122 // place. We expect the SHL to be eliminated by other optimizations.
2123 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2124 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2125 if (!OpVTLegal && OpVTSizeInBits > 32)
2126 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2127 unsigned ShVal = Op.getValueSizeInBits() - 1;
2128 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2129 return TLO.CombineTo(Op,
2130 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2131 }
2132 }
2133
2134 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2135 // Demand the elt/bit if any of the original elts/bits are demanded.
2136 // TODO - bigendian once we have test coverage.
2137 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2138 TLO.DAG.getDataLayout().isLittleEndian()) {
2139 unsigned Scale = BitWidth / NumSrcEltBits;
2140 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2141 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2142 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2143 for (unsigned i = 0; i != Scale; ++i) {
2144 unsigned Offset = i * NumSrcEltBits;
2145 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2146 if (!Sub.isNullValue()) {
2147 DemandedSrcBits |= Sub;
2148 for (unsigned j = 0; j != NumElts; ++j)
2149 if (DemandedElts[j])
2150 DemandedSrcElts.setBit((j * Scale) + i);
2151 }
2152 }
2153
2154 APInt KnownSrcUndef, KnownSrcZero;
2155 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2156 KnownSrcZero, TLO, Depth + 1))
2157 return true;
2158
2159 KnownBits KnownSrcBits;
2160 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2161 KnownSrcBits, TLO, Depth + 1))
2162 return true;
2163 } else if ((NumSrcEltBits % BitWidth) == 0 &&
2164 TLO.DAG.getDataLayout().isLittleEndian()) {
2165 unsigned Scale = NumSrcEltBits / BitWidth;
2166 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2167 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2168 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2169 for (unsigned i = 0; i != NumElts; ++i)
2170 if (DemandedElts[i]) {
2171 unsigned Offset = (i % Scale) * BitWidth;
2172 DemandedSrcBits.insertBits(DemandedBits, Offset);
2173 DemandedSrcElts.setBit(i / Scale);
2174 }
2175
2176 if (SrcVT.isVector()) {
2177 APInt KnownSrcUndef, KnownSrcZero;
2178 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2179 KnownSrcZero, TLO, Depth + 1))
2180 return true;
2181 }
2182
2183 KnownBits KnownSrcBits;
2184 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2185 KnownSrcBits, TLO, Depth + 1))
2186 return true;
2187 }
2188
2189 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2190 // recursive call where Known may be useful to the caller.
2191 if (Depth > 0) {
2192 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2193 return false;
2194 }
2195 break;
2196 }
2197 case ISD::ADD:
2198 case ISD::MUL:
2199 case ISD::SUB: {
2200 // Add, Sub, and Mul don't demand any bits in positions beyond that
2201 // of the highest bit demanded of them.
2202 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2203 SDNodeFlags Flags = Op.getNode()->getFlags();
2204 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2205 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2206 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2207 Depth + 1) ||
2208 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2209 Depth + 1) ||
2210 // See if the operation should be performed at a smaller bit width.
2211 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2212 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2213 // Disable the nsw and nuw flags. We can no longer guarantee that we
2214 // won't wrap after simplification.
2215 Flags.setNoSignedWrap(false);
2216 Flags.setNoUnsignedWrap(false);
2217 SDValue NewOp =
2218 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2219 return TLO.CombineTo(Op, NewOp);
2220 }
2221 return true;
2222 }
2223
2224 // Attempt to avoid multi-use ops if we don't need anything from them.
2225 if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
2226 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2227 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2228 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2229 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2230 if (DemandedOp0 || DemandedOp1) {
2231 Flags.setNoSignedWrap(false);
2232 Flags.setNoUnsignedWrap(false);
2233 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2234 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2235 SDValue NewOp =
2236 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2237 return TLO.CombineTo(Op, NewOp);
2238 }
2239 }
2240
2241 // If we have a constant operand, we may be able to turn it into -1 if we
2242 // do not demand the high bits. This can make the constant smaller to
2243 // encode, allow more general folding, or match specialized instruction
2244 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2245 // is probably not useful (and could be detrimental).
2246 ConstantSDNode *C = isConstOrConstSplat(Op1);
2247 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2248 if (C && !C->isAllOnesValue() && !C->isOne() &&
2249 (C->getAPIntValue() | HighMask).isAllOnesValue()) {
2250 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2251 // Disable the nsw and nuw flags. We can no longer guarantee that we
2252 // won't wrap after simplification.
2253 Flags.setNoSignedWrap(false);
2254 Flags.setNoUnsignedWrap(false);
2255 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2256 return TLO.CombineTo(Op, NewOp);
2257 }
2258
2259 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2260 }
2261 default:
2262 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2263 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2264 Known, TLO, Depth))
2265 return true;
2266 break;
2267 }
2268
2269 // Just use computeKnownBits to compute output bits.
2270 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2271 break;
2272 }
2273
2274 // If we know the value of all of the demanded bits, return this as a
2275 // constant.
2276 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2277 // Avoid folding to a constant if any OpaqueConstant is involved.
2278 const SDNode *N = Op.getNode();
2279 for (SDNode *Op :
2280 llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
2281 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2282 if (C->isOpaque())
2283 return false;
2284 }
2285 if (VT.isInteger())
2286 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2287 if (VT.isFloatingPoint())
2288 return TLO.CombineTo(
2289 Op,
2290 TLO.DAG.getConstantFP(
2291 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2292 }
2293
2294 return false;
2295}
2296
2297bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2298 const APInt &DemandedElts,
2299 APInt &KnownUndef,
2300 APInt &KnownZero,
2301 DAGCombinerInfo &DCI) const {
2302 SelectionDAG &DAG = DCI.DAG;
2303 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2304 !DCI.isBeforeLegalizeOps());
2305
2306 bool Simplified =
2307 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2308 if (Simplified) {
2309 DCI.AddToWorklist(Op.getNode());
2310 DCI.CommitTargetLoweringOpt(TLO);
2311 }
2312
2313 return Simplified;
2314}
2315
2316/// Given a vector binary operation and known undefined elements for each input
2317/// operand, compute whether each element of the output is undefined.
2318static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2319 const APInt &UndefOp0,
2320 const APInt &UndefOp1) {
2321 EVT VT = BO.getValueType();
2322 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&((void)0)
2323 "Vector binop only")((void)0);
2324
2325 EVT EltVT = VT.getVectorElementType();
2326 unsigned NumElts = VT.getVectorNumElements();
2327 assert(UndefOp0.getBitWidth() == NumElts &&((void)0)
2328 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")((void)0);
2329
2330 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2331 const APInt &UndefVals) {
2332 if (UndefVals[Index])
2333 return DAG.getUNDEF(EltVT);
2334
2335 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2336 // Try hard to make sure that the getNode() call is not creating temporary
2337 // nodes. Ignore opaque integers because they do not constant fold.
2338 SDValue Elt = BV->getOperand(Index);
2339 auto *C = dyn_cast<ConstantSDNode>(Elt);
2340 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2341 return Elt;
2342 }
2343
2344 return SDValue();
2345 };
2346
2347 APInt KnownUndef = APInt::getNullValue(NumElts);
2348 for (unsigned i = 0; i != NumElts; ++i) {
2349 // If both inputs for this element are either constant or undef and match
2350 // the element type, compute the constant/undef result for this element of
2351 // the vector.
2352 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2353 // not handle FP constants. The code within getNode() should be refactored
2354 // to avoid the danger of creating a bogus temporary node here.
2355 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2356 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2357 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2358 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2359 KnownUndef.setBit(i);
2360 }
2361 return KnownUndef;
2362}
2363
2364bool TargetLowering::SimplifyDemandedVectorElts(
2365 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2366 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2367 bool AssumeSingleUse) const {
2368 EVT VT = Op.getValueType();
2369 unsigned Opcode = Op.getOpcode();
2370 APInt DemandedElts = OriginalDemandedElts;
2371 unsigned NumElts = DemandedElts.getBitWidth();
2372 assert(VT.isVector() && "Expected vector op")((void)0);
2373
2374 KnownUndef = KnownZero = APInt::getNullValue(NumElts);
2375
2376 // TODO: For now we assume we know nothing about scalable vectors.
2377 if (VT.isScalableVector())
2378 return false;
2379
2380 assert(VT.getVectorNumElements() == NumElts &&((void)0)
2381 "Mask size mismatches value type element count!")((void)0);
2382
2383 // Undef operand.
2384 if (Op.isUndef()) {
2385 KnownUndef.setAllBits();
2386 return false;
2387 }
2388
2389 // If Op has other users, assume that all elements are needed.
2390 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2391 DemandedElts.setAllBits();
2392
2393 // Not demanding any elements from Op.
2394 if (DemandedElts == 0) {
2395 KnownUndef.setAllBits();
2396 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2397 }
2398
2399 // Limit search depth.
2400 if (Depth >= SelectionDAG::MaxRecursionDepth)
2401 return false;
2402
2403 SDLoc DL(Op);
2404 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2405
2406 // Helper for demanding the specified elements and all the bits of both binary
2407 // operands.
2408 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2409 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2410 TLO.DAG, Depth + 1);
2411 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2412 TLO.DAG, Depth + 1);
2413 if (NewOp0 || NewOp1) {
2414 SDValue NewOp = TLO.DAG.getNode(
2415 Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2416 return TLO.CombineTo(Op, NewOp);
2417 }
2418 return false;
2419 };
2420
2421 switch (Opcode) {
2422 case ISD::SCALAR_TO_VECTOR: {
2423 if (!DemandedElts[0]) {
2424 KnownUndef.setAllBits();
2425 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2426 }
2427 SDValue ScalarSrc = Op.getOperand(0);
2428 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2429 SDValue Src = ScalarSrc.getOperand(0);
2430 SDValue Idx = ScalarSrc.getOperand(1);
2431 EVT SrcVT = Src.getValueType();
2432
2433 ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2434
2435 if (SrcEltCnt.isScalable())
2436 return false;
2437
2438 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2439 if (isNullConstant(Idx)) {
2440 APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2441 APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2442 APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2443 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2444 TLO, Depth + 1))
2445 return true;
2446 }
2447 }
2448 KnownUndef.setHighBits(NumElts - 1);
2449 break;
2450 }
2451 case ISD::BITCAST: {
2452 SDValue Src = Op.getOperand(0);
2453 EVT SrcVT = Src.getValueType();
2454
2455 // We only handle vectors here.
2456 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2457 if (!SrcVT.isVector())
2458 break;
2459
2460 // Fast handling of 'identity' bitcasts.
2461 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2462 if (NumSrcElts == NumElts)
2463 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2464 KnownZero, TLO, Depth + 1);
2465
2466 APInt SrcZero, SrcUndef;
2467 APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
2468
2469 // Bitcast from 'large element' src vector to 'small element' vector, we
2470 // must demand a source element if any DemandedElt maps to it.
2471 if ((NumElts % NumSrcElts) == 0) {
2472 unsigned Scale = NumElts / NumSrcElts;
2473 for (unsigned i = 0; i != NumElts; ++i)
2474 if (DemandedElts[i])
2475 SrcDemandedElts.setBit(i / Scale);
2476
2477 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2478 TLO, Depth + 1))
2479 return true;
2480
2481 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2482 // of the large element.
2483 // TODO - bigendian once we have test coverage.
2484 if (TLO.DAG.getDataLayout().isLittleEndian()) {
2485 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2486 APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
2487 for (unsigned i = 0; i != NumElts; ++i)
2488 if (DemandedElts[i]) {
2489 unsigned Ofs = (i % Scale) * EltSizeInBits;
2490 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2491 }
2492
2493 KnownBits Known;
2494 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2495 TLO, Depth + 1))
2496 return true;
2497 }
2498
2499 // If the src element is zero/undef then all the output elements will be -
2500 // only demanded elements are guaranteed to be correct.
2501 for (unsigned i = 0; i != NumSrcElts; ++i) {
2502 if (SrcDemandedElts[i]) {
2503 if (SrcZero[i])
2504 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2505 if (SrcUndef[i])
2506 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2507 }
2508 }
2509 }
2510
2511 // Bitcast from 'small element' src vector to 'large element' vector, we
2512 // demand all smaller source elements covered by the larger demanded element
2513 // of this vector.
2514 if ((NumSrcElts % NumElts) == 0) {
2515 unsigned Scale = NumSrcElts / NumElts;
2516 for (unsigned i = 0; i != NumElts; ++i)
2517 if (DemandedElts[i])
2518 SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
2519
2520 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2521 TLO, Depth + 1))
2522 return true;
2523
2524 // If all the src elements covering an output element are zero/undef, then
2525 // the output element will be as well, assuming it was demanded.
2526 for (unsigned i = 0; i != NumElts; ++i) {
2527 if (DemandedElts[i]) {
2528 if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
2529 KnownZero.setBit(i);
2530 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
2531 KnownUndef.setBit(i);
2532 }
2533 }
2534 }
2535 break;
2536 }
2537 case ISD::BUILD_VECTOR: {
2538 // Check all elements and simplify any unused elements with UNDEF.
2539 if (!DemandedElts.isAllOnesValue()) {
2540 // Don't simplify BROADCASTS.
2541 if (llvm::any_of(Op->op_values(),
2542 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2543 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2544 bool Updated = false;
2545 for (unsigned i = 0; i != NumElts; ++i) {
2546 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2547 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2548 KnownUndef.setBit(i);
2549 Updated = true;
2550 }
2551 }
2552 if (Updated)
2553 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2554 }
2555 }
2556 for (unsigned i = 0; i != NumElts; ++i) {
2557 SDValue SrcOp = Op.getOperand(i);
2558 if (SrcOp.isUndef()) {
2559 KnownUndef.setBit(i);
2560 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2561 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2562 KnownZero.setBit(i);
2563 }
2564 }
2565 break;
2566 }
2567 case ISD::CONCAT_VECTORS: {
2568 EVT SubVT = Op.getOperand(0).getValueType();
2569 unsigned NumSubVecs = Op.getNumOperands();
2570 unsigned NumSubElts = SubVT.getVectorNumElements();
2571 for (unsigned i = 0; i != NumSubVecs; ++i) {
2572 SDValue SubOp = Op.getOperand(i);
2573 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2574 APInt SubUndef, SubZero;
2575 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2576 Depth + 1))
2577 return true;
2578 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2579 KnownZero.insertBits(SubZero, i * NumSubElts);
2580 }
2581 break;
2582 }
2583 case ISD::INSERT_SUBVECTOR: {
2584 // Demand any elements from the subvector and the remainder from the src its
2585 // inserted into.
2586 SDValue Src = Op.getOperand(0);
2587 SDValue Sub = Op.getOperand(1);
2588 uint64_t Idx = Op.getConstantOperandVal(2);
2589 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2590 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2591 APInt DemandedSrcElts = DemandedElts;
2592 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2593
2594 APInt SubUndef, SubZero;
2595 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2596 Depth + 1))
2597 return true;
2598
2599 // If none of the src operand elements are demanded, replace it with undef.
2600 if (!DemandedSrcElts && !Src.isUndef())
2601 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2602 TLO.DAG.getUNDEF(VT), Sub,
2603 Op.getOperand(2)));
2604
2605 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2606 TLO, Depth + 1))
2607 return true;
2608 KnownUndef.insertBits(SubUndef, Idx);
2609 KnownZero.insertBits(SubZero, Idx);
2610
2611 // Attempt to avoid multi-use ops if we don't need anything from them.
2612 if (!DemandedSrcElts.isAllOnesValue() ||
2613 !DemandedSubElts.isAllOnesValue()) {
2614 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2615 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2616 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2617 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2618 if (NewSrc || NewSub) {
2619 NewSrc = NewSrc ? NewSrc : Src;
2620 NewSub = NewSub ? NewSub : Sub;
2621 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2622 NewSub, Op.getOperand(2));
2623 return TLO.CombineTo(Op, NewOp);
2624 }
2625 }
2626 break;
2627 }
2628 case ISD::EXTRACT_SUBVECTOR: {
2629 // Offset the demanded elts by the subvector index.
2630 SDValue Src = Op.getOperand(0);
2631 if (Src.getValueType().isScalableVector())
2632 break;
2633 uint64_t Idx = Op.getConstantOperandVal(1);
2634 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2635 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2636
2637 APInt SrcUndef, SrcZero;
2638 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2639 Depth + 1))
2640 return true;
2641 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2642 KnownZero = SrcZero.extractBits(NumElts, Idx);
2643
2644 // Attempt to avoid multi-use ops if we don't need anything from them.
2645 if (!DemandedElts.isAllOnesValue()) {
2646 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2647 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2648 if (NewSrc) {
2649 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2650 Op.getOperand(1));
2651 return TLO.CombineTo(Op, NewOp);
2652 }
2653 }
2654 break;
2655 }
2656 case ISD::INSERT_VECTOR_ELT: {
2657 SDValue Vec = Op.getOperand(0);
2658 SDValue Scl = Op.getOperand(1);
2659 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2660
2661 // For a legal, constant insertion index, if we don't need this insertion
2662 // then strip it, else remove it from the demanded elts.
2663 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2664 unsigned Idx = CIdx->getZExtValue();
2665 if (!DemandedElts[Idx])
2666 return TLO.CombineTo(Op, Vec);
2667
2668 APInt DemandedVecElts(DemandedElts);
2669 DemandedVecElts.clearBit(Idx);
2670 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2671 KnownZero, TLO, Depth + 1))
2672 return true;
2673
2674 KnownUndef.setBitVal(Idx, Scl.isUndef());
2675
2676 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2677 break;
2678 }
2679
2680 APInt VecUndef, VecZero;
2681 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2682 Depth + 1))
2683 return true;
2684 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2685 break;
2686 }
2687 case ISD::VSELECT: {
2688 // Try to transform the select condition based on the current demanded
2689 // elements.
2690 // TODO: If a condition element is undef, we can choose from one arm of the
2691 // select (and if one arm is undef, then we can propagate that to the
2692 // result).
2693 // TODO - add support for constant vselect masks (see IR version of this).
2694 APInt UnusedUndef, UnusedZero;
2695 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2696 UnusedZero, TLO, Depth + 1))
2697 return true;
2698
2699 // See if we can simplify either vselect operand.
2700 APInt DemandedLHS(DemandedElts);
2701 APInt DemandedRHS(DemandedElts);
2702 APInt UndefLHS, ZeroLHS;
2703 APInt UndefRHS, ZeroRHS;
2704 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2705 ZeroLHS, TLO, Depth + 1))
2706 return true;
2707 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2708 ZeroRHS, TLO, Depth + 1))
2709 return true;
2710
2711 KnownUndef = UndefLHS & UndefRHS;
2712 KnownZero = ZeroLHS & ZeroRHS;
2713 break;
2714 }
2715 case ISD::VECTOR_SHUFFLE: {
2716 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2717
2718 // Collect demanded elements from shuffle operands..
2719 APInt DemandedLHS(NumElts, 0);
2720 APInt DemandedRHS(NumElts, 0);
2721 for (unsigned i = 0; i != NumElts; ++i) {
2722 int M = ShuffleMask[i];
2723 if (M < 0 || !DemandedElts[i])
2724 continue;
2725 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((void)0);
2726 if (M < (int)NumElts)
2727 DemandedLHS.setBit(M);
2728 else
2729 DemandedRHS.setBit(M - NumElts);
2730 }
2731
2732 // See if we can simplify either shuffle operand.
2733 APInt UndefLHS, ZeroLHS;
2734 APInt UndefRHS, ZeroRHS;
2735 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2736 ZeroLHS, TLO, Depth + 1))
2737 return true;
2738 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2739 ZeroRHS, TLO, Depth + 1))
2740 return true;
2741
2742 // Simplify mask using undef elements from LHS/RHS.
2743 bool Updated = false;
2744 bool IdentityLHS = true, IdentityRHS = true;
2745 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2746 for (unsigned i = 0; i != NumElts; ++i) {
2747 int &M = NewMask[i];
2748 if (M < 0)
2749 continue;
2750 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2751 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2752 Updated = true;
2753 M = -1;
2754 }
2755 IdentityLHS &= (M < 0) || (M == (int)i);
2756 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2757 }
2758
2759 // Update legal shuffle masks based on demanded elements if it won't reduce
2760 // to Identity which can cause premature removal of the shuffle mask.
2761 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2762 SDValue LegalShuffle =
2763 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2764 NewMask, TLO.DAG);
2765 if (LegalShuffle)
2766 return TLO.CombineTo(Op, LegalShuffle);
2767 }
2768
2769 // Propagate undef/zero elements from LHS/RHS.
2770 for (unsigned i = 0; i != NumElts; ++i) {
2771 int M = ShuffleMask[i];
2772 if (M < 0) {
2773 KnownUndef.setBit(i);
2774 } else if (M < (int)NumElts) {
2775 if (UndefLHS[M])
2776 KnownUndef.setBit(i);
2777 if (ZeroLHS[M])
2778 KnownZero.setBit(i);
2779 } else {
2780 if (UndefRHS[M - NumElts])
2781 KnownUndef.setBit(i);
2782 if (ZeroRHS[M - NumElts])
2783 KnownZero.setBit(i);
2784 }
2785 }
2786 break;
2787 }
2788 case ISD::ANY_EXTEND_VECTOR_INREG:
2789 case ISD::SIGN_EXTEND_VECTOR_INREG:
2790 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2791 APInt SrcUndef, SrcZero;
2792 SDValue Src = Op.getOperand(0);
2793 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2794 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2795 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2796 Depth + 1))
2797 return true;
2798 KnownZero = SrcZero.zextOrTrunc(NumElts);
2799 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2800
2801 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2802 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2803 DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2804 // aext - if we just need the bottom element then we can bitcast.
2805 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2806 }
2807
2808 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2809 // zext(undef) upper bits are guaranteed to be zero.
2810 if (DemandedElts.isSubsetOf(KnownUndef))
2811 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2812 KnownUndef.clearAllBits();
2813 }
2814 break;
2815 }
2816
2817 // TODO: There are more binop opcodes that could be handled here - MIN,
2818 // MAX, saturated math, etc.
2819 case ISD::OR:
2820 case ISD::XOR:
2821 case ISD::ADD:
2822 case ISD::SUB:
2823 case ISD::FADD:
2824 case ISD::FSUB:
2825 case ISD::FMUL:
2826 case ISD::FDIV:
2827 case ISD::FREM: {
2828 SDValue Op0 = Op.getOperand(0);
2829 SDValue Op1 = Op.getOperand(1);
2830
2831 APInt UndefRHS, ZeroRHS;
2832 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2833 Depth + 1))
2834 return true;
2835 APInt UndefLHS, ZeroLHS;
2836 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2837 Depth + 1))
2838 return true;
2839
2840 KnownZero = ZeroLHS & ZeroRHS;
2841 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2842
2843 // Attempt to avoid multi-use ops if we don't need anything from them.
2844 // TODO - use KnownUndef to relax the demandedelts?
2845 if (!DemandedElts.isAllOnesValue())
2846 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2847 return true;
2848 break;
2849 }
2850 case ISD::SHL:
2851 case ISD::SRL:
2852 case ISD::SRA:
2853 case ISD::ROTL:
2854 case ISD::ROTR: {
2855 SDValue Op0 = Op.getOperand(0);
2856 SDValue Op1 = Op.getOperand(1);
2857
2858 APInt UndefRHS, ZeroRHS;
2859 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2860 Depth + 1))
2861 return true;
2862 APInt UndefLHS, ZeroLHS;
2863 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2864 Depth + 1))
2865 return true;
2866
2867 KnownZero = ZeroLHS;
2868 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2869
2870 // Attempt to avoid multi-use ops if we don't need anything from them.
2871 // TODO - use KnownUndef to relax the demandedelts?
2872 if (!DemandedElts.isAllOnesValue())
2873 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2874 return true;
2875 break;
2876 }
2877 case ISD::MUL:
2878 case ISD::AND: {
2879 SDValue Op0 = Op.getOperand(0);
2880 SDValue Op1 = Op.getOperand(1);
2881
2882 APInt SrcUndef, SrcZero;
2883 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2884 Depth + 1))
2885 return true;
2886 if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2887 TLO, Depth + 1))
2888 return true;
2889
2890 // If either side has a zero element, then the result element is zero, even
2891 // if the other is an UNDEF.
2892 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2893 // and then handle 'and' nodes with the rest of the binop opcodes.
2894 KnownZero |= SrcZero;
2895 KnownUndef &= SrcUndef;
2896 KnownUndef &= ~KnownZero;
2897
2898 // Attempt to avoid multi-use ops if we don't need anything from them.
2899 // TODO - use KnownUndef to relax the demandedelts?
2900 if (!DemandedElts.isAllOnesValue())
2901 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2902 return true;
2903 break;
2904 }
2905 case ISD::TRUNCATE:
2906 case ISD::SIGN_EXTEND:
2907 case ISD::ZERO_EXTEND:
2908 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2909 KnownZero, TLO, Depth + 1))
2910 return true;
2911
2912 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2913 // zext(undef) upper bits are guaranteed to be zero.
2914 if (DemandedElts.isSubsetOf(KnownUndef))
2915 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2916 KnownUndef.clearAllBits();
2917 }
2918 break;
2919 default: {
2920 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2921 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2922 KnownZero, TLO, Depth))
2923 return true;
2924 } else {
2925 KnownBits Known;
2926 APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2927 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2928 TLO, Depth, AssumeSingleUse))
2929 return true;
2930 }
2931 break;
2932 }
2933 }
2934 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")((void)0);
2935
2936 // Constant fold all undef cases.
2937 // TODO: Handle zero cases as well.
2938 if (DemandedElts.isSubsetOf(KnownUndef))
2939 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2940
2941 return false;
2942}
2943
2944/// Determine which of the bits specified in Mask are known to be either zero or
2945/// one and return them in the Known.
2946void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2947 KnownBits &Known,
2948 const APInt &DemandedElts,
2949 const SelectionDAG &DAG,
2950 unsigned Depth) const {
2951 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
2952 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
2953 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
2954 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
2955 "Should use MaskedValueIsZero if you don't know whether Op"((void)0)
2956 " is a target node!")((void)0);
2957 Known.resetAll();
2958}
2959
2960void TargetLowering::computeKnownBitsForTargetInstr(
2961 GISelKnownBits &Analysis, Register R, KnownBits &Known,
2962 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2963 unsigned Depth) const {
2964 Known.resetAll();
2965}
2966
2967void TargetLowering::computeKnownBitsForFrameIndex(
2968 const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
2969 // The low bits are known zero if the pointer is aligned.
2970 Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2971}
2972
2973Align TargetLowering::computeKnownAlignForTargetInstr(
2974 GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
2975 unsigned Depth) const {
2976 return Align(1);
2977}
2978
2979/// This method can be implemented by targets that want to expose additional
2980/// information about sign bits to the DAG Combiner.
2981unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2982 const APInt &,
2983 const SelectionDAG &,
2984 unsigned Depth) const {
2985 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
2986 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
2987 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
2988 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
2989 "Should use ComputeNumSignBits if you don't know whether Op"((void)0)
2990 " is a target node!")((void)0);
2991 return 1;
2992}
2993
2994unsigned TargetLowering::computeNumSignBitsForTargetInstr(
2995 GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
2996 const MachineRegisterInfo &MRI, unsigned Depth) const {
2997 return 1;
2998}
2999
3000bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
3001 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3002 TargetLoweringOpt &TLO, unsigned Depth) const {
3003 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3004 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3005 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3006 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3007 "Should use SimplifyDemandedVectorElts if you don't know whether Op"((void)0)
3008 " is a target node!")((void)0);
3009 return false;
3010}
3011
3012bool TargetLowering::SimplifyDemandedBitsForTargetNode(
3013 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3014 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3015 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3016 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3017 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3018 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3019 "Should use SimplifyDemandedBits if you don't know whether Op"((void)0)
3020 " is a target node!")((void)0);
3021 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3022 return false;
3023}
3024
3025SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
3026 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3027 SelectionDAG &DAG, unsigned Depth) const {
3028 assert(((void)0)
3029 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3030 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3031 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3032 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3033 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"((void)0)
3034 " is a target node!")((void)0);
3035 return SDValue();
3036}
3037
3038SDValue
3039TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3040 SDValue N1, MutableArrayRef<int> Mask,
3041 SelectionDAG &DAG) const {
3042 bool LegalMask = isShuffleMaskLegal(Mask, VT);
3043 if (!LegalMask) {
3044 std::swap(N0, N1);
3045 ShuffleVectorSDNode::commuteMask(Mask);
3046 LegalMask = isShuffleMaskLegal(Mask, VT);
3047 }
3048
3049 if (!LegalMask)
3050 return SDValue();
3051
3052 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3053}
3054
3055const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
3056 return nullptr;
3057}
3058
3059bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
3060 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3061 bool PoisonOnly, unsigned Depth) const {
3062 assert(((void)0)
3063 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3064 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3065 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3066 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3067 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"((void)0)
3068 " is a target node!")((void)0);
3069 return false;
3070}
3071
3072bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3073 const SelectionDAG &DAG,
3074 bool SNaN,
3075 unsigned Depth) const {
3076 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3077 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3078 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3079 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3080 "Should use isKnownNeverNaN if you don't know whether Op"((void)0)
3081 " is a target node!")((void)0);
3082 return false;
3083}
3084
3085// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3086// work with truncating build vectors and vectors with elements of less than
3087// 8 bits.
3088bool TargetLowering::isConstTrueVal(const SDNode *N) const {
3089 if (!N)
3090 return false;
3091
3092 APInt CVal;
3093 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3094 CVal = CN->getAPIntValue();
3095 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3096 auto *CN = BV->getConstantSplatNode();
3097 if (!CN)
3098 return false;
3099
3100 // If this is a truncating build vector, truncate the splat value.
3101 // Otherwise, we may fail to match the expected values below.
3102 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3103 CVal = CN->getAPIntValue();
3104 if (BVEltWidth < CVal.getBitWidth())
3105 CVal = CVal.trunc(BVEltWidth);
3106 } else {
3107 return false;
3108 }
3109
3110 switch (getBooleanContents(N->getValueType(0))) {
3111 case UndefinedBooleanContent:
3112 return CVal[0];
3113 case ZeroOrOneBooleanContent:
3114 return CVal.isOneValue();
3115 case ZeroOrNegativeOneBooleanContent:
3116 return CVal.isAllOnesValue();
3117 }
3118
3119 llvm_unreachable("Invalid boolean contents")__builtin_unreachable();
3120}
3121
3122bool TargetLowering::isConstFalseVal(const SDNode *N) const {
3123 if (!N)
3124 return false;
3125
3126 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3127 if (!CN) {
3128 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3129 if (!BV)
3130 return false;
3131
3132 // Only interested in constant splats, we don't care about undef
3133 // elements in identifying boolean constants and getConstantSplatNode
3134 // returns NULL if all ops are undef;
3135 CN = BV->getConstantSplatNode();
3136 if (!CN)
3137 return false;
3138 }
3139
3140 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3141 return !CN->getAPIntValue()[0];
3142
3143 return CN->isNullValue();
3144}
3145
3146bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
3147 bool SExt) const {
3148 if (VT == MVT::i1)
3149 return N->isOne();
3150
3151 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
3152 switch (Cnt) {
3153 case TargetLowering::ZeroOrOneBooleanContent:
3154 // An extended value of 1 is always true, unless its original type is i1,
3155 // in which case it will be sign extended to -1.
3156 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3157 case TargetLowering::UndefinedBooleanContent:
3158 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3159 return N->isAllOnesValue() && SExt;
3160 }
3161 llvm_unreachable("Unexpected enumeration.")__builtin_unreachable();
3162}
3163
3164/// This helper function of SimplifySetCC tries to optimize the comparison when
3165/// either operand of the SetCC node is a bitwise-and instruction.
3166SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3167 ISD::CondCode Cond, const SDLoc &DL,
3168 DAGCombinerInfo &DCI) const {
3169 // Match these patterns in any of their permutations:
3170 // (X & Y) == Y
3171 // (X & Y) != Y
3172 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3173 std::swap(N0, N1);
3174
3175 EVT OpVT = N0.getValueType();
3176 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3177 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3178 return SDValue();
3179
3180 SDValue X, Y;
3181 if (N0.getOperand(0) == N1) {
3182 X = N0.getOperand(1);
3183 Y = N0.getOperand(0);
3184 } else if (N0.getOperand(1) == N1) {
3185 X = N0.getOperand(0);
3186 Y = N0.getOperand(1);
3187 } else {
3188 return SDValue();
3189 }
3190
3191 SelectionDAG &DAG = DCI.DAG;
3192 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3193 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3194 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3195 // Note that where Y is variable and is known to have at most one bit set
3196 // (for example, if it is Z & 1) we cannot do this; the expressions are not
3197 // equivalent when Y == 0.
3198 assert(OpVT.isInteger())((void)0);
3199 Cond = ISD::getSetCCInverse(Cond, OpVT);
3200 if (DCI.isBeforeLegalizeOps() ||
3201 isCondCodeLegal(Cond, N0.getSimpleValueType()))
3202 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3203 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3204 // If the target supports an 'and-not' or 'and-complement' logic operation,
3205 // try to use that to make a comparison operation more efficient.
3206 // But don't do this transform if the mask is a single bit because there are
3207 // more efficient ways to deal with that case (for example, 'bt' on x86 or
3208 // 'rlwinm' on PPC).
3209
3210 // Bail out if the compare operand that we want to turn into a zero is
3211 // already a zero (otherwise, infinite loop).
3212 auto *YConst = dyn_cast<ConstantSDNode>(Y);
3213 if (YConst && YConst->isNullValue())
3214 return SDValue();
3215
3216 // Transform this into: ~X & Y == 0.
3217 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3218 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3219 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3220 }
3221
3222 return SDValue();
3223}
3224
3225/// There are multiple IR patterns that could be checking whether certain
3226/// truncation of a signed number would be lossy or not. The pattern which is
3227/// best at IR level, may not lower optimally. Thus, we want to unfold it.
3228/// We are looking for the following pattern: (KeptBits is a constant)
3229/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3230/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3231/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3232/// We will unfold it into the natural trunc+sext pattern:
3233/// ((%x << C) a>> C) dstcond %x
3234/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3235SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3236 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3237 const SDLoc &DL) const {
3238 // We must be comparing with a constant.
3239 ConstantSDNode *C1;
3240 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3241 return SDValue();
3242
3243 // N0 should be: add %x, (1 << (KeptBits-1))
3244 if (N0->getOpcode() != ISD::ADD)
3245 return SDValue();
3246
3247 // And we must be 'add'ing a constant.
3248 ConstantSDNode *C01;
3249 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3250 return SDValue();
3251
3252 SDValue X = N0->getOperand(0);
3253 EVT XVT = X.getValueType();
3254
3255 // Validate constants ...
3256
3257 APInt I1 = C1->getAPIntValue();
3258
3259 ISD::CondCode NewCond;
3260 if (Cond == ISD::CondCode::SETULT) {
3261 NewCond = ISD::CondCode::SETEQ;
3262 } else if (Cond == ISD::CondCode::SETULE) {
3263 NewCond = ISD::CondCode::SETEQ;
3264 // But need to 'canonicalize' the constant.
3265 I1 += 1;
3266 } else if (Cond == ISD::CondCode::SETUGT) {
3267 NewCond = ISD::CondCode::SETNE;
3268 // But need to 'canonicalize' the constant.
3269 I1 += 1;
3270 } else if (Cond == ISD::CondCode::SETUGE) {
3271 NewCond = ISD::CondCode::SETNE;
3272 } else
3273 return SDValue();
3274
3275 APInt I01 = C01->getAPIntValue();
3276
3277 auto checkConstants = [&I1, &I01]() -> bool {
3278 // Both of them must be power-of-two, and the constant from setcc is bigger.
3279 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3280 };
3281
3282 if (checkConstants()) {
3283 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3284 } else {
3285 // What if we invert constants? (and the target predicate)
3286 I1.negate();
3287 I01.negate();
3288 assert(XVT.isInteger())((void)0);
3289 NewCond = getSetCCInverse(NewCond, XVT);
3290 if (!checkConstants())
3291 return SDValue();
3292 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3293 }
3294
3295 // They are power-of-two, so which bit is set?
3296 const unsigned KeptBits = I1.logBase2();
3297 const unsigned KeptBitsMinusOne = I01.logBase2();
3298
3299 // Magic!
3300 if (KeptBits != (KeptBitsMinusOne + 1))
3301 return SDValue();
3302 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")((void)0);
3303
3304 // We don't want to do this in every single case.
3305 SelectionDAG &DAG = DCI.DAG;
3306 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3307 XVT, KeptBits))
3308 return SDValue();
3309
3310 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3311 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")((void)0);
3312
3313 // Unfold into: ((%x << C) a>> C) cond %x
3314 // Where 'cond' will be either 'eq' or 'ne'.
3315 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3316 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3317 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3318 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3319
3320 return T2;
3321}
3322
3323// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3324SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3325 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3326 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3327 assert(isConstOrConstSplat(N1C) &&((void)0)
3328 isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&((void)0)
3329 "Should be a comparison with 0.")((void)0);
3330 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&((void)0)
3331 "Valid only for [in]equality comparisons.")((void)0);
3332
3333 unsigned NewShiftOpcode;
3334 SDValue X, C, Y;
3335
3336 SelectionDAG &DAG = DCI.DAG;
3337 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3338
3339 // Look for '(C l>>/<< Y)'.
3340 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3341 // The shift should be one-use.
3342 if (!V.hasOneUse())
3343 return false;
3344 unsigned OldShiftOpcode = V.getOpcode();
3345 switch (OldShiftOpcode) {
3346 case ISD::SHL:
3347 NewShiftOpcode = ISD::SRL;
3348 break;
3349 case ISD::SRL:
3350 NewShiftOpcode = ISD::SHL;
3351 break;
3352 default:
3353 return false; // must be a logical shift.
3354 }
3355 // We should be shifting a constant.
3356 // FIXME: best to use isConstantOrConstantVector().
3357 C = V.getOperand(0);
3358 ConstantSDNode *CC =
3359 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3360 if (!CC)
3361 return false;
3362 Y = V.getOperand(1);
3363
3364 ConstantSDNode *XC =
3365 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3366 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3367 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3368 };
3369
3370 // LHS of comparison should be an one-use 'and'.
3371 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3372 return SDValue();
3373
3374 X = N0.getOperand(0);
3375 SDValue Mask = N0.getOperand(1);
3376
3377 // 'and' is commutative!
3378 if (!Match(Mask)) {
3379 std::swap(X, Mask);
3380 if (!Match(Mask))
3381 return SDValue();
3382 }
3383
3384 EVT VT = X.getValueType();
3385
3386 // Produce:
3387 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3388 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3389 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3390 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3391 return T2;
3392}
3393
3394/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3395/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3396/// handle the commuted versions of these patterns.
3397SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3398 ISD::CondCode Cond, const SDLoc &DL,
3399 DAGCombinerInfo &DCI) const {
3400 unsigned BOpcode = N0.getOpcode();
3401 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&((void)0)
3402 "Unexpected binop")((void)0);
3403 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")((void)0);
3404
3405 // (X + Y) == X --> Y == 0
3406 // (X - Y) == X --> Y == 0
3407 // (X ^ Y) == X --> Y == 0
3408 SelectionDAG &DAG = DCI.DAG;
3409 EVT OpVT = N0.getValueType();
3410 SDValue X = N0.getOperand(0);
3411 SDValue Y = N0.getOperand(1);
3412 if (X == N1)
3413 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3414
3415 if (Y != N1)
3416 return SDValue();
3417
3418 // (X + Y) == Y --> X == 0
3419 // (X ^ Y) == Y --> X == 0
3420 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3421 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3422
3423 // The shift would not be valid if the operands are boolean (i1).
3424 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3425 return SDValue();
3426
3427 // (X - Y) == Y --> X == Y << 1
3428 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3429 !DCI.isBeforeLegalize());
3430 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3431 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3432 if (!DCI.isCalledByLegalizer())
3433 DCI.AddToWorklist(YShl1.getNode());
3434 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3435}
3436
3437static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
3438 SDValue N0, const APInt &C1,
3439 ISD::CondCode Cond, const SDLoc &dl,
3440 SelectionDAG &DAG) {
3441 // Look through truncs that don't change the value of a ctpop.
3442 // FIXME: Add vector support? Need to be careful with setcc result type below.
3443 SDValue CTPOP = N0;
3444 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3445 N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
3446 CTPOP = N0.getOperand(0);
3447
3448 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3449 return SDValue();
3450
3451 EVT CTVT = CTPOP.getValueType();
3452 SDValue CTOp = CTPOP.getOperand(0);
3453
3454 // If this is a vector CTPOP, keep the CTPOP if it is legal.
3455 // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3456 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3457 return SDValue();
3458
3459 // (ctpop x) u< 2 -> (x & x-1) == 0
3460 // (ctpop x) u> 1 -> (x & x-1) != 0
3461 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3462 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3463 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3464 return SDValue();
3465 if (C1 == 0 && (Cond == ISD::SETULT))
3466 return SDValue(); // This is handled elsewhere.
3467
3468 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3469
3470 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3471 SDValue Result = CTOp;
3472 for (unsigned i = 0; i < Passes; i++) {
3473 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3474 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3475 }
3476 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3477 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3478 }
3479
3480 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3481 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3482 // For scalars, keep CTPOP if it is legal or custom.
3483 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3484 return SDValue();
3485 // This is based on X86's custom lowering for CTPOP which produces more
3486 // instructions than the expansion here.
3487
3488 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3489 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3490 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3491 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3492 assert(CTVT.isInteger())((void)0);
3493 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3494 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3495 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3496 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3497 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3498 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3499 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3500 }
3501
3502 return SDValue();
3503}
3504
3505/// Try to simplify a setcc built with the specified operands and cc. If it is
3506/// unable to simplify it, return a null SDValue.
3507SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3508 ISD::CondCode Cond, bool foldBooleans,
3509 DAGCombinerInfo &DCI,
3510 const SDLoc &dl) const {
3511 SelectionDAG &DAG = DCI.DAG;
3512 const DataLayout &Layout = DAG.getDataLayout();
3513 EVT OpVT = N0.getValueType();
3514
3515 // Constant fold or commute setcc.
3516 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3517 return Fold;
3518
3519 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3520 // TODO: Handle non-splat vector constants. All undef causes trouble.
3521 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3522 // infinite loop here when we encounter one.
3523 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3524 if (isConstOrConstSplat(N0) &&
3525 (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3526 (DCI.isBeforeLegalizeOps() ||
3527 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3528 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3529
3530 // If we have a subtract with the same 2 non-constant operands as this setcc
3531 // -- but in reverse order -- then try to commute the operands of this setcc
3532 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3533 // instruction on some targets.
3534 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3535 (DCI.isBeforeLegalizeOps() ||
3536 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3537 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3538 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3539 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3540
3541 if (auto *N1C = isConstOrConstSplat(N1)) {
3542 const APInt &C1 = N1C->getAPIntValue();
3543
3544 // Optimize some CTPOP cases.
3545 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3546 return V;
3547
3548 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3549 // equality comparison, then we're just comparing whether X itself is
3550 // zero.
3551 if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3552 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3553 isPowerOf2_32(N0.getScalarValueSizeInBits())) {
3554 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3555 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3556 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3557 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3558 // (srl (ctlz x), 5) == 0 -> X != 0
3559 // (srl (ctlz x), 5) != 1 -> X != 0
3560 Cond = ISD::SETNE;
3561 } else {
3562 // (srl (ctlz x), 5) != 0 -> X == 0
3563 // (srl (ctlz x), 5) == 1 -> X == 0
3564 Cond = ISD::SETEQ;
3565 }
3566 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3567 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3568 Cond);
3569 }
3570 }
3571 }
3572 }
3573
3574 // FIXME: Support vectors.
3575 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3576 const APInt &C1 = N1C->getAPIntValue();
3577
3578 // (zext x) == C --> x == (trunc C)
3579 // (sext x) == C --> x == (trunc C)
3580 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3581 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3582 unsigned MinBits = N0.getValueSizeInBits();
3583 SDValue PreExt;
3584 bool Signed = false;
3585 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3586 // ZExt
3587 MinBits = N0->getOperand(0).getValueSizeInBits();
3588 PreExt = N0->getOperand(0);
3589 } else if (N0->getOpcode() == ISD::AND) {
3590 // DAGCombine turns costly ZExts into ANDs
3591 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3592 if ((C->getAPIntValue()+1).isPowerOf2()) {
3593 MinBits = C->getAPIntValue().countTrailingOnes();
3594 PreExt = N0->getOperand(0);
3595 }
3596 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3597 // SExt
3598 MinBits = N0->getOperand(0).getValueSizeInBits();
3599 PreExt = N0->getOperand(0);
3600 Signed = true;
3601 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3602 // ZEXTLOAD / SEXTLOAD
3603 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3604 MinBits = LN0->getMemoryVT().getSizeInBits();
3605 PreExt = N0;
3606 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3607 Signed = true;
3608 MinBits = LN0->getMemoryVT().getSizeInBits();
3609 PreExt = N0;
3610 }
3611 }
3612
3613 // Figure out how many bits we need to preserve this constant.
3614 unsigned ReqdBits = Signed ?
3615 C1.getBitWidth() - C1.getNumSignBits() + 1 :
3616 C1.getActiveBits();
3617
3618 // Make sure we're not losing bits from the constant.
3619 if (MinBits > 0 &&
3620 MinBits < C1.getBitWidth() &&
3621 MinBits >= ReqdBits) {
3622 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3623 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3624 // Will get folded away.
3625 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3626 if (MinBits == 1 && C1 == 1)
3627 // Invert the condition.
3628 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3629 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3630 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3631 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3632 }
3633
3634 // If truncating the setcc operands is not desirable, we can still
3635 // simplify the expression in some cases:
3636 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3637 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3638 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3639 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3640 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3641 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3642 SDValue TopSetCC = N0->getOperand(0);
3643 unsigned N0Opc = N0->getOpcode();
3644 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3645 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3646 TopSetCC.getOpcode() == ISD::SETCC &&
3647 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3648 (isConstFalseVal(N1C) ||
3649 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3650
3651 bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3652 (!N1C->isNullValue() && Cond == ISD::SETNE);
3653
3654 if (!Inverse)
3655 return TopSetCC;
3656
3657 ISD::CondCode InvCond = ISD::getSetCCInverse(
3658 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3659 TopSetCC.getOperand(0).getValueType());
3660 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3661 TopSetCC.getOperand(1),
3662 InvCond);
3663 }
3664 }
3665 }
3666
3667 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3668 // equality or unsigned, and all 1 bits of the const are in the same
3669 // partial word, see if we can shorten the load.
3670 if (DCI.isBeforeLegalize() &&
3671 !ISD::isSignedIntSetCC(Cond) &&
3672 N0.getOpcode() == ISD::AND && C1 == 0 &&
3673 N0.getNode()->hasOneUse() &&
3674 isa<LoadSDNode>(N0.getOperand(0)) &&
3675 N0.getOperand(0).getNode()->hasOneUse() &&
3676 isa<ConstantSDNode>(N0.getOperand(1))) {
3677 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3678 APInt bestMask;
3679 unsigned bestWidth = 0, bestOffset = 0;
3680 if (Lod->isSimple() && Lod->isUnindexed()) {
3681 unsigned origWidth = N0.getValueSizeInBits();
3682 unsigned maskWidth = origWidth;
3683 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3684 // 8 bits, but have to be careful...
3685 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3686 origWidth = Lod->getMemoryVT().getSizeInBits();
3687 const APInt &Mask = N0.getConstantOperandAPInt(1);
3688 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3689 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3690 for (unsigned offset=0; offset<origWidth/width; offset++) {
3691 if (Mask.isSubsetOf(newMask)) {
3692 if (Layout.isLittleEndian())
3693 bestOffset = (uint64_t)offset * (width/8);
3694 else
3695 bestOffset = (origWidth/width - offset - 1) * (width/8);
3696 bestMask = Mask.lshr(offset * (width/8) * 8);
3697 bestWidth = width;
3698 break;
3699 }
3700 newMask <<= width;
3701 }
3702 }
3703 }
3704 if (bestWidth) {
3705 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3706 if (newVT.isRound() &&
3707 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3708 SDValue Ptr = Lod->getBasePtr();
3709 if (bestOffset != 0)
3710 Ptr =
3711 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3712 SDValue NewLoad =
3713 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3714 Lod->getPointerInfo().getWithOffset(bestOffset),
3715 Lod->getOriginalAlign());
3716 return DAG.getSetCC(dl, VT,
3717 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3718 DAG.getConstant(bestMask.trunc(bestWidth),
3719 dl, newVT)),
3720 DAG.getConstant(0LL, dl, newVT), Cond);
3721 }
3722 }
3723 }
3724
3725 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3726 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3727 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3728
3729 // If the comparison constant has bits in the upper part, the
3730 // zero-extended value could never match.
3731 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3732 C1.getBitWidth() - InSize))) {
3733 switch (Cond) {
3734 case ISD::SETUGT:
3735 case ISD::SETUGE:
3736 case ISD::SETEQ:
3737 return DAG.getConstant(0, dl, VT);
3738 case ISD::SETULT:
3739 case ISD::SETULE:
3740 case ISD::SETNE:
3741 return DAG.getConstant(1, dl, VT);
3742 case ISD::SETGT:
3743 case ISD::SETGE:
3744 // True if the sign bit of C1 is set.
3745 return DAG.getConstant(C1.isNegative(), dl, VT);
3746 case ISD::SETLT:
3747 case ISD::SETLE:
3748 // True if the sign bit of C1 isn't set.
3749 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3750 default:
3751 break;
3752 }
3753 }
3754
3755 // Otherwise, we can perform the comparison with the low bits.
3756 switch (Cond) {
3757 case ISD::SETEQ:
3758 case ISD::SETNE:
3759 case ISD::SETUGT:
3760 case ISD::SETUGE:
3761 case ISD::SETULT:
3762 case ISD::SETULE: {
3763 EVT newVT = N0.getOperand(0).getValueType();
3764 if (DCI.isBeforeLegalizeOps() ||
3765 (isOperationLegal(ISD::SETCC, newVT) &&
3766 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3767 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3768 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3769
3770 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3771 NewConst, Cond);
3772 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3773 }
3774 break;
3775 }
3776 default:
3777 break; // todo, be more careful with signed comparisons
3778 }
3779 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3780 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3781 !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3782 OpVT)) {
3783 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3784 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3785 EVT ExtDstTy = N0.getValueType();
3786 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3787
3788 // If the constant doesn't fit into the number of bits for the source of
3789 // the sign extension, it is impossible for both sides to be equal.
3790 if (C1.getMinSignedBits() > ExtSrcTyBits)
3791 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3792
3793 assert(ExtDstTy == N0.getOperand(0).getValueType() &&((void)0)
3794 ExtDstTy != ExtSrcTy && "Unexpected types!")((void)0);
3795 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3796 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3797 DAG.getConstant(Imm, dl, ExtDstTy));
3798 if (!DCI.isCalledByLegalizer())
3799 DCI.AddToWorklist(ZextOp.getNode());
3800 // Otherwise, make this a use of a zext.
3801 return DAG.getSetCC(dl, VT, ZextOp,
3802 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3803 } else if ((N1C->isNullValue() || N1C->isOne()) &&
3804 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3805 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3806 if (N0.getOpcode() == ISD::SETCC &&
3807 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3808 (N0.getValueType() == MVT::i1 ||
3809 getBooleanContents(N0.getOperand(0).getValueType()) ==
3810 ZeroOrOneBooleanContent)) {
3811 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3812 if (TrueWhenTrue)
3813 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3814 // Invert the condition.
3815 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3816 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3817 if (DCI.isBeforeLegalizeOps() ||
3818 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3819 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3820 }
3821
3822 if ((N0.getOpcode() == ISD::XOR ||
3823 (N0.getOpcode() == ISD::AND &&
3824 N0.getOperand(0).getOpcode() == ISD::XOR &&
3825 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3826 isOneConstant(N0.getOperand(1))) {
3827 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3828 // can only do this if the top bits are known zero.
3829 unsigned BitWidth = N0.getValueSizeInBits();
3830 if (DAG.MaskedValueIsZero(N0,
3831 APInt::getHighBitsSet(BitWidth,
3832 BitWidth-1))) {
3833 // Okay, get the un-inverted input value.
3834 SDValue Val;
3835 if (N0.getOpcode() == ISD::XOR) {
3836 Val = N0.getOperand(0);
3837 } else {
3838 assert(N0.getOpcode() == ISD::AND &&((void)0)
3839 N0.getOperand(0).getOpcode() == ISD::XOR)((void)0);
3840 // ((X^1)&1)^1 -> X & 1
3841 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3842 N0.getOperand(0).getOperand(0),
3843 N0.getOperand(1));
3844 }
3845
3846 return DAG.getSetCC(dl, VT, Val, N1,
3847 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3848 }
3849 } else if (N1C->isOne()) {
3850 SDValue Op0 = N0;
3851 if (Op0.getOpcode() == ISD::TRUNCATE)
3852 Op0 = Op0.getOperand(0);
3853
3854 if ((Op0.getOpcode() == ISD::XOR) &&
3855 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3856 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3857 SDValue XorLHS = Op0.getOperand(0);
3858 SDValue XorRHS = Op0.getOperand(1);
3859 // Ensure that the input setccs return an i1 type or 0/1 value.
3860 if (Op0.getValueType() == MVT::i1 ||
3861 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3862 ZeroOrOneBooleanContent &&
3863 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3864 ZeroOrOneBooleanContent)) {
3865 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3866 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3867 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3868 }
3869 }
3870 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3871 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3872 if (Op0.getValueType().bitsGT(VT))
3873 Op0 = DAG.getNode(ISD::AND, dl, VT,
3874 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3875 DAG.getConstant(1, dl, VT));
3876 else if (Op0.getValueType().bitsLT(VT))
3877 Op0 = DAG.getNode(ISD::AND, dl, VT,
3878 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3879 DAG.getConstant(1, dl, VT));
3880
3881 return DAG.getSetCC(dl, VT, Op0,
3882 DAG.getConstant(0, dl, Op0.getValueType()),
3883 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3884 }
3885 if (Op0.getOpcode() == ISD::AssertZext &&
3886 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3887 return DAG.getSetCC(dl, VT, Op0,
3888 DAG.getConstant(0, dl, Op0.getValueType()),
3889 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3890 }
3891 }
3892
3893 // Given:
3894 // icmp eq/ne (urem %x, %y), 0
3895 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3896 // icmp eq/ne %x, 0
3897 if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3898 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3899 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3900 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3901 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3902 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3903 }
3904
3905 if (SDValue V =
3906 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3907 return V;
3908 }
3909
3910 // These simplifications apply to splat vectors as well.
3911 // TODO: Handle more splat vector cases.
3912 if (auto *N1C = isConstOrConstSplat(N1)) {
3913 const APInt &C1 = N1C->getAPIntValue();
3914
3915 APInt MinVal, MaxVal;
3916 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3917 if (ISD::isSignedIntSetCC(Cond)) {
3918 MinVal = APInt::getSignedMinValue(OperandBitSize);
3919 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3920 } else {
3921 MinVal = APInt::getMinValue(OperandBitSize);
3922 MaxVal = APInt::getMaxValue(OperandBitSize);
3923 }
3924
3925 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3926 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3927 // X >= MIN --> true
3928 if (C1 == MinVal)
3929 return DAG.getBoolConstant(true, dl, VT, OpVT);
3930
3931 if (!VT.isVector()) { // TODO: Support this for vectors.
3932 // X >= C0 --> X > (C0 - 1)
3933 APInt C = C1 - 1;
3934 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3935 if ((DCI.isBeforeLegalizeOps() ||
3936 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3937 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3938 isLegalICmpImmediate(C.getSExtValue())))) {
3939 return DAG.getSetCC(dl, VT, N0,
3940 DAG.getConstant(C, dl, N1.getValueType()),
3941 NewCC);
3942 }
3943 }
3944 }
3945
3946 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3947 // X <= MAX --> true
3948 if (C1 == MaxVal)
3949 return DAG.getBoolConstant(true, dl, VT, OpVT);
3950
3951 // X <= C0 --> X < (C0 + 1)
3952 if (!VT.isVector()) { // TODO: Support this for vectors.
3953 APInt C = C1 + 1;
3954 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3955 if ((DCI.isBeforeLegalizeOps() ||
3956 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3957 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3958 isLegalICmpImmediate(C.getSExtValue())))) {
3959 return DAG.getSetCC(dl, VT, N0,
3960 DAG.getConstant(C, dl, N1.getValueType()),
3961 NewCC);
3962 }
3963 }
3964 }
3965
3966 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3967 if (C1 == MinVal)
3968 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3969
3970 // TODO: Support this for vectors after legalize ops.
3971 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3972 // Canonicalize setlt X, Max --> setne X, Max
3973 if (C1 == MaxVal)
3974 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3975
3976 // If we have setult X, 1, turn it into seteq X, 0
3977 if (C1 == MinVal+1)
3978 return DAG.getSetCC(dl, VT, N0,
3979 DAG.getConstant(MinVal, dl, N0.getValueType()),
3980 ISD::SETEQ);
3981 }
3982 }
3983
3984 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3985 if (C1 == MaxVal)
3986 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3987
3988 // TODO: Support this for vectors after legalize ops.
3989 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3990 // Canonicalize setgt X, Min --> setne X, Min
3991 if (C1 == MinVal)
3992 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3993
3994 // If we have setugt X, Max-1, turn it into seteq X, Max
3995 if (C1 == MaxVal-1)
3996 return DAG.getSetCC(dl, VT, N0,
3997 DAG.getConstant(MaxVal, dl, N0.getValueType()),
3998 ISD::SETEQ);
3999 }
4000 }
4001
4002 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4003 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4004 if (C1.isNullValue())
4005 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4006 VT, N0, N1, Cond, DCI, dl))
4007 return CC;
4008
4009 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4010 // For example, when high 32-bits of i64 X are known clear:
4011 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4012 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4013 bool CmpZero = N1C->getAPIntValue().isNullValue();
4014 bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();
4015 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4016 // Match or(lo,shl(hi,bw/2)) pattern.
4017 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4018 unsigned EltBits = V.getScalarValueSizeInBits();
4019 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4020 return false;
4021 SDValue LHS = V.getOperand(0);
4022 SDValue RHS = V.getOperand(1);
4023 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4024 // Unshifted element must have zero upperbits.
4025 if (RHS.getOpcode() == ISD::SHL &&
4026 isa<ConstantSDNode>(RHS.getOperand(1)) &&
4027 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4028 DAG.MaskedValueIsZero(LHS, HiBits)) {
4029 Lo = LHS;
4030 Hi = RHS.getOperand(0);
4031 return true;
4032 }
4033 if (LHS.getOpcode() == ISD::SHL &&
4034 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4035 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4036 DAG.MaskedValueIsZero(RHS, HiBits)) {
4037 Lo = RHS;
4038 Hi = LHS.getOperand(0);
4039 return true;
4040 }
4041 return false;
4042 };
4043
4044 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4045 unsigned EltBits = N0.getScalarValueSizeInBits();
4046 unsigned HalfBits = EltBits / 2;
4047 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4048 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4049 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4050 SDValue NewN0 =
4051 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4052 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4053 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4054 };
4055
4056 SDValue Lo, Hi;
4057 if (IsConcat(N0, Lo, Hi))
4058 return MergeConcat(Lo, Hi);
4059
4060 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4061 SDValue Lo0, Lo1, Hi0, Hi1;
4062 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4063 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4064 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4065 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4066 }
4067 }
4068 }
4069 }
4070
4071 // If we have "setcc X, C0", check to see if we can shrink the immediate
4072 // by changing cc.
4073 // TODO: Support this for vectors after legalize ops.
4074 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4075 // SETUGT X, SINTMAX -> SETLT X, 0
4076 // SETUGE X, SINTMIN -> SETLT X, 0
4077 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4078 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4079 return DAG.getSetCC(dl, VT, N0,
4080 DAG.getConstant(0, dl, N1.getValueType()),
4081 ISD::SETLT);
4082
4083 // SETULT X, SINTMIN -> SETGT X, -1
4084 // SETULE X, SINTMAX -> SETGT X, -1
4085 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4086 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4087 return DAG.getSetCC(dl, VT, N0,
4088 DAG.getAllOnesConstant(dl, N1.getValueType()),
4089 ISD::SETGT);
4090 }
4091 }
4092
4093 // Back to non-vector simplifications.
4094 // TODO: Can we do these for vector splats?
4095 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4096 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4097 const APInt &C1 = N1C->getAPIntValue();
4098 EVT ShValTy = N0.getValueType();
4099
4100 // Fold bit comparisons when we can. This will result in an
4101 // incorrect value when boolean false is negative one, unless
4102 // the bitsize is 1 in which case the false value is the same
4103 // in practice regardless of the representation.
4104 if ((VT.getSizeInBits() == 1 ||
4105 getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
4106 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4107 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4108 N0.getOpcode() == ISD::AND) {
4109 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4110 EVT ShiftTy =
4111 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4112 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4113 // Perform the xform if the AND RHS is a single bit.
4114 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4115 if (AndRHS->getAPIntValue().isPowerOf2() &&
4116 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4117 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4118 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4119 DAG.getConstant(ShCt, dl, ShiftTy)));
4120 }
4121 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4122 // (X & 8) == 8 --> (X & 8) >> 3
4123 // Perform the xform if C1 is a single bit.
4124 unsigned ShCt = C1.logBase2();
4125 if (C1.isPowerOf2() &&
4126 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4127 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4128 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4129 DAG.getConstant(ShCt, dl, ShiftTy)));
4130 }
4131 }
4132 }
4133 }
4134
4135 if (C1.getMinSignedBits() <= 64 &&
4136 !isLegalICmpImmediate(C1.getSExtValue())) {
4137 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4138 // (X & -256) == 256 -> (X >> 8) == 1
4139 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4140 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4141 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4142 const APInt &AndRHSC = AndRHS->getAPIntValue();
4143 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
4144 unsigned ShiftBits = AndRHSC.countTrailingZeros();
4145 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4146 SDValue Shift =
4147 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4148 DAG.getConstant(ShiftBits, dl, ShiftTy));
4149 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4150 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4151 }
4152 }
4153 }
4154 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4155 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4156 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4157 // X < 0x100000000 -> (X >> 32) < 1
4158 // X >= 0x100000000 -> (X >> 32) >= 1
4159 // X <= 0x0ffffffff -> (X >> 32) < 1
4160 // X > 0x0ffffffff -> (X >> 32) >= 1
4161 unsigned ShiftBits;
4162 APInt NewC = C1;
4163 ISD::CondCode NewCond = Cond;
4164 if (AdjOne) {
4165 ShiftBits = C1.countTrailingOnes();
4166 NewC = NewC + 1;
4167 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4168 } else {
4169 ShiftBits = C1.countTrailingZeros();
4170 }
4171 NewC.lshrInPlace(ShiftBits);
4172 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4173 isLegalICmpImmediate(NewC.getSExtValue()) &&
4174 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4175 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4176 DAG.getConstant(ShiftBits, dl, ShiftTy));
4177 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4178 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4179 }
4180 }
4181 }
4182 }
4183
4184 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4185 auto *CFP = cast<ConstantFPSDNode>(N1);
4186 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")((void)0);
4187
4188 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4189 // constant if knowing that the operand is non-nan is enough. We prefer to
4190 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4191 // materialize 0.0.
4192 if (Cond == ISD::SETO || Cond == ISD::SETUO)
4193 return DAG.getSetCC(dl, VT, N0, N0, Cond);
4194
4195 // setcc (fneg x), C -> setcc swap(pred) x, -C
4196 if (N0.getOpcode() == ISD::FNEG) {
4197 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4198 if (DCI.isBeforeLegalizeOps() ||
4199 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4200 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4201 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4202 }
4203 }
4204
4205 // If the condition is not legal, see if we can find an equivalent one
4206 // which is legal.
4207 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4208 // If the comparison was an awkward floating-point == or != and one of
4209 // the comparison operands is infinity or negative infinity, convert the
4210 // condition to a less-awkward <= or >=.
4211 if (CFP->getValueAPF().isInfinity()) {
4212 bool IsNegInf = CFP->getValueAPF().isNegative();
4213 ISD::CondCode NewCond = ISD::SETCC_INVALID;
4214 switch (Cond) {
4215 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4216 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4217 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4218 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4219 default: break;
4220 }
4221 if (NewCond != ISD::SETCC_INVALID &&
4222 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4223 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4224 }
4225 }
4226 }
4227
4228 if (N0 == N1) {
4229 // The sext(setcc()) => setcc() optimization relies on the appropriate
4230 // constant being emitted.
4231 assert(!N0.getValueType().isInteger() &&((void)0)
4232 "Integer types should be handled by FoldSetCC")((void)0);
4233
4234 bool EqTrue = ISD::isTrueWhenEqual(Cond);
4235 unsigned UOF = ISD::getUnorderedFlavor(Cond);
4236 if (UOF == 2) // FP operators that are undefined on NaNs.
4237 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4238 if (UOF == unsigned(EqTrue))
4239 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4240 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4241 // if it is not already.
4242 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4243 if (NewCond != Cond &&
4244 (DCI.isBeforeLegalizeOps() ||
4245 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4246 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4247 }
4248
4249 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4250 N0.getValueType().isInteger()) {
4251 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4252 N0.getOpcode() == ISD::XOR) {
4253 // Simplify (X+Y) == (X+Z) --> Y == Z
4254 if (N0.getOpcode() == N1.getOpcode()) {
4255 if (N0.getOperand(0) == N1.getOperand(0))
4256 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4257 if (N0.getOperand(1) == N1.getOperand(1))
4258 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4259 if (isCommutativeBinOp(N0.getOpcode())) {
4260 // If X op Y == Y op X, try other combinations.
4261 if (N0.getOperand(0) == N1.getOperand(1))
4262 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4263 Cond);
4264 if (N0.getOperand(1) == N1.getOperand(0))
4265 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4266 Cond);
4267 }
4268 }
4269
4270 // If RHS is a legal immediate value for a compare instruction, we need
4271 // to be careful about increasing register pressure needlessly.
4272 bool LegalRHSImm = false;
4273
4274 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4275 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4276 // Turn (X+C1) == C2 --> X == C2-C1
4277 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4278 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4279 DAG.getConstant(RHSC->getAPIntValue()-
4280 LHSR->getAPIntValue(),
4281 dl, N0.getValueType()), Cond);
4282 }
4283
4284 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4285 if (N0.getOpcode() == ISD::XOR)
4286 // If we know that all of the inverted bits are zero, don't bother
4287 // performing the inversion.
4288 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4289 return
4290 DAG.getSetCC(dl, VT, N0.getOperand(0),
4291 DAG.getConstant(LHSR->getAPIntValue() ^
4292 RHSC->getAPIntValue(),
4293 dl, N0.getValueType()),
4294 Cond);
4295 }
4296
4297 // Turn (C1-X) == C2 --> X == C1-C2
4298 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4299 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4300 return
4301 DAG.getSetCC(dl, VT, N0.getOperand(1),
4302 DAG.getConstant(SUBC->getAPIntValue() -
4303 RHSC->getAPIntValue(),
4304 dl, N0.getValueType()),
4305 Cond);
4306 }
4307 }
4308
4309 // Could RHSC fold directly into a compare?
4310 if (RHSC->getValueType(0).getSizeInBits() <= 64)
4311 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4312 }
4313
4314 // (X+Y) == X --> Y == 0 and similar folds.
4315 // Don't do this if X is an immediate that can fold into a cmp
4316 // instruction and X+Y has other uses. It could be an induction variable
4317 // chain, and the transform would increase register pressure.
4318 if (!LegalRHSImm || N0.hasOneUse())
4319 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4320 return V;
4321 }
4322
4323 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4324 N1.getOpcode() == ISD::XOR)
4325 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4326 return V;
4327
4328 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4329 return V;
4330 }
4331
4332 // Fold remainder of division by a constant.
4333 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4334 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4335 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4336
4337 // When division is cheap or optimizing for minimum size,
4338 // fall through to DIVREM creation by skipping this fold.
4339 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
4340 if (N0.getOpcode() == ISD::UREM) {
4341 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4342 return Folded;
4343 } else if (N0.getOpcode() == ISD::SREM) {
4344 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4345 return Folded;
4346 }
4347 }
4348 }
4349
4350 // Fold away ALL boolean setcc's.
4351 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4352 SDValue Temp;
4353 switch (Cond) {
4354 default: llvm_unreachable("Unknown integer setcc!")__builtin_unreachable();
4355 case ISD::SETEQ: // X == Y -> ~(X^Y)
4356 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4357 N0 = DAG.getNOT(dl, Temp, OpVT);
4358 if (!DCI.isCalledByLegalizer())
4359 DCI.AddToWorklist(Temp.getNode());
4360 break;
4361 case ISD::SETNE: // X != Y --> (X^Y)
4362 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4363 break;
4364 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4365 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4366 Temp = DAG.getNOT(dl, N0, OpVT);
4367 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4368 if (!DCI.isCalledByLegalizer())
4369 DCI.AddToWorklist(Temp.getNode());
4370 break;
4371 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4372 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4373 Temp = DAG.getNOT(dl, N1, OpVT);
4374 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4375 if (!DCI.isCalledByLegalizer())
4376 DCI.AddToWorklist(Temp.getNode());
4377 break;
4378 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4379 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4380 Temp = DAG.getNOT(dl, N0, OpVT);
4381 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4382 if (!DCI.isCalledByLegalizer())
4383 DCI.AddToWorklist(Temp.getNode());
4384 break;
4385 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4386 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4387 Temp = DAG.getNOT(dl, N1, OpVT);
4388 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4389 break;
4390 }
4391 if (VT.getScalarType() != MVT::i1) {
4392 if (!DCI.isCalledByLegalizer())
4393 DCI.AddToWorklist(N0.getNode());
4394 // FIXME: If running after legalize, we probably can't do this.
4395 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4396 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4397 }
4398 return N0;
4399 }
4400
4401 // Could not fold it.
4402 return SDValue();
4403}
4404
4405/// Returns true (and the GlobalValue and the offset) if the node is a
4406/// GlobalAddress + offset.
4407bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4408 int64_t &Offset) const {
4409
4410 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4411
4412 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4413 GA = GASD->getGlobal();
4414 Offset += GASD->getOffset();
4415 return true;
4416 }
4417
4418 if (N->getOpcode() == ISD::ADD) {
4419 SDValue N1 = N->getOperand(0);
4420 SDValue N2 = N->getOperand(1);
4421 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4422 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4423 Offset += V->getSExtValue();
4424 return true;
4425 }
4426 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4427 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4428 Offset += V->getSExtValue();
4429 return true;
4430 }
4431 }
4432 }
4433
4434 return false;
4435}
4436
4437SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4438 DAGCombinerInfo &DCI) const {
4439 // Default implementation: no optimization.
4440 return SDValue();
4441}
4442
4443//===----------------------------------------------------------------------===//
4444// Inline Assembler Implementation Methods
4445//===----------------------------------------------------------------------===//
4446
4447TargetLowering::ConstraintType
4448TargetLowering::getConstraintType(StringRef Constraint) const {
4449 unsigned S = Constraint.size();
4450
4451 if (S == 1) {
4452 switch (Constraint[0]) {
4453 default: break;
4454 case 'r':
4455 return C_RegisterClass;
4456 case 'm': // memory
4457 case 'o': // offsetable
4458 case 'V': // not offsetable
4459 return C_Memory;
4460 case 'n': // Simple Integer
4461 case 'E': // Floating Point Constant
4462 case 'F': // Floating Point Constant
4463 return C_Immediate;
4464 case 'i': // Simple Integer or Relocatable Constant
4465 case 's': // Relocatable Constant
4466 case 'p': // Address.
4467 case 'X': // Allow ANY value.
4468 case 'I': // Target registers.
4469 case 'J':
4470 case 'K':
4471 case 'L':
4472 case 'M':
4473 case 'N':
4474 case 'O':
4475 case 'P':
4476 case '<':
4477 case '>':
4478 return C_Other;
4479 }
4480 }
4481
4482 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4483 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4484 return C_Memory;
4485 return C_Register;
4486 }
4487 return C_Unknown;
4488}
4489
4490/// Try to replace an X constraint, which matches anything, with another that
4491/// has more specific requirements based on the type of the corresponding
4492/// operand.
4493const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4494 if (ConstraintVT.isInteger())
4495 return "r";
4496 if (ConstraintVT.isFloatingPoint())
4497 return "f"; // works for many targets
4498 return nullptr;
4499}
4500
4501SDValue TargetLowering::LowerAsmOutputForConstraint(
4502 SDValue &Chain, SDValue &Flag, const SDLoc &DL,
4503 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
4504 return SDValue();
4505}
4506
4507/// Lower the specified operand into the Ops vector.
4508/// If it is invalid, don't add anything to Ops.
4509void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4510 std::string &Constraint,
4511 std::vector<SDValue> &Ops,
4512 SelectionDAG &DAG) const {
4513
4514 if (Constraint.length() > 1) return;
10
←
Assuming the condition is false
→
11
←
Taking false branch
→
4515
4516 char ConstraintLetter = Constraint[0];
4517 switch (ConstraintLetter) {
12
←
Control jumps to 'case 105:' at line 4526
→
4518 default: break;
4519 case 'X': // Allows any operand; labels (basic block) use this.
4520 if (Op.getOpcode() == ISD::BasicBlock ||
4521 Op.getOpcode() == ISD::TargetBlockAddress) {
4522 Ops.push_back(Op);
4523 return;
4524 }
4525 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4526 case 'i': // Simple Integer or Relocatable Constant
4527 case 'n': // Simple Integer
4528 case 's': { // Relocatable Constant
4529
4530 GlobalAddressSDNode *GA;
4531 ConstantSDNode *C;
4532 BlockAddressSDNode *BA;
4533 uint64_t Offset = 0;
4534
4535 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4536 // etc., since getelementpointer is variadic. We can't use
4537 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4538 // while in this case the GA may be furthest from the root node which is
4539 // likely an ISD::ADD.
4540 while (1) {
13
←
Loop condition is true. Entering loop body
→
38
←
Loop condition is true. Entering loop body
→
4541 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
14
←
Calling 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
18
←
Returning from 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
19
←
Assuming 'GA' is null
→
39
←
Calling 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
43
←
Returning from 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
44
←
Assuming 'GA' is null
→
4542 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4543 GA->getValueType(0),
4544 Offset + GA->getOffset()));
4545 return;
4546 }
4547 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
20
←
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
24
←
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
25
←
Assuming 'C' is null
→
45
←
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
49
←
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
50
←
Assuming 'C' is null
→
4548 // gcc prints these as sign extended. Sign extend value to 64 bits
4549 // now; without this it would get ZExt'd later in
4550 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4551 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4552 BooleanContent BCont = getBooleanContents(MVT::i64);
4553 ISD::NodeType ExtOpc =
4554 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
4555 int64_t ExtVal =
4556 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
4557 Ops.push_back(
4558 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
4559 return;
4560 }
4561 if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {
26
←
Calling 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
30
←
Returning from 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
31
←
Assuming 'BA' is null
→
51
←
Calling 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
64
←
Returning from 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
65
←
Assuming 'BA' is null
→
66
←
Assuming pointer value is null
→
4562 Ops.push_back(DAG.getTargetBlockAddress(
4563 BA->getBlockAddress(), BA->getValueType(0),
4564 Offset + BA->getOffset(), BA->getTargetFlags()));
4565 return;
4566 }
4567 const unsigned OpCode = Op.getOpcode();
67
←
Calling 'SDValue::getOpcode'
→
4568 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
32
←
Assuming 'OpCode' is equal to ADD
→
4569 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
33
←
Assuming 'C' is non-null
→
34
←
Taking true branch
→
4570 Op = Op.getOperand(1);
35
←
Value assigned to 'Op.Node'
→
4571 // Subtraction is not commutative.
4572 else if (OpCode == ISD::ADD &&
4573 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4574 Op = Op.getOperand(0);
4575 else
4576 return;
4577 Offset += (OpCode
35.1
'OpCode' is equal to ADD
35.1
'OpCode' is equal to ADD
35.1
'OpCode' is equal to ADD
 == ISD::ADD ? 1 : -1) * C->getSExtValue();
36
←
'?' condition is true
→
4578 continue;
37
←
 Execution continues on line 4540
→
4579 }
4580 return;
4581 }
4582 break;
4583 }
4584 }
4585}
4586
4587std::pair<unsigned, const TargetRegisterClass *>
4588TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4589 StringRef Constraint,
4590 MVT VT) const {
4591 if (Constraint.empty() || Constraint[0] != '{')
4592 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4593 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")((void)0);
4594
4595 // Remove the braces from around the name.
4596 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4597
4598 std::pair<unsigned, const TargetRegisterClass *> R =
4599 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4600
4601 // Figure out which register class contains this reg.
4602 for (const TargetRegisterClass *RC : RI->regclasses()) {
4603 // If none of the value types for this register class are valid, we
4604 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4605 if (!isLegalRC(*RI, *RC))
4606 continue;
4607
4608 for (const MCPhysReg &PR : *RC) {
4609 if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
4610 std::pair<unsigned, const TargetRegisterClass *> S =
4611 std::make_pair(PR, RC);
4612
4613 // If this register class has the requested value type, return it,
4614 // otherwise keep searching and return the first class found
4615 // if no other is found which explicitly has the requested type.
4616 if (RI->isTypeLegalForClass(*RC, VT))
4617 return S;
4618 if (!R.second)
4619 R = S;
4620 }
4621 }
4622 }
4623
4624 return R;
4625}
4626
4627//===----------------------------------------------------------------------===//
4628// Constraint Selection.
4629
4630/// Return true of this is an input operand that is a matching constraint like
4631/// "4".
4632bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4633 assert(!ConstraintCode.empty() && "No known constraint!")((void)0);
4634 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4635}
4636
4637/// If this is an input matching constraint, this method returns the output
4638/// operand it matches.
4639unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4640 assert(!ConstraintCode.empty() && "No known constraint!")((void)0);
4641 return atoi(ConstraintCode.c_str());
4642}
4643
4644/// Split up the constraint string from the inline assembly value into the
4645/// specific constraints and their prefixes, and also tie in the associated
4646/// operand values.
4647/// If this returns an empty vector, and if the constraint string itself
4648/// isn't empty, there was an error parsing.
4649TargetLowering::AsmOperandInfoVector
4650TargetLowering::ParseConstraints(const DataLayout &DL,
4651 const TargetRegisterInfo *TRI,
4652 const CallBase &Call) const {
4653 /// Information about all of the constraints.
4654 AsmOperandInfoVector ConstraintOperands;
4655 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
4656 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4657
4658 // Do a prepass over the constraints, canonicalizing them, and building up the
4659 // ConstraintOperands list.
4660 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4661 unsigned ResNo = 0; // ResNo - The result number of the next output.
4662
4663 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4664 ConstraintOperands.emplace_back(std::move(CI));
4665 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4666
4667 // Update multiple alternative constraint count.
4668 if (OpInfo.multipleAlternatives.size() > maCount)
4669 maCount = OpInfo.multipleAlternatives.size();
4670
4671 OpInfo.ConstraintVT = MVT::Other;
4672
4673 // Compute the value type for each operand.
4674 switch (OpInfo.Type) {
4675 case InlineAsm::isOutput:
4676 // Indirect outputs just consume an argument.
4677 if (OpInfo.isIndirect) {
4678 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4679 break;
4680 }
4681
4682 // The return value of the call is this value. As such, there is no
4683 // corresponding argument.
4684 assert(!Call.getType()->isVoidTy() && "Bad inline asm!")((void)0);
4685 if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
4686 OpInfo.ConstraintVT =
4687 getSimpleValueType(DL, STy->getElementType(ResNo));
4688 } else {
4689 assert(ResNo == 0 && "Asm only has one result!")((void)0);
4690 OpInfo.ConstraintVT =
4691 getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
4692 }
4693 ++ResNo;
4694 break;
4695 case InlineAsm::isInput:
4696 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4697 break;
4698 case InlineAsm::isClobber:
4699 // Nothing to do.
4700 break;
4701 }
4702
4703 if (OpInfo.CallOperandVal) {
4704 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4705 if (OpInfo.isIndirect) {
4706 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
4707 if (!PtrTy)
4708 report_fatal_error("Indirect operand for inline asm not a pointer!");
4709 OpTy = PtrTy->getElementType();
4710 }
4711
4712 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4713 if (StructType *STy = dyn_cast<StructType>(OpTy))
4714 if (STy->getNumElements() == 1)
4715 OpTy = STy->getElementType(0);
4716
4717 // If OpTy is not a single value, it may be a struct/union that we
4718 // can tile with integers.
4719 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4720 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4721 switch (BitSize) {
4722 default: break;
4723 case 1:
4724 case 8:
4725 case 16:
4726 case 32:
4727 case 64:
4728 case 128:
4729 OpInfo.ConstraintVT =
4730 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4731 break;
4732 }
4733 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4734 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4735 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4736 } else {
4737 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4738 }
4739 }
4740 }
4741
4742 // If we have multiple alternative constraints, select the best alternative.
4743 if (!ConstraintOperands.empty()) {
4744 if (maCount) {
4745 unsigned bestMAIndex = 0;
4746 int bestWeight = -1;
4747 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4748 int weight = -1;
4749 unsigned maIndex;
4750 // Compute the sums of the weights for each alternative, keeping track
4751 // of the best (highest weight) one so far.
4752 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4753 int weightSum = 0;
4754 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4755 cIndex != eIndex; ++cIndex) {
4756 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4757 if (OpInfo.Type == InlineAsm::isClobber)
4758 continue;
4759
4760 // If this is an output operand with a matching input operand,
4761 // look up the matching input. If their types mismatch, e.g. one
4762 // is an integer, the other is floating point, or their sizes are
4763 // different, flag it as an maCantMatch.
4764 if (OpInfo.hasMatchingInput()) {
4765 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4766 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4767 if ((OpInfo.ConstraintVT.isInteger() !=
4768 Input.ConstraintVT.isInteger()) ||
4769 (OpInfo.ConstraintVT.getSizeInBits() !=
4770 Input.ConstraintVT.getSizeInBits())) {
4771 weightSum = -1; // Can't match.
4772 break;
4773 }
4774 }
4775 }
4776 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4777 if (weight == -1) {
4778 weightSum = -1;
4779 break;
4780 }
4781 weightSum += weight;
4782 }
4783 // Update best.
4784 if (weightSum > bestWeight) {
4785 bestWeight = weightSum;
4786 bestMAIndex = maIndex;
4787 }
4788 }
4789
4790 // Now select chosen alternative in each constraint.
4791 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4792 cIndex != eIndex; ++cIndex) {
4793 AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
4794 if (cInfo.Type == InlineAsm::isClobber)
4795 continue;
4796 cInfo.selectAlternative(bestMAIndex);
4797 }
4798 }
4799 }
4800
4801 // Check and hook up tied operands, choose constraint code to use.
4802 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4803 cIndex != eIndex; ++cIndex) {
4804 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4805
4806 // If this is an output operand with a matching input operand, look up the
4807 // matching input. If their types mismatch, e.g. one is an integer, the
4808 // other is floating point, or their sizes are different, flag it as an
4809 // error.
4810 if (OpInfo.hasMatchingInput()) {
4811 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4812
4813 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4814 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4815 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4816 OpInfo.ConstraintVT);
4817 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4818 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4819 Input.ConstraintVT);
4820 if ((OpInfo.ConstraintVT.isInteger() !=
4821 Input.ConstraintVT.isInteger()) ||
4822 (MatchRC.second != InputRC.second)) {
4823 report_fatal_error("Unsupported asm: input constraint"
4824 " with a matching output constraint of"
4825 " incompatible type!");
4826 }
4827 }
4828 }
4829 }
4830
4831 return ConstraintOperands;
4832}
4833
4834/// Return an integer indicating how general CT is.
4835static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4836 switch (CT) {
4837 case TargetLowering::C_Immediate:
4838 case TargetLowering::C_Other:
4839 case TargetLowering::C_Unknown:
4840 return 0;
4841 case TargetLowering::C_Register:
4842 return 1;
4843 case TargetLowering::C_RegisterClass:
4844 return 2;
4845 case TargetLowering::C_Memory:
4846 return 3;
4847 }
4848 llvm_unreachable("Invalid constraint type")__builtin_unreachable();
4849}
4850
4851/// Examine constraint type and operand type and determine a weight value.
4852/// This object must already have been set up with the operand type
4853/// and the current alternative constraint selected.
4854TargetLowering::ConstraintWeight
4855 TargetLowering::getMultipleConstraintMatchWeight(
4856 AsmOperandInfo &info, int maIndex) const {
4857 InlineAsm::ConstraintCodeVector *rCodes;
4858 if (maIndex >= (int)info.multipleAlternatives.size())
4859 rCodes = &info.Codes;
4860 else
4861 rCodes = &info.multipleAlternatives[maIndex].Codes;
4862 ConstraintWeight BestWeight = CW_Invalid;
4863
4864 // Loop over the options, keeping track of the most general one.
4865 for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
4866 ConstraintWeight weight =
4867 getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
4868 if (weight > BestWeight)
4869 BestWeight = weight;
4870 }
4871
4872 return BestWeight;
4873}
4874
4875/// Examine constraint type and operand type and determine a weight value.
4876/// This object must already have been set up with the operand type
4877/// and the current alternative constraint selected.
4878TargetLowering::ConstraintWeight
4879 TargetLowering::getSingleConstraintMatchWeight(
4880 AsmOperandInfo &info, const char *constraint) const {
4881 ConstraintWeight weight = CW_Invalid;
4882 Value *CallOperandVal = info.CallOperandVal;
4883 // If we don't have a value, we can't do a match,
4884 // but allow it at the lowest weight.
4885 if (!CallOperandVal)
4886 return CW_Default;
4887 // Look at the constraint type.
4888 switch (*constraint) {
4889 case 'i': // immediate integer.
4890 case 'n': // immediate integer with a known value.
4891 if (isa<ConstantInt>(CallOperandVal))
4892 weight = CW_Constant;
4893 break;
4894 case 's': // non-explicit intregal immediate.
4895 if (isa<GlobalValue>(CallOperandVal))
4896 weight = CW_Constant;
4897 break;
4898 case 'E': // immediate float if host format.
4899 case 'F': // immediate float.
4900 if (isa<ConstantFP>(CallOperandVal))
4901 weight = CW_Constant;
4902 break;
4903 case '<': // memory operand with autodecrement.
4904 case '>': // memory operand with autoincrement.
4905 case 'm': // memory operand.
4906 case 'o': // offsettable memory operand
4907 case 'V': // non-offsettable memory operand
4908 weight = CW_Memory;
4909 break;
4910 case 'r': // general register.
4911 case 'g': // general register, memory operand or immediate integer.
4912 // note: Clang converts "g" to "imr".
4913 if (CallOperandVal->getType()->isIntegerTy())
4914 weight = CW_Register;
4915 break;
4916 case 'X': // any operand.
4917 default:
4918 weight = CW_Default;
4919 break;
4920 }
4921 return weight;
4922}
4923
4924/// If there are multiple different constraints that we could pick for this
4925/// operand (e.g. "imr") try to pick the 'best' one.
4926/// This is somewhat tricky: constraints fall into four classes:
4927/// Other -> immediates and magic values
4928/// Register -> one specific register
4929/// RegisterClass -> a group of regs
4930/// Memory -> memory
4931/// Ideally, we would pick the most specific constraint possible: if we have
4932/// something that fits into a register, we would pick it. The problem here
4933/// is that if we have something that could either be in a register or in
4934/// memory that use of the register could cause selection of *other*
4935/// operands to fail: they might only succeed if we pick memory. Because of
4936/// this the heuristic we use is:
4937///
4938/// 1) If there is an 'other' constraint, and if the operand is valid for
4939/// that constraint, use it. This makes us take advantage of 'i'
4940/// constraints when available.
4941/// 2) Otherwise, pick the most general constraint present. This prefers
4942/// 'm' over 'r', for example.
4943///
4944static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
4945 const TargetLowering &TLI,
4946 SDValue Op, SelectionDAG *DAG) {
4947 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")((void)0);
4948 unsigned BestIdx = 0;
4949 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
4950 int BestGenerality = -1;
4951
4952 // Loop over the options, keeping track of the most general one.
4953 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
4
←
Assuming 'i' is not equal to 'e'
→
5
←
Loop condition is true. Entering loop body
→
4954 TargetLowering::ConstraintType CType =
4955 TLI.getConstraintType(OpInfo.Codes[i]);
4956
4957 // Indirect 'other' or 'immediate' constraints are not allowed.
4958 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6
←
Assuming field 'isIndirect' is false
→
4959 CType == TargetLowering::C_Register ||
4960 CType == TargetLowering::C_RegisterClass))
4961 continue;
4962
4963 // If this is an 'other' or 'immediate' constraint, see if the operand is
4964 // valid for it. For example, on X86 we might have an 'rI' constraint. If
4965 // the operand is an integer in the range [0..31] we want to use I (saving a
4966 // load of a register), otherwise we must use 'r'.
4967 if ((CType
6.1
'CType' is equal to C_Other
6.1
'CType' is equal to C_Other
6.1
'CType' is equal to C_Other
 == TargetLowering::C_Other ||
8
←
Taking true branch
→
4968 CType == TargetLowering::C_Immediate) && Op.getNode()) {
7
←
Assuming the condition is true
→
4969 assert(OpInfo.Codes[i].size() == 1 &&((void)0)
4970 "Unhandled multi-letter 'other' constraint")((void)0);
4971 std::vector<SDValue> ResultOps;
4972 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
9
←
Calling 'TargetLowering::LowerAsmOperandForConstraint'
→
4973 ResultOps, *DAG);
4974 if (!ResultOps.empty()) {
4975 BestType = CType;
4976 BestIdx = i;
4977 break;
4978 }
4979 }
4980
4981 // Things with matching constraints can only be registers, per gcc
4982 // documentation. This mainly affects "g" constraints.
4983 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
4984 continue;
4985
4986 // This constraint letter is more general than the previous one, use it.
4987 int Generality = getConstraintGenerality(CType);
4988 if (Generality > BestGenerality) {
4989 BestType = CType;
4990 BestIdx = i;
4991 BestGenerality = Generality;
4992 }
4993 }
4994
4995 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
4996 OpInfo.ConstraintType = BestType;
4997}
4998
4999/// Determines the constraint code and constraint type to use for the specific
5000/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5001void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
5002 SDValue Op,
5003 SelectionDAG *DAG) const {
5004 assert(!OpInfo.Codes.empty() && "Must have at least one constraint")((void)0);
5005
5006 // Single-letter constraints ('r') are very common.
5007 if (OpInfo.Codes.size() == 1) {
1
Assuming the condition is false
→
2
←
Taking false branch
→
5008 OpInfo.ConstraintCode = OpInfo.Codes[0];
5009 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5010 } else {
5011 ChooseConstraint(OpInfo, *this, Op, DAG);
3
←
Calling 'ChooseConstraint'
→
5012 }
5013
5014 // 'X' matches anything.
5015 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
5016 // Labels and constants are handled elsewhere ('X' is the only thing
5017 // that matches labels). For Functions, the type here is the type of
5018 // the result, which is not what we want to look at; leave them alone.
5019 Value *v = OpInfo.CallOperandVal;
5020 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
5021 OpInfo.CallOperandVal = v;
5022 return;
5023 }
5024
5025 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
5026 return;
5027
5028 // Otherwise, try to resolve it to something we know about by looking at
5029 // the actual operand type.
5030 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
5031 OpInfo.ConstraintCode = Repl;
5032 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5033 }
5034 }
5035}
5036
5037/// Given an exact SDIV by a constant, create a multiplication
5038/// with the multiplicative inverse of the constant.
5039static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
5040 const SDLoc &dl, SelectionDAG &DAG,
5041 SmallVectorImpl<SDNode *> &Created) {
5042 SDValue Op0 = N->getOperand(0);
5043 SDValue Op1 = N->getOperand(1);
5044 EVT VT = N->getValueType(0);
5045 EVT SVT = VT.getScalarType();
5046 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5047 EVT ShSVT = ShVT.getScalarType();
5048
5049 bool UseSRA = false;
5050 SmallVector<SDValue, 16> Shifts, Factors;
5051
5052 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5053 if (C->isNullValue())
5054 return false;
5055 APInt Divisor = C->getAPIntValue();
5056 unsigned Shift = Divisor.countTrailingZeros();
5057 if (Shift) {
5058 Divisor.ashrInPlace(Shift);
5059 UseSRA = true;
5060 }
5061 // Calculate the multiplicative inverse, using Newton's method.
5062 APInt t;
5063 APInt Factor = Divisor;
5064 while ((t = Divisor * Factor) != 1)
5065 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
5066 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
5067 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
5068 return true;
5069 };
5070
5071 // Collect all magic values from the build vector.
5072 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
5073 return SDValue();
5074
5075 SDValue Shift, Factor;
5076 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
5077 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5078 Factor = DAG.getBuildVector(VT, dl, Factors);
5079 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
5080 assert(Shifts.size() == 1 && Factors.size() == 1 &&((void)0)
5081 "Expected matchUnaryPredicate to return one element for scalable "((void)0)
5082 "vectors")((void)0);
5083 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5084 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5085 } else {
5086 assert(isa<ConstantSDNode>(Op1) && "Expected a constant")((void)0);
5087 Shift = Shifts[0];
5088 Factor = Factors[0];
5089 }
5090
5091 SDValue Res = Op0;
5092
5093 // Shift the value upfront if it is even, so the LSB is one.
5094 if (UseSRA) {
5095 // TODO: For UDIV use SRL instead of SRA.
5096 SDNodeFlags Flags;
5097 Flags.setExact(true);
5098 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
5099 Created.push_back(Res.getNode());
5100 }
5101
5102 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5103}
5104
5105SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5106 SelectionDAG &DAG,
5107 SmallVectorImpl<SDNode *> &Created) const {
5108 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5109 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5110 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
5111 return SDValue(N, 0); // Lower SDIV as SDIV
5112 return SDValue();
5113}
5114
5115/// Given an ISD::SDIV node expressing a divide by constant,
5116/// return a DAG expression to select that will generate the same value by
5117/// multiplying by a magic number.
5118/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5119SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
5120 bool IsAfterLegalization,
5121 SmallVectorImpl<SDNode *> &Created) const {
5122 SDLoc dl(N);
5123 EVT VT = N->getValueType(0);
5124 EVT SVT = VT.getScalarType();
5125 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5126 EVT ShSVT = ShVT.getScalarType();
5127 unsigned EltBits = VT.getScalarSizeInBits();
5128 EVT MulVT;
5129
5130 // Check to see if we can do this.
5131 // FIXME: We should be more aggressive here.
5132 if (!isTypeLegal(VT)) {
5133 // Limit this to simple scalars for now.
5134 if (VT.isVector() || !VT.isSimple())
5135 return SDValue();
5136
5137 // If this type will be promoted to a large enough type with a legal
5138 // multiply operation, we can go ahead and do this transform.
5139 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5140 return SDValue();
5141
5142 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5143 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5144 !isOperationLegal(ISD::MUL, MulVT))
5145 return SDValue();
5146 }
5147
5148 // If the sdiv has an 'exact' bit we can use a simpler lowering.
5149 if (N->getFlags().hasExact())
5150 return BuildExactSDIV(*this, N, dl, DAG, Created);
5151
5152 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5153
5154 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5155 if (C->isNullValue())
5156 return false;
5157
5158 const APInt &Divisor = C->getAPIntValue();
5159 APInt::ms magics = Divisor.magic();
5160 int NumeratorFactor = 0;
5161 int ShiftMask = -1;
5162
5163 if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
5164 // If d is +1/-1, we just multiply the numerator by +1/-1.
5165 NumeratorFactor = Divisor.getSExtValue();
5166 magics.m = 0;
5167 magics.s = 0;
5168 ShiftMask = 0;
5169 } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
5170 // If d > 0 and m < 0, add the numerator.
5171 NumeratorFactor = 1;
5172 } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
5173 // If d < 0 and m > 0, subtract the numerator.
5174 NumeratorFactor = -1;
5175 }
5176
5177 MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
5178 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
5179 Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
5180 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
5181 return true;
5182 };
5183
5184 SDValue N0 = N->getOperand(0);
5185 SDValue N1 = N->getOperand(1);
5186
5187 // Collect the shifts / magic values from each element.
5188 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5189 return SDValue();
5190
5191 SDValue MagicFactor, Factor, Shift, ShiftMask;
5192 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5193 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5194 Factor = DAG.getBuildVector(VT, dl, Factors);
5195 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5196 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
5197 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5198 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&((void)0)
5199 Shifts.size() == 1 && ShiftMasks.size() == 1 &&((void)0)
5200 "Expected matchUnaryPredicate to return one element for scalable "((void)0)
5201 "vectors")((void)0);
5202 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5203 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5204 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5205 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
5206 } else {
5207 assert(isa<ConstantSDNode>(N1) && "Expected a constant")((void)0);
5208 MagicFactor = MagicFactors[0];
5209 Factor = Factors[0];
5210 Shift = Shifts[0];
5211 ShiftMask = ShiftMasks[0];
5212 }
5213
5214 // Multiply the numerator (operand 0) by the magic value.
5215 // FIXME: We should support doing a MUL in a wider type.
5216 auto GetMULHS = [&](SDValue X, SDValue Y) {
5217 // If the type isn't legal, use a wider mul of the the type calculated
5218 // earlier.
5219 if (!isTypeLegal(VT)) {
5220 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
5221 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
5222 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5223 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5224 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5225 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5226 }
5227
5228 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
5229 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
5230 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
5231 SDValue LoHi =
5232 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5233 return SDValue(LoHi.getNode(), 1);
5234 }
5235 return SDValue();
5236 };
5237
5238 SDValue Q = GetMULHS(N0, MagicFactor);
5239 if (!Q)
5240 return SDValue();
5241
5242 Created.push_back(Q.getNode());
5243
5244 // (Optionally) Add/subtract the numerator using Factor.
5245 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5246 Created.push_back(Factor.getNode());
5247 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5248 Created.push_back(Q.getNode());
5249
5250 // Shift right algebraic by shift value.
5251 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5252 Created.push_back(Q.getNode());
5253
5254 // Extract the sign bit, mask it and add it to the quotient.
5255 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5256 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5257 Created.push_back(T.getNode());
5258 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5259 Created.push_back(T.getNode());
5260 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5261}
5262
5263/// Given an ISD::UDIV node expressing a divide by constant,
5264/// return a DAG expression to select that will generate the same value by
5265/// multiplying by a magic number.
5266/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5267SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5268 bool IsAfterLegalization,
5269 SmallVectorImpl<SDNode *> &Created) const {
5270 SDLoc dl(N);
5271 EVT VT = N->getValueType(0);
5272 EVT SVT = VT.getScalarType();
5273 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5274 EVT ShSVT = ShVT.getScalarType();
5275 unsigned EltBits = VT.getScalarSizeInBits();
5276 EVT MulVT;
5277
5278 // Check to see if we can do this.
5279 // FIXME: We should be more aggressive here.
5280 if (!isTypeLegal(VT)) {
5281 // Limit this to simple scalars for now.
5282 if (VT.isVector() || !VT.isSimple())
5283 return SDValue();
5284
5285 // If this type will be promoted to a large enough type with a legal
5286 // multiply operation, we can go ahead and do this transform.
5287 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5288 return SDValue();
5289
5290 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5291 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5292 !isOperationLegal(ISD::MUL, MulVT))
5293 return SDValue();
5294 }
5295
5296 bool UseNPQ = false;
5297 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5298
5299 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
5300 if (C->isNullValue())
5301 return false;
5302 // FIXME: We should use a narrower constant when the upper
5303 // bits are known to be zero.
5304 const APInt& Divisor = C->getAPIntValue();
5305 APInt::mu magics = Divisor.magicu();
5306 unsigned PreShift = 0, PostShift = 0;
5307
5308 // If the divisor is even, we can avoid using the expensive fixup by
5309 // shifting the divided value upfront.
5310 if (magics.a != 0 && !Divisor[0]) {
5311 PreShift = Divisor.countTrailingZeros();
5312 // Get magic number for the shifted divisor.
5313 magics = Divisor.lshr(PreShift).magicu(PreShift);
5314 assert(magics.a == 0 && "Should use cheap fixup now")((void)0);
5315 }
5316
5317 APInt Magic = magics.m;
5318
5319 unsigned SelNPQ;
5320 if (magics.a == 0 || Divisor.isOneValue()) {
5321 assert(magics.s < Divisor.getBitWidth() &&((void)0)
5322 "We shouldn't generate an undefined shift!")((void)0);
5323 PostShift = magics.s;
5324 SelNPQ = false;
5325 } else {
5326 PostShift = magics.s - 1;
5327 SelNPQ = true;
5328 }
5329
5330 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
5331 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
5332 NPQFactors.push_back(
5333 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5334 : APInt::getNullValue(EltBits),
5335 dl, SVT));
5336 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
5337 UseNPQ |= SelNPQ;
5338 return true;
5339 };
5340
5341 SDValue N0 = N->getOperand(0);
5342 SDValue N1 = N->getOperand(1);
5343
5344 // Collect the shifts/magic values from each element.
5345 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5346 return SDValue();
5347
5348 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5349 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5350 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
5351 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5352 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
5353 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
5354 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5355 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&((void)0)
5356 NPQFactors.size() == 1 && PostShifts.size() == 1 &&((void)0)
5357 "Expected matchUnaryPredicate to return one for scalable vectors")((void)0);
5358 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
5359 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5360 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
5361 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
5362 } else {
5363 assert(isa<ConstantSDNode>(N1) && "Expected a constant")((void)0);
5364 PreShift = PreShifts[0];
5365 MagicFactor = MagicFactors[0];
5366 PostShift = PostShifts[0];
5367 }
5368
5369 SDValue Q = N0;
5370 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5371 Created.push_back(Q.getNode());
5372
5373 // FIXME: We should support doing a MUL in a wider type.
5374 auto GetMULHU = [&](SDValue X, SDValue Y) {
5375 // If the type isn't legal, use a wider mul of the the type calculated
5376 // earlier.
5377 if (!isTypeLegal(VT)) {
5378 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
5379 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
5380 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5381 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5382 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5383 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5384 }
5385
5386 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
5387 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5388 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
5389 SDValue LoHi =
5390 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5391 return SDValue(LoHi.getNode(), 1);
5392 }
5393 return SDValue(); // No mulhu or equivalent
5394 };
5395
5396 // Multiply the numerator (operand 0) by the magic value.
5397 Q = GetMULHU(Q, MagicFactor);
5398 if (!Q)
5399 return SDValue();
5400
5401 Created.push_back(Q.getNode());
5402
5403 if (UseNPQ) {
5404 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5405 Created.push_back(NPQ.getNode());
5406
5407 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5408 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
5409 if (VT.isVector())
5410 NPQ = GetMULHU(NPQ, NPQFactor);
5411 else
5412 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5413
5414 Created.push_back(NPQ.getNode());
5415
5416 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5417 Created.push_back(Q.getNode());
5418 }
5419
5420 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5421 Created.push_back(Q.getNode());
5422
5423 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5424
5425 SDValue One = DAG.getConstant(1, dl, VT);
5426 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
5427 return DAG.getSelect(dl, VT, IsOne, N0, Q);
5428}
5429
5430/// If all values in Values that *don't* match the predicate are same 'splat'
5431/// value, then replace all values with that splat value.
5432/// Else, if AlternativeReplacement was provided, then replace all values that
5433/// do match predicate with AlternativeReplacement value.
5434static void
5435turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5436 std::function<bool(SDValue)> Predicate,
5437 SDValue AlternativeReplacement = SDValue()) {
5438 SDValue Replacement;
5439 // Is there a value for which the Predicate does *NOT* match? What is it?
5440 auto SplatValue = llvm::find_if_not(Values, Predicate);
5441 if (SplatValue != Values.end()) {
5442 // Does Values consist only of SplatValue's and values matching Predicate?
5443 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5444 return Value == *SplatValue || Predicate(Value);
5445 })) // Then we shall replace values matching predicate with SplatValue.
5446 Replacement = *SplatValue;
5447 }
5448 if (!Replacement) {
5449 // Oops, we did not find the "baseline" splat value.
5450 if (!AlternativeReplacement)
5451 return; // Nothing to do.
5452 // Let's replace with provided value then.
5453 Replacement = AlternativeReplacement;
5454 }
5455 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5456}
5457
5458/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5459/// where the divisor is constant and the comparison target is zero,
5460/// return a DAG expression that will generate the same comparison result
5461/// using only multiplications, additions and shifts/rotations.
5462/// Ref: "Hacker's Delight" 10-17.
5463SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5464 SDValue CompTargetNode,
5465 ISD::CondCode Cond,
5466 DAGCombinerInfo &DCI,
5467 const SDLoc &DL) const {
5468 SmallVector<SDNode *, 5> Built;
5469 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5470 DCI, DL, Built)) {
5471 for (SDNode *N : Built)
5472 DCI.AddToWorklist(N);
5473 return Folded;
5474 }
5475
5476 return SDValue();
5477}
5478
5479SDValue
5480TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5481 SDValue CompTargetNode, ISD::CondCode Cond,
5482 DAGCombinerInfo &DCI, const SDLoc &DL,
5483 SmallVectorImpl<SDNode *> &Created) const {
5484 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5485 // - D must be constant, with D = D0 * 2^K where D0 is odd
5486 // - P is the multiplicative inverse of D0 modulo 2^W
5487 // - Q = floor(((2^W) - 1) / D)
5488 // where W is the width of the common type of N and D.
5489 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&((void)0)
5490 "Only applicable for (in)equality comparisons.")((void)0);
5491
5492 SelectionDAG &DAG = DCI.DAG;
5493
5494 EVT VT = REMNode.getValueType();
5495 EVT SVT = VT.getScalarType();
5496 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5497 EVT ShSVT = ShVT.getScalarType();
5498
5499 // If MUL is unavailable, we cannot proceed in any case.
5500 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5501 return SDValue();
5502
5503 bool ComparingWithAllZeros = true;
5504 bool AllComparisonsWithNonZerosAreTautological = true;
5505 bool HadTautologicalLanes = false;
5506 bool AllLanesAreTautological = true;
5507 bool HadEvenDivisor = false;
5508 bool AllDivisorsArePowerOfTwo = true;
5509 bool HadTautologicalInvertedLanes = false;
5510 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5511
5512 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5513 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5514 if (CDiv->isNullValue())
5515 return false;
5516
5517 const APInt &D = CDiv->getAPIntValue();
5518 const APInt &Cmp = CCmp->getAPIntValue();
5519
5520 ComparingWithAllZeros &= Cmp.isNullValue();
5521
5522 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5523 // if C2 is not less than C1, the comparison is always false.
5524 // But we will only be able to produce the comparison that will give the
5525 // opposive tautological answer. So this lane would need to be fixed up.
5526 bool TautologicalInvertedLane = D.ule(Cmp);
5527 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
5528
5529 // If all lanes are tautological (either all divisors are ones, or divisor
5530 // is not greater than the constant we are comparing with),
5531 // we will prefer to avoid the fold.
5532 bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;
5533 HadTautologicalLanes |= TautologicalLane;
5534 AllLanesAreTautological &= TautologicalLane;
5535
5536 // If we are comparing with non-zero, we need'll need to subtract said
5537 // comparison value from the LHS. But there is no point in doing that if
5538 // every lane where we are comparing with non-zero is tautological..
5539 if (!Cmp.isNullValue())
5540 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
5541
5542 // Decompose D into D0 * 2^K
5543 unsigned K = D.countTrailingZeros();
5544 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")((void)0);
5545 APInt D0 = D.lshr(K);
5546
5547 // D is even if it has trailing zeros.
5548 HadEvenDivisor |= (K != 0);
5549 // D is a power-of-two if D0 is one.
5550 // If all divisors are power-of-two, we will prefer to avoid the fold.
5551 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5552
5553 // P = inv(D0, 2^W)
5554 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5555 unsigned W = D.getBitWidth();
5556 APInt P = D0.zext(W + 1)
5557 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5558 .trunc(W);
5559 assert(!P.isNullValue() && "No multiplicative inverse!")((void)0); // unreachable
5560 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")((void)0);
5561
5562 // Q = floor((2^W - 1) u/ D)
5563 // R = ((2^W - 1) u% D)
5564 APInt Q, R;
5565 APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);
5566
5567 // If we are comparing with zero, then that comparison constant is okay,
5568 // else it may need to be one less than that.
5569 if (Cmp.ugt(R))
5570 Q -= 1;
5571
5572 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((void)0)
5573 "We are expecting that K is always less than all-ones for ShSVT")((void)0);
5574
5575 // If the lane is tautological the result can be constant-folded.
5576 if (TautologicalLane) {
5577 // Set P and K amount to a bogus values so we can try to splat them.
5578 P = 0;
5579 K = -1;
5580 // And ensure that comparison constant is tautological,
5581 // it will always compare true/false.
5582 Q = -1;
5583 }
5584
5585 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5586 KAmts.push_back(
5587 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5588 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5589 return true;
5590 };
5591
5592 SDValue N = REMNode.getOperand(0);
5593 SDValue D = REMNode.getOperand(1);
5594
5595 // Collect the values from each element.
5596 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5597 return SDValue();
5598
5599 // If all lanes are tautological, the result can be constant-folded.
5600 if (AllLanesAreTautological)
5601 return SDValue();
5602
5603 // If this is a urem by a powers-of-two, avoid the fold since it can be
5604 // best implemented as a bit test.
5605 if (AllDivisorsArePowerOfTwo)
5606 return SDValue();
5607
5608 SDValue PVal, KVal, QVal;
5609 if (D.getOpcode() == ISD::BUILD_VECTOR) {
5610 if (HadTautologicalLanes) {
5611 // Try to turn PAmts into a splat, since we don't care about the values
5612 // that are currently '0'. If we can't, just keep '0'`s.
5613 turnVectorIntoSplatVector(PAmts, isNullConstant);
5614 // Try to turn KAmts into a splat, since we don't care about the values
5615 // that are currently '-1'. If we can't, change them to '0'`s.
5616 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5617 DAG.getConstant(0, DL, ShSVT));
5618 }
5619
5620 PVal = DAG.getBuildVector(VT, DL, PAmts);
5621 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5622 QVal = DAG.getBuildVector(VT, DL, QAmts);
5623 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
5624 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&((void)0)
5625 "Expected matchBinaryPredicate to return one element for "((void)0)
5626 "SPLAT_VECTORs")((void)0);
5627 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
5628 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
5629 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
5630 } else {
5631 PVal = PAmts[0];
5632 KVal = KAmts[0];
5633 QVal = QAmts[0];
5634 }
5635
5636 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
5637 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
5638 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5639 assert(CompTargetNode.getValueType() == N.getValueType() &&((void)0)
5640 "Expecting that the types on LHS and RHS of comparisons match.")((void)0);
5641 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5642 }
5643
5644 // (mul N, P)
5645 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5646 Created.push_back(Op0.getNode());
5647
5648 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5649 // divisors as a performance improvement, since rotating by 0 is a no-op.
5650 if (HadEvenDivisor) {
5651 // We need ROTR to do this.
5652 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5653 return SDValue();
5654 // UREM: (rotr (mul N, P), K)
5655 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
5656 Created.push_back(Op0.getNode());
5657 }
5658
5659 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
5660 SDValue NewCC =
5661 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5662 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5663 if (!HadTautologicalInvertedLanes)
5664 return NewCC;
5665
5666 // If any lanes previously compared always-false, the NewCC will give
5667 // always-true result for them, so we need to fixup those lanes.
5668 // Or the other way around for inequality predicate.
5669 assert(VT.isVector() && "Can/should only get here for vectors.")((void)0);
5670 Created.push_back(NewCC.getNode());
5671
5672 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5673 // if C2 is not less than C1, the comparison is always false.
5674 // But we have produced the comparison that will give the
5675 // opposive tautological answer. So these lanes would need to be fixed up.
5676 SDValue TautologicalInvertedChannels =
5677 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5678 Created.push_back(TautologicalInvertedChannels.getNode());
5679
5680 // NOTE: we avoid letting illegal types through even if we're before legalize
5681 // ops – legalization has a hard time producing good code for this.
5682 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5683 // If we have a vector select, let's replace the comparison results in the
5684 // affected lanes with the correct tautological result.
5685 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5686 DL, SETCCVT, SETCCVT);
5687 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5688 Replacement, NewCC);
5689 }
5690
5691 // Else, we can just invert the comparison result in the appropriate lanes.
5692 //
5693 // NOTE: see the note above VSELECT above.
5694 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5695 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5696 TautologicalInvertedChannels);
5697
5698 return SDValue(); // Don't know how to lower.
5699}
5700
5701/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5702/// where the divisor is constant and the comparison target is zero,
5703/// return a DAG expression that will generate the same comparison result
5704/// using only multiplications, additions and shifts/rotations.
5705/// Ref: "Hacker's Delight" 10-17.
5706SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5707 SDValue CompTargetNode,
5708 ISD::CondCode Cond,
5709 DAGCombinerInfo &DCI,
5710 const SDLoc &DL) const {
5711 SmallVector<SDNode *, 7> Built;
5712 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5713 DCI, DL, Built)) {
5714 assert(Built.size() <= 7 && "Max size prediction failed.")((void)0);
5715 for (SDNode *N : Built)
5716 DCI.AddToWorklist(N);
5717 return Folded;
5718 }
5719
5720 return SDValue();
5721}
5722
5723SDValue
5724TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5725 SDValue CompTargetNode, ISD::CondCode Cond,
5726 DAGCombinerInfo &DCI, const SDLoc &DL,
5727 SmallVectorImpl<SDNode *> &Created) const {
5728 // Fold:
5729 // (seteq/ne (srem N, D), 0)
5730 // To:
5731 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
5732 //
5733 // - D must be constant, with D = D0 * 2^K where D0 is odd
5734 // - P is the multiplicative inverse of D0 modulo 2^W
5735 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
5736 // - Q = floor((2 * A) / (2^K))
5737 // where W is the width of the common type of N and D.
5738 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&((void)0)
5739 "Only applicable for (in)equality comparisons.")((void)0);
5740
5741 SelectionDAG &DAG = DCI.DAG;
5742
5743 EVT VT = REMNode.getValueType();
5744 EVT SVT = VT.getScalarType();
5745 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5746 EVT ShSVT = ShVT.getScalarType();
5747
5748 // If we are after ops legalization, and MUL is unavailable, we can not
5749 // proceed.
5750 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5751 return SDValue();
5752
5753 // TODO: Could support comparing with non-zero too.
5754 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
5755 if (!CompTarget || !CompTarget->isNullValue())
5756 return SDValue();
5757
5758 bool HadIntMinDivisor = false;
5759 bool HadOneDivisor = false;
5760 bool AllDivisorsAreOnes = true;
5761 bool HadEvenDivisor = false;
5762 bool NeedToApplyOffset = false;
5763 bool AllDivisorsArePowerOfTwo = true;
5764 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5765
5766 auto BuildSREMPattern = [&](ConstantSDNode *C) {
5767 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5768 if (C->isNullValue())
5769 return false;
5770
5771 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
5772
5773 // WARNING: this fold is only valid for positive divisors!
5774 APInt D = C->getAPIntValue();
5775 if (D.isNegative())
5776 D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`
5777
5778 HadIntMinDivisor |= D.isMinSignedValue();
5779
5780 // If all divisors are ones, we will prefer to avoid the fold.
5781 HadOneDivisor |= D.isOneValue();
5782 AllDivisorsAreOnes &= D.isOneValue();
5783
5784 // Decompose D into D0 * 2^K
5785 unsigned K = D.countTrailingZeros();
5786 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")((void)0);
5787 APInt D0 = D.lshr(K);
5788
5789 if (!D.isMinSignedValue()) {
5790 // D is even if it has trailing zeros; unless it's INT_MIN, in which case
5791 // we don't care about this lane in this fold, we'll special-handle it.
5792 HadEvenDivisor |= (K != 0);
5793 }
5794
5795 // D is a power-of-two if D0 is one. This includes INT_MIN.
5796 // If all divisors are power-of-two, we will prefer to avoid the fold.
5797 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5798
5799 // P = inv(D0, 2^W)
5800 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5801 unsigned W = D.getBitWidth();
5802 APInt P = D0.zext(W + 1)
5803 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5804 .trunc(W);
5805 assert(!P.isNullValue() && "No multiplicative inverse!")((void)0); // unreachable
5806 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")((void)0);
5807
5808 // A = floor((2^(W - 1) - 1) / D0) & -2^K
5809 APInt A = APInt::getSignedMaxValue(W).udiv(D0);
5810 A.clearLowBits(K);
5811
5812 if (!D.isMinSignedValue()) {
5813 // If divisor INT_MIN, then we don't care about this lane in this fold,
5814 // we'll special-handle it.
5815 NeedToApplyOffset |= A != 0;
5816 }
5817
5818 // Q = floor((2 * A) / (2^K))
5819 APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
5820
5821 assert(APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&((void)0)
5822 "We are expecting that A is always less than all-ones for SVT")((void)0);
5823 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((void)0)
5824 "We are expecting that K is always less than all-ones for ShSVT")((void)0);
5825
5826 // If the divisor is 1 the result can be constant-folded. Likewise, we
5827 // don't care about INT_MIN lanes, those can be set to undef if appropriate.
5828 if (D.isOneValue()) {
5829 // Set P, A and K to a bogus values so we can try to splat them.
5830 P = 0;
5831 A = -1;
5832 K = -1;
5833
5834 // x ?% 1 == 0 <--> true <--> x u<= -1
5835 Q = -1;
5836 }
5837
5838 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5839 AAmts.push_back(DAG.getConstant(A, DL, SVT));
5840 KAmts.push_back(
5841 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5842 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5843 return true;
5844 };
5845
5846 SDValue N = REMNode.getOperand(0);
5847 SDValue D = REMNode.getOperand(1);
5848
5849 // Collect the values from each element.
5850 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
5851 return SDValue();
5852
5853 // If this is a srem by a one, avoid the fold since it can be constant-folded.
5854 if (AllDivisorsAreOnes)
5855 return SDValue();
5856
5857 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
5858 // since it can be best implemented as a bit test.
5859 if (AllDivisorsArePowerOfTwo)
5860 return SDValue();
5861
5862 SDValue PVal, AVal, KVal, QVal;
5863 if (D.getOpcode() == ISD::BUILD_VECTOR) {
5864 if (HadOneDivisor) {
5865 // Try to turn PAmts into a splat, since we don't care about the values
5866 // that are currently '0'. If we can't, just keep '0'`s.
5867 turnVectorIntoSplatVector(PAmts, isNullConstant);
5868 // Try to turn AAmts into a splat, since we don't care about the
5869 // values that are currently '-1'. If we can't, change them to '0'`s.
5870 turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
5871 DAG.getConstant(0, DL, SVT));
5872 // Try to turn KAmts into a splat, since we don't care about the values
5873 // that are currently '-1'. If we can't, change them to '0'`s.
5874 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5875 DAG.getConstant(0, DL, ShSVT));
5876 }
5877
5878 PVal = DAG.getBuildVector(VT, DL, PAmts);
5879 AVal = DAG.getBuildVector(VT, DL, AAmts);
5880 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5881 QVal = DAG.getBuildVector(VT, DL, QAmts);
5882 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
5883 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&((void)0)
5884 QAmts.size() == 1 &&((void)0)
5885 "Expected matchUnaryPredicate to return one element for scalable "((void)0)
5886 "vectors")((void)0);
5887 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
5888 AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
5889 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
5890 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
5891 } else {
5892 assert(isa<ConstantSDNode>(D) && "Expected a constant")((void)0);
5893 PVal = PAmts[0];
5894 AVal = AAmts[0];
5895 KVal = KAmts[0];
5896 QVal = QAmts[0];
5897 }
5898
5899 // (mul N, P)
5900 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5901 Created.push_back(Op0.getNode());
5902
5903 if (NeedToApplyOffset) {
5904 // We need ADD to do this.
5905 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
5906 return SDValue();
5907
5908 // (add (mul N, P), A)
5909 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
5910 Created.push_back(Op0.getNode());
5911 }
5912
5913 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5914 // divisors as a performance improvement, since rotating by 0 is a no-op.
5915 if (HadEvenDivisor) {
5916 // We need ROTR to do this.
5917 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5918 return SDValue();
5919 // SREM: (rotr (add (mul N, P), A), K)
5920 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
5921 Created.push_back(Op0.getNode());
5922 }
5923
5924 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
5925 SDValue Fold =
5926 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5927 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5928
5929 // If we didn't have lanes with INT_MIN divisor, then we're done.
5930 if (!HadIntMinDivisor)
5931 return Fold;
5932
5933 // That fold is only valid for positive divisors. Which effectively means,
5934 // it is invalid for INT_MIN divisors. So if we have such a lane,
5935 // we must fix-up results for said lanes.
5936 assert(VT.isVector() && "Can/should only get here for vectors.")((void)0);
5937
5938 // NOTE: we avoid letting illegal types through even if we're before legalize
5939 // ops – legalization has a hard time producing good code for the code that
5940 // follows.
5941 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
5942 !isOperationLegalOrCustom(ISD::AND, VT) ||
5943 !isOperationLegalOrCustom(Cond, VT) ||
5944 !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT))
5945 return SDValue();
5946
5947 Created.push_back(Fold.getNode());
5948
5949 SDValue IntMin = DAG.getConstant(
5950 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
5951 SDValue IntMax = DAG.getConstant(
5952 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
5953 SDValue Zero =
5954 DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT);
5955
5956 // Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
5957 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5958 Created.push_back(DivisorIsIntMin.getNode());
5959
5960 // (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0
5961 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5962 Created.push_back(Masked.getNode());
5963 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
5964 Created.push_back(MaskedIsZero.getNode());
5965
5966 // To produce final result we need to blend 2 vectors: 'SetCC' and
5967 // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
5968 // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
5969 // constant-folded, select can get lowered to a shuffle with constant mask.
5970 SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin,
5971 MaskedIsZero, Fold);
5972
5973 return Blended;
5974}
5975
5976bool TargetLowering::
5977verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
5978 if (!isa<ConstantSDNode>(Op.getOperand(0))) {
5979 DAG.getContext()->emitError("argument to '__builtin_return_address' must "
5980 "be a constant integer");
5981 return true;
5982 }
5983
5984 return false;
5985}
5986
5987SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
5988 const DenormalMode &Mode) const {
5989 SDLoc DL(Op);
5990 EVT VT = Op.getValueType();
5991 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5992 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
5993 // Testing it with denormal inputs to avoid wrong estimate.
5994 if (Mode.Input == DenormalMode::IEEE) {
5995 // This is specifically a check for the handling of denormal inputs,
5996 // not the result.
5997
5998 // Test = fabs(X) < SmallestNormal
5999 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
6000 APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
6001 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
6002 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
6003 return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
6004 }
6005 // Test = X == 0.0
6006 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
6007}
6008
6009SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
6010 bool LegalOps, bool OptForSize,
6011 NegatibleCost &Cost,
6012 unsigned Depth) const {
6013 // fneg is removable even if it has multiple uses.
6014 if (Op.getOpcode() == ISD::FNEG) {
6015 Cost = NegatibleCost::Cheaper;
6016 return Op.getOperand(0);
6017 }
6018
6019 // Don't recurse exponentially.
6020 if (Depth > SelectionDAG::MaxRecursionDepth)
6021 return SDValue();
6022
6023 // Pre-increment recursion depth for use in recursive calls.
6024 ++Depth;
6025 const SDNodeFlags Flags = Op->getFlags();
6026 const TargetOptions &Options = DAG.getTarget().Options;
6027 EVT VT = Op.getValueType();
6028 unsigned Opcode = Op.getOpcode();
6029
6030 // Don't allow anything with multiple uses unless we know it is free.
6031 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
6032 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
6033 isFPExtFree(VT, Op.getOperand(0).getValueType());
6034 if (!IsFreeExtend)
6035 return SDValue();
6036 }
6037
6038 auto RemoveDeadNode = [&](SDValue N) {
6039 if (N && N.getNode()->use_empty())
6040 DAG.RemoveDeadNode(N.getNode());
6041 };
6042
6043 SDLoc DL(Op);
6044
6045 // Because getNegatedExpression can delete nodes we need a handle to keep
6046 // temporary nodes alive in case the recursion manages to create an identical
6047 // node.
6048 std::list<HandleSDNode> Handles;
6049
6050 switch (Opcode) {
6051 case ISD::ConstantFP: {
6052 // Don't invert constant FP values after legalization unless the target says
6053 // the negated constant is legal.
6054 bool IsOpLegal =
6055 isOperationLegal(ISD::ConstantFP, VT) ||
6056 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
6057 OptForSize);
6058
6059 if (LegalOps && !IsOpLegal)
6060 break;
6061
6062 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
6063 V.changeSign();
6064 SDValue CFP = DAG.getConstantFP(V, DL, VT);
6065
6066 // If we already have the use of the negated floating constant, it is free
6067 // to negate it even it has multiple uses.
6068 if (!Op.hasOneUse() && CFP.use_empty())
6069 break;
6070 Cost = NegatibleCost::Neutral;
6071 return CFP;
6072 }
6073 case ISD::BUILD_VECTOR: {
6074 // Only permit BUILD_VECTOR of constants.
6075 if (llvm::any_of(Op->op_values(), [&](SDValue N) {
6076 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
6077 }))
6078 break;
6079
6080 bool IsOpLegal =
6081 (isOperationLegal(ISD::ConstantFP, VT) &&
6082 isOperationLegal(ISD::BUILD_VECTOR, VT)) ||
6083 llvm::all_of(Op->op_values(), [&](SDValue N) {
6084 return N.isUndef() ||
6085 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
6086 OptForSize);
6087 });
6088
6089 if (LegalOps && !IsOpLegal)
6090 break;
6091
6092 SmallVector<SDValue, 4> Ops;
6093 for (SDValue C : Op->op_values()) {
6094 if (C.isUndef()) {
6095 Ops.push_back(C);
6096 continue;
6097 }
6098 APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
6099 V.changeSign();
6100 Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
6101 }
6102 Cost = NegatibleCost::Neutral;
6103 return DAG.getBuildVector(VT, DL, Ops);
6104 }
6105 case ISD::FADD: {
6106 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6107 break;
6108
6109 // After operation legalization, it might not be legal to create new FSUBs.
6110 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
6111 break;
6112 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6113
6114 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
6115 NegatibleCost CostX = NegatibleCost::Expensive;
6116 SDValue NegX =
6117 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6118 // Prevent this node from being deleted by the next call.
6119 if (NegX)
6120 Handles.emplace_back(NegX);
6121
6122 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
6123 NegatibleCost CostY = NegatibleCost::Expensive;
6124 SDValue NegY =
6125 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6126
6127 // We're done with the handles.
6128 Handles.clear();
6129
6130 // Negate the X if its cost is less or equal than Y.
6131 if (NegX && (CostX <= CostY)) {
6132 Cost = CostX;
6133 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
6134 if (NegY != N)
6135 RemoveDeadNode(NegY);
6136 return N;
6137 }
6138
6139 // Negate the Y if it is not expensive.
6140 if (NegY) {
6141 Cost = CostY;
6142 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
6143 if (NegX != N)
6144 RemoveDeadNode(NegX);
6145 return N;
6146 }
6147 break;
6148 }
6149 case ISD::FSUB: {
6150 // We can't turn -(A-B) into B-A when we honor signed zeros.
6151 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6152 break;
6153
6154 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6155 // fold (fneg (fsub 0, Y)) -> Y
6156 if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
6157 if (C->isZero()) {
6158 Cost = NegatibleCost::Cheaper;
6159 return Y;
6160 }
6161
6162 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
6163 Cost = NegatibleCost::Neutral;
6164 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
6165 }
6166 case ISD::FMUL:
6167 case ISD::FDIV: {
6168 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6169
6170 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
6171 NegatibleCost CostX = NegatibleCost::Expensive;
6172 SDValue NegX =
6173 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6174 // Prevent this node from being deleted by the next call.
6175 if (NegX)
6176 Handles.emplace_back(NegX);
6177
6178 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
6179 NegatibleCost CostY = NegatibleCost::Expensive;
6180 SDValue NegY =
6181 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6182
6183 // We're done with the handles.
6184 Handles.clear();
6185
6186 // Negate the X if its cost is less or equal than Y.
6187 if (NegX && (CostX <= CostY)) {
6188 Cost = CostX;
6189 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
6190 if (NegY != N)
6191 RemoveDeadNode(NegY);
6192 return N;
6193 }
6194
6195 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
6196 if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
6197 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
6198 break;
6199
6200 // Negate the Y if it is not expensive.
6201 if (NegY) {
6202 Cost = CostY;
6203 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
6204 if (NegX != N)
6205 RemoveDeadNode(NegX);
6206 return N;
6207 }
6208 break;
6209 }
6210 case ISD::FMA:
6211 case ISD::FMAD: {
6212 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6213 break;
6214
6215 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
6216 NegatibleCost CostZ = NegatibleCost::Expensive;
6217 SDValue NegZ =
6218 getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
6219 // Give up if fail to negate the Z.
6220 if (!NegZ)
6221 break;
6222
6223 // Prevent this node from being deleted by the next two calls.
6224 Handles.emplace_back(NegZ);
6225
6226 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
6227 NegatibleCost CostX = NegatibleCost::Expensive;
6228 SDValue NegX =
6229 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6230 // Prevent this node from being deleted by the next call.
6231 if (NegX)
6232 Handles.emplace_back(NegX);
6233
6234 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
6235 NegatibleCost CostY = NegatibleCost::Expensive;
6236 SDValue NegY =
6237 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6238
6239 // We're done with the handles.
6240 Handles.clear();
6241
6242 // Negate the X if its cost is less or equal than Y.
6243 if (NegX && (CostX <= CostY)) {
6244 Cost = std::min(CostX, CostZ);
6245 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
6246 if (NegY != N)
6247 RemoveDeadNode(NegY);
6248 return N;
6249 }
6250
6251 // Negate the Y if it is not expensive.
6252 if (NegY) {
6253 Cost = std::min(CostY, CostZ);
6254 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
6255 if (NegX != N)
6256 RemoveDeadNode(NegX);
6257 return N;
6258 }
6259 break;
6260 }
6261
6262 case ISD::FP_EXTEND:
6263 case ISD::FSIN:
6264 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
6265 OptForSize, Cost, Depth))
6266 return DAG.getNode(Opcode, DL, VT, NegV);
6267 break;
6268 case ISD::FP_ROUND:
6269 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
6270 OptForSize, Cost, Depth))
6271 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
6272 break;
6273 }
6274
6275 return SDValue();
6276}
6277
6278//===----------------------------------------------------------------------===//
6279// Legalization Utilities
6280//===----------------------------------------------------------------------===//
6281
6282bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
6283 SDValue LHS, SDValue RHS,
6284 SmallVectorImpl<SDValue> &Result,
6285 EVT HiLoVT, SelectionDAG &DAG,
6286 MulExpansionKind Kind, SDValue LL,
6287 SDValue LH, SDValue RL, SDValue RH) const {
6288 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||((void)0)
6289 Opcode == ISD::SMUL_LOHI)((void)0);
6290
6291 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
6292 isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
6293 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
6294 isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
6295 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
6296 isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
6297 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
6298 isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);
6299
6300 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
6301 return false;
6302
6303 unsigned OuterBitSize = VT.getScalarSizeInBits();
6304 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
6305
6306 // LL, LH, RL, and RH must be either all NULL or all set to a value.
6307 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||((void)0)
6308 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()))((void)0);
6309
6310 SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
6311 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
6312 bool Signed) -> bool {
6313 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
6314 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
6315 Hi = SDValue(Lo.getNode(), 1);
6316 return true;
6317 }
6318 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
6319 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
6320 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
6321 return true;
6322 }
6323 return false;
6324 };
6325
6326 SDValue Lo, Hi;
6327
6328 if (!LL.getNode() && !RL.getNode() &&
6329 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
6330 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
6331 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
6332 }
6333
6334 if (!LL.getNode())
6335 return false;
6336
6337 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
6338 if (DAG.MaskedValueIsZero(LHS, HighMask) &&
6339 DAG.MaskedValueIsZero(RHS, HighMask)) {
6340 // The inputs are both zero-extended.
6341 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
6342 Result.push_back(Lo);
6343 Result.push_back(Hi);
6344 if (Opcode != ISD::MUL) {
6345 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6346 Result.push_back(Zero);
6347 Result.push_back(Zero);
6348 }
6349 return true;
6350 }
6351 }
6352
6353 if (!VT.isVector() && Opcode == ISD::MUL &&
6354 DAG.ComputeNumSignBits(LHS) > InnerBitSize &&
6355 DAG.ComputeNumSignBits(RHS) > InnerBitSize) {
6356 // The input values are both sign-extended.
6357 // TODO non-MUL case?
6358 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
6359 Result.push_back(Lo);
6360 Result.push_back(Hi);
6361 return true;
6362 }
6363 }
6364
6365 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
6366 EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
6367 if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
6368 // FIXME getShiftAmountTy does not always return a sensible result when VT
6369 // is an illegal type, and so the type may be too small to fit the shift
6370 // amount. Override it with i32. The shift will have to be legalized.
6371 ShiftAmountTy = MVT::i32;
6372 }
6373 SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
6374
6375 if (!LH.getNode() && !RH.getNode() &&
6376 isOperationLegalOrCustom(ISD::SRL, VT) &&
6377 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
6378 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
6379 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
6380 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
6381 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
6382 }
6383
6384 if (!LH.getNode())
6385 return false;
6386
6387 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
6388 return false;
6389
6390 Result.push_back(Lo);
6391
6392 if (Opcode == ISD::MUL) {
6393 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
6394 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
6395 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
6396 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
6397 Result.push_back(Hi);
6398 return true;
6399 }
6400
6401 // Compute the full width result.
6402 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
6403 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
6404 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6405 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
6406 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
6407 };
6408
6409 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6410 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
6411 return false;
6412
6413 // This is effectively the add part of a multiply-add of half-sized operands,
6414 // so it cannot overflow.
6415 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6416
6417 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
6418 return false;
6419
6420 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6421 EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
6422
6423 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
6424 isOperationLegalOrCustom(ISD::ADDE, VT));
6425 if (UseGlue)
6426 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
6427 Merge(Lo, Hi));
6428 else
6429 Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
6430 Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
6431
6432 SDValue Carry = Next.getValue(1);
6433 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6434 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6435
6436 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
6437 return false;
6438
6439 if (UseGlue)
6440 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
6441 Carry);
6442 else
6443 Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
6444 Zero, Carry);
6445
6446 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6447
6448 if (Opcode == ISD::SMUL_LOHI) {
6449 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6450 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
6451 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
6452
6453 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6454 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
6455 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
6456 }
6457
6458 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6459 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6460 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6461 return true;
6462}
6463
6464bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
6465 SelectionDAG &DAG, MulExpansionKind Kind,
6466 SDValue LL, SDValue LH, SDValue RL,
6467 SDValue RH) const {
6468 SmallVector<SDValue, 2> Result;
6469 bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
6470 N->getOperand(0), N->getOperand(1), Result, HiLoVT,
6471 DAG, Kind, LL, LH, RL, RH);
6472 if (Ok) {
6473 assert(Result.size() == 2)((void)0);
6474 Lo = Result[0];
6475 Hi = Result[1];
6476 }
6477 return Ok;
6478}
6479
6480// Check that (every element of) Z is undef or not an exact multiple of BW.
6481static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
6482 return ISD::matchUnaryPredicate(
6483 Z,
6484 [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
6485 true);
6486}
6487
6488bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
6489 SelectionDAG &DAG) const {
6490 EVT VT = Node->getValueType(0);
6491
6492 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6493 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6494 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6495 !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
6496 return false;
6497
6498 SDValue X = Node->getOperand(0);
6499 SDValue Y = Node->getOperand(1);
6500 SDValue Z = Node->getOperand(2);
6501
6502 unsigned BW = VT.getScalarSizeInBits();
6503 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
6504 SDLoc DL(SDValue(Node, 0));
6505
6506 EVT ShVT = Z.getValueType();
6507
6508 // If a funnel shift in the other direction is more supported, use it.
6509 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
6510 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
6511 isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {
6512 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
6513 // fshl X, Y, Z -> fshr X, Y, -Z
6514 // fshr X, Y, Z -> fshl X, Y, -Z
6515 SDValue Zero = DAG.getConstant(0, DL, ShVT);
6516 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z);
6517 } else {
6518 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
6519 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
6520 SDValue One = DAG.getConstant(1, DL, ShVT);
6521 if (IsFSHL) {
6522 Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
6523 X = DAG.getNode(ISD::SRL, DL, VT, X, One);
6524 } else {
6525 X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
6526 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);
6527 }
6528 Z = DAG.getNOT(DL, Z, ShVT);
6529 }
6530 Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
6531 return true;
6532 }
6533
6534 SDValue ShX, ShY;
6535 SDValue ShAmt, InvShAmt;
6536 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
6537 // fshl: X << C | Y >> (BW - C)
6538 // fshr: X << (BW - C) | Y >> C
6539 // where C = Z % BW is not zero
6540 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6541 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6542 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
6543 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
6544 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
6545 } else {
6546 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
6547 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
6548 SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
6549 if (isPowerOf2_32(BW)) {
6550 // Z % BW -> Z & (BW - 1)
6551 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
6552 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
6553 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
6554 } else {
6555 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6556 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6557 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
6558 }
6559
6560 SDValue One = DAG.getConstant(1, DL, ShVT);
6561 if (IsFSHL) {
6562 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
6563 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
6564 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
6565 } else {
6566 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
6567 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
6568 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
6569 }
6570 }
6571 Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
6572 return true;
6573}
6574
6575// TODO: Merge with expandFunnelShift.
6576bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
6577 SDValue &Result, SelectionDAG &DAG) const {
6578 EVT VT = Node->getValueType(0);
6579 unsigned EltSizeInBits = VT.getScalarSizeInBits();
6580 bool IsLeft = Node->getOpcode() == ISD::ROTL;
6581 SDValue Op0 = Node->getOperand(0);
6582 SDValue Op1 = Node->getOperand(1);
6583 SDLoc DL(SDValue(Node, 0));
6584
6585 EVT ShVT = Op1.getValueType();
6586 SDValue Zero = DAG.getConstant(0, DL, ShVT);
6587
6588 // If a rotate in the other direction is supported, use it.
6589 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
6590 if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
6591 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6592 Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
6593 return true;
6594 }
6595
6596 if (!AllowVectorOps && VT.isVector() &&
6597 (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6598 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6599 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6600 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
6601 !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
6602 return false;
6603
6604 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
6605 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
6606 SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
6607 SDValue ShVal;
6608 SDValue HsVal;
6609 if (isPowerOf2_32(EltSizeInBits)) {
6610 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
6611 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
6612 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6613 SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
6614 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
6615 SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
6616 HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);
6617 } else {
6618 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
6619 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
6620 SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
6621 SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);
6622 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
6623 SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);
6624 SDValue One = DAG.getConstant(1, DL, ShVT);
6625 HsVal =
6626 DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
6627 }
6628 Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
6629 return true;
6630}
6631
6632void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
6633 SelectionDAG &DAG) const {
6634 assert(Node->getNumOperands() == 3 && "Not a double-shift!")((void)0);
6635 EVT VT = Node->getValueType(0);
6636 unsigned VTBits = VT.getScalarSizeInBits();
6637 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected")((void)0);
6638
6639 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
6640 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
6641 SDValue ShOpLo = Node->getOperand(0);
6642 SDValue ShOpHi = Node->getOperand(1);
6643 SDValue ShAmt = Node->getOperand(2);
6644 EVT ShAmtVT = ShAmt.getValueType();
6645 EVT ShAmtCCVT =
6646 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);
6647 SDLoc dl(Node);
6648
6649 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
6650 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
6651 // away during isel.
6652 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
6653 DAG.getConstant(VTBits - 1, dl, ShAmtVT));
6654 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
6655 DAG.getConstant(VTBits - 1, dl, ShAmtVT))
6656 : DAG.getConstant(0, dl, VT);
6657
6658 SDValue Tmp2, Tmp3;
6659 if (IsSHL) {
6660 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
6661 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
6662 } else {
6663 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
6664 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
6665 }
6666
6667 // If the shift amount is larger or equal than the width of a part we don't
6668 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
6669 // values for large shift amounts.
6670 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
6671 DAG.getConstant(VTBits, dl, ShAmtVT));
6672 SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,
6673 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);
6674
6675 if (IsSHL) {
6676 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
6677 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
6678 } else {
6679 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
6680 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
6681 }
6682}
6683
6684bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
6685 SelectionDAG &DAG) const {
6686 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
6687 SDValue Src = Node->getOperand(OpNo);
6688 EVT SrcVT = Src.getValueType();
6689 EVT DstVT = Node->getValueType(0);
6690 SDLoc dl(SDValue(Node, 0));
6691
6692 // FIXME: Only f32 to i64 conversions are supported.
6693 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
6694 return false;
6695
6696 if (Node->isStrictFPOpcode())
6697 // When a NaN is converted to an integer a trap is allowed. We can't
6698 // use this expansion here because it would eliminate that trap. Other
6699 // traps are also allowed and cannot be eliminated. See
6700 // IEEE 754-2008 sec 5.8.
6701 return false;
6702
6703 // Expand f32 -> i64 conversion
6704 // This algorithm comes from compiler-rt's implementation of fixsfdi:
6705 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
6706 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
6707 EVT IntVT = SrcVT.changeTypeToInteger();
6708 EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
6709
6710 SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
6711 SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
6712 SDValue Bias = DAG.getConstant(127, dl, IntVT);
6713 SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
6714 SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
6715 SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
6716
6717 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
6718
6719 SDValue ExponentBits = DAG.getNode(
6720 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
6721 DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
6722 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
6723
6724 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
6725 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
6726 DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
6727 Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);
6728
6729 SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
6730 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
6731 DAG.getConstant(0x00800000, dl, IntVT));
6732
6733 R = DAG.getZExtOrTrunc(R, dl, DstVT);
6734
6735 R = DAG.getSelectCC(
6736 dl, Exponent, ExponentLoBit,
6737 DAG.getNode(ISD::SHL, dl, DstVT, R,
6738 DAG.getZExtOrTrunc(
6739 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
6740 dl, IntShVT)),
6741 DAG.getNode(ISD::SRL, dl, DstVT, R,
6742 DAG.getZExtOrTrunc(
6743 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
6744 dl, IntShVT)),
6745 ISD::SETGT);
6746
6747 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
6748 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
6749
6750 Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
6751 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
6752 return true;
6753}
6754
6755bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
6756 SDValue &Chain,
6757 SelectionDAG &DAG) const {
6758 SDLoc dl(SDValue(Node, 0));
6759 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
6760 SDValue Src = Node->getOperand(OpNo);
6761
6762 EVT SrcVT = Src.getValueType();
6763 EVT DstVT = Node->getValueType(0);
6764 EVT SetCCVT =
6765 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
6766 EVT DstSetCCVT =
6767 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);
6768
6769 // Only expand vector types if we have the appropriate vector bit operations.
6770 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
6771 ISD::FP_TO_SINT;
6772 if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||
6773 !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))
6774 return false;
6775
6776 // If the maximum float value is smaller then the signed integer range,
6777 // the destination signmask can't be represented by the float, so we can
6778 // just use FP_TO_SINT directly.
6779 const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);
6780 APFloat APF(APFSem, APInt::getNullValue(SrcVT.getScalarSizeInBits()));
6781 APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
6782 if (APFloat::opOverflow &
6783 APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
6784 if (Node->isStrictFPOpcode()) {
6785 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6786 { Node->getOperand(0), Src });
6787 Chain = Result.getValue(1);
6788 } else
6789 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6790 return true;
6791 }
6792
6793 // Don't expand it if there isn't cheap fsub instruction.
6794 if (!isOperationLegalOrCustom(
6795 Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))
6796 return false;
6797
6798 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
6799 SDValue Sel;
6800
6801 if (Node->isStrictFPOpcode()) {
6802 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
6803 Node->getOperand(0), /*IsSignaling*/ true);
6804 Chain = Sel.getValue(1);
6805 } else {
6806 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
6807 }
6808
6809 bool Strict = Node->isStrictFPOpcode() ||
6810 shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);
6811
6812 if (Strict) {
6813 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
6814 // signmask then offset (the result of which should be fully representable).
6815 // Sel = Src < 0x8000000000000000
6816 // FltOfs = select Sel, 0, 0x8000000000000000
6817 // IntOfs = select Sel, 0, 0x8000000000000000
6818 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
6819
6820 // TODO: Should any fast-math-flags be set for the FSUB?
6821 SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
6822 DAG.getConstantFP(0.0, dl, SrcVT), Cst);
6823 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
6824 SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
6825 DAG.getConstant(0, dl, DstVT),
6826 DAG.getConstant(SignMask, dl, DstVT));
6827 SDValue SInt;
6828 if (Node->isStrictFPOpcode()) {
6829 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
6830 { Chain, Src, FltOfs });
6831 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6832 { Val.getValue(1), Val });
6833 Chain = SInt.getValue(1);
6834 } else {
6835 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
6836 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
6837 }
6838 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
6839 } else {
6840 // Expand based on maximum range of FP_TO_SINT:
6841 // True = fp_to_sint(Src)
6842 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
6843 // Result = select (Src < 0x8000000000000000), True, False
6844
6845 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6846 // TODO: Should any fast-math-flags be set for the FSUB?
6847 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
6848 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
6849 False = DAG.getNode(ISD::XOR, dl, DstVT, False,
6850 DAG.getConstant(SignMask, dl, DstVT));
6851 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
6852 Result = DAG.getSelect(dl, DstVT, Sel, True, False);
6853 }
6854 return true;
6855}
6856
6857bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
6858 SDValue &Chain,
6859 SelectionDAG &DAG) const {
6860 // This transform is not correct for converting 0 when rounding mode is set
6861 // to round toward negative infinity which will produce -0.0. So disable under
6862 // strictfp.
6863 if (Node->isStrictFPOpcode())
6864 return false;
6865
6866 SDValue Src = Node->getOperand(0);
6867 EVT SrcVT = Src.getValueType();
6868 EVT DstVT = Node->getValueType(0);
6869
6870 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
6871 return false;
6872
6873 // Only expand vector types if we have the appropriate vector bit operations.
6874 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
6875 !isOperationLegalOrCustom(ISD::FADD, DstVT) ||
6876 !isOperationLegalOrCustom(ISD::FSUB, DstVT) ||
6877 !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||
6878 !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))
6879 return false;
6880
6881 SDLoc dl(SDValue(Node, 0));
6882 EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());
6883
6884 // Implementation of unsigned i64 to f64 following the algorithm in
6885 // __floatundidf in compiler_rt. This implementation performs rounding
6886 // correctly in all rounding modes with the exception of converting 0
6887 // when rounding toward negative infinity. In that case the fsub will produce
6888 // -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.
6889 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000ULL, dl, SrcVT);
6890 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
6891 BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000ULL), dl, DstVT);
6892 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000ULL, dl, SrcVT);
6893 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF)0x00000000FFFFFFFFULL, dl, SrcVT);
6894 SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
6895
6896 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
6897 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
6898 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
6899 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
6900 SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
6901 SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
6902 SDValue HiSub =
6903 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
6904 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
6905 return true;
6906}
6907
6908SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
6909 SelectionDAG &DAG) const {
6910 SDLoc dl(Node);
6911 unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
6912 ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
6913 EVT VT = Node->getValueType(0);
6914
6915 if (VT.isScalableVector())
6916 report_fatal_error(
6917 "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
6918
6919 if (isOperationLegalOrCustom(NewOp, VT)) {
6920 SDValue Quiet0 = Node->getOperand(0);
6921 SDValue Quiet1 = Node->getOperand(1);
6922
6923 if (!Node->getFlags().hasNoNaNs()) {
6924 // Insert canonicalizes if it's possible we need to quiet to get correct
6925 // sNaN behavior.
6926 if (!DAG.isKnownNeverSNaN(Quiet0)) {
6927 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
6928 Node->getFlags());
6929 }
6930 if (!DAG.isKnownNeverSNaN(Quiet1)) {
6931 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
6932 Node->getFlags());
6933 }
6934 }
6935
6936 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
6937 }
6938
6939 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
6940 // instead if there are no NaNs.
6941 if (Node->getFlags().hasNoNaNs()) {
6942 unsigned IEEE2018Op =
6943 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
6944 if (isOperationLegalOrCustom(IEEE2018Op, VT)) {
6945 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
6946 Node->getOperand(1), Node->getFlags());
6947 }
6948 }
6949
6950 // If none of the above worked, but there are no NaNs, then expand to
6951 // a compare/select sequence. This is required for correctness since
6952 // InstCombine might have canonicalized a fcmp+select sequence to a
6953 // FMINNUM/FMAXNUM node. If we were to fall through to the default
6954 // expansion to libcall, we might introduce a link-time dependency
6955 // on libm into a file that originally did not have one.
6956 if (Node->getFlags().hasNoNaNs()) {
6957 ISD::CondCode Pred =
6958 Node->getOpcode() == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
6959 SDValue Op1 = Node->getOperand(0);
6960 SDValue Op2 = Node->getOperand(1);
6961 SDValue SelCC = DAG.getSelectCC(dl, Op1, Op2, Op1, Op2, Pred);
6962 // Copy FMF flags, but always set the no-signed-zeros flag
6963 // as this is implied by the FMINNUM/FMAXNUM semantics.
6964 SDNodeFlags Flags = Node->getFlags();
6965 Flags.setNoSignedZeros(true);
6966 SelCC->setFlags(Flags);
6967 return SelCC;
6968 }
6969
6970 return SDValue();
6971}
6972
6973bool TargetLowering::expandCTPOP(SDNode *Node, SDValue &Result,
6974 SelectionDAG &DAG) const {
6975 SDLoc dl(Node);
6976 EVT VT = Node->getValueType(0);
6977 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
6978 SDValue Op = Node->getOperand(0);
6979 unsigned Len = VT.getScalarSizeInBits();
6980 assert(VT.isInteger() && "CTPOP not implemented for this type.")((void)0);
6981
6982 // TODO: Add support for irregular type lengths.
6983 if (!(Len <= 128 && Len % 8 == 0))
6984 return false;
6985
6986 // Only expand vector types if we have the appropriate vector bit operations.
6987 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) ||
6988 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6989 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6990 (Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) ||
6991 !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
6992 return false;
6993
6994 // This is the "best" algorithm from
6995 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
6996 SDValue Mask55 =
6997 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
6998 SDValue Mask33 =
6999 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
7000 SDValue Mask0F =
7001 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
7002 SDValue Mask01 =
7003 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
7004
7005 // v = v - ((v >> 1) & 0x55555555...)
7006 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
7007 DAG.getNode(ISD::AND, dl, VT,
7008 DAG.getNode(ISD::SRL, dl, VT, Op,
7009 DAG.getConstant(1, dl, ShVT)),
7010 Mask55));
7011 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
7012 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
7013 DAG.getNode(ISD::AND, dl, VT,
7014 DAG.getNode(ISD::SRL, dl, VT, Op,
7015 DAG.getConstant(2, dl, ShVT)),
7016 Mask33));
7017 // v = (v + (v >> 4)) & 0x0F0F0F0F...
7018 Op = DAG.getNode(ISD::AND, dl, VT,
7019 DAG.getNode(ISD::ADD, dl, VT, Op,
7020 DAG.getNode(ISD::SRL, dl, VT, Op,
7021 DAG.getConstant(4, dl, ShVT))),
7022 Mask0F);
7023 // v = (v * 0x01010101...) >> (Len - 8)
7024 if (Len > 8)
7025 Op =
7026 DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
7027 DAG.getConstant(Len - 8, dl, ShVT));
7028
7029 Result = Op;
7030 return true;
7031}
7032
7033bool TargetLowering::expandCTLZ(SDNode *Node, SDValue &Result,
7034 SelectionDAG &DAG) const {
7035 SDLoc dl(Node);
7036 EVT VT = Node->getValueType(0);
7037 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7038 SDValue Op = Node->getOperand(0);
7039 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
7040
7041 // If the non-ZERO_UNDEF version is supported we can use that instead.
7042 if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
7043 isOperationLegalOrCustom(ISD::CTLZ, VT)) {
7044 Result = DAG.getNode(ISD::CTLZ, dl, VT, Op);
7045 return true;
7046 }
7047
7048 // If the ZERO_UNDEF version is supported use that and handle the zero case.
7049 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
7050 EVT SetCCVT =
7051 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7052 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
7053 SDValue Zero = DAG.getConstant(0, dl, VT);
7054 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
7055 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
7056 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
7057 return true;
7058 }
7059
7060 // Only expand vector types if we have the appropriate vector bit operations.
7061 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
7062 !isOperationLegalOrCustom(ISD::CTPOP, VT) ||
7063 !isOperationLegalOrCustom(ISD::SRL, VT) ||
7064 !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
7065 return false;
7066
7067 // for now, we do this:
7068 // x = x | (x >> 1);
7069 // x = x | (x >> 2);
7070 // ...
7071 // x = x | (x >>16);
7072 // x = x | (x >>32); // for 64-bit input
7073 // return popcount(~x);
7074 //
7075 // Ref: "Hacker's Delight" by Henry Warren
7076 for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {
7077 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
7078 Op = DAG.getNode(ISD::OR, dl, VT, Op,
7079 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
7080 }
7081 Op = DAG.getNOT(dl, Op, VT);
7082 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
7083 return true;
7084}
7085
7086bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
7087 SelectionDAG &DAG) const {
7088 SDLoc dl(Node);
7089 EVT VT = Node->getValueType(0);
7090 SDValue Op = Node->getOperand(0);
7091 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
7092
7093 // If the non-ZERO_UNDEF version is supported we can use that instead.
7094 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
7095 isOperationLegalOrCustom(ISD::CTTZ, VT)) {
7096 Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);
7097 return true;
7098 }
7099
7100 // If the ZERO_UNDEF version is supported use that and handle the zero case.
7101 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
7102 EVT SetCCVT =
7103 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7104 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
7105 SDValue Zero = DAG.getConstant(0, dl, VT);
7106 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
7107 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
7108 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
7109 return true;
7110 }
7111
7112 // Only expand vector types if we have the appropriate vector bit operations.
7113 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
7114 (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
7115 !isOperationLegalOrCustom(ISD::CTLZ, VT)) ||
7116 !isOperationLegalOrCustom(ISD::SUB, VT) ||
7117 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
7118 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
7119 return false;
7120
7121 // for now, we use: { return popcount(~x & (x - 1)); }
7122 // unless the target has ctlz but not ctpop, in which case we use:
7123 // { return 32 - nlz(~x & (x-1)); }
7124 // Ref: "Hacker's Delight" by Henry Warren
7125 SDValue Tmp = DAG.getNode(
7126 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
7127 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
7128
7129 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
7130 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
7131 Result =
7132 DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
7133 DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
7134 return true;
7135 }
7136
7137 Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
7138 return true;
7139}
7140
7141bool TargetLowering::expandABS(SDNode *N, SDValue &Result,
7142 SelectionDAG &DAG, bool IsNegative) const {
7143 SDLoc dl(N);
7144 EVT VT = N->getValueType(0);
7145 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7146 SDValue Op = N->getOperand(0);
7147
7148 // abs(x) -> smax(x,sub(0,x))
7149 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
7150 isOperationLegal(ISD::SMAX, VT)) {
7151 SDValue Zero = DAG.getConstant(0, dl, VT);
7152 Result = DAG.getNode(ISD::SMAX, dl, VT, Op,
7153 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7154 return true;
7155 }
7156
7157 // abs(x) -> umin(x,sub(0,x))
7158 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
7159 isOperationLegal(ISD::UMIN, VT)) {
7160 SDValue Zero = DAG.getConstant(0, dl, VT);
7161 Result = DAG.getNode(ISD::UMIN, dl, VT, Op,
7162 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7163 return true;
7164 }
7165
7166 // 0 - abs(x) -> smin(x, sub(0,x))
7167 if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
7168 isOperationLegal(ISD::SMIN, VT)) {
7169 SDValue Zero = DAG.getConstant(0, dl, VT);
7170 Result = DAG.getNode(ISD::SMIN, dl, VT, Op,
7171 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7172 return true;
7173 }
7174
7175 // Only expand vector types if we have the appropriate vector operations.
7176 if (VT.isVector() &&
7177 (!isOperationLegalOrCustom(ISD::SRA, VT) ||
7178 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
7179 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
7180 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
7181 return false;
7182
7183 SDValue Shift =
7184 DAG.getNode(ISD::SRA, dl, VT, Op,
7185 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));
7186 if (!IsNegative) {
7187 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
7188 Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
7189 } else {
7190 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
7191 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);
7192 Result = DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
7193 }
7194 return true;
7195}
7196
7197SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
7198 SDLoc dl(N);
7199 EVT VT = N->getValueType(0);
7200 SDValue Op = N->getOperand(0);
7201
7202 if (!VT.isSimple())
7203 return SDValue();
7204
7205 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
7206 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
7207 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
7208 default:
7209 return SDValue();
7210 case MVT::i16:
7211 // Use a rotate by 8. This can be further expanded if necessary.
7212 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7213 case MVT::i32:
7214 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7215 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7216 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7217 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7218 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
7219 DAG.getConstant(0xFF0000, dl, VT));
7220 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
7221 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
7222 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
7223 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
7224 case MVT::i64:
7225 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
7226 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
7227 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7228 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7229 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7230 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7231 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
7232 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
7233 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
7234 DAG.getConstant(255ULL<<48, dl, VT));
7235 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
7236 DAG.getConstant(255ULL<<40, dl, VT));
7237 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
7238 DAG.getConstant(255ULL<<32, dl, VT));
7239 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
7240 DAG.getConstant(255ULL<<24, dl, VT));
7241 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
7242 DAG.getConstant(255ULL<<16, dl, VT));
7243 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
7244 DAG.getConstant(255ULL<<8 , dl, VT));
7245 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
7246 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
7247 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
7248 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
7249 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
7250 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
7251 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
7252 }
7253}
7254
7255SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
7256 SDLoc dl(N);
7257 EVT VT = N->getValueType(0);
7258 SDValue Op = N->getOperand(0);
7259 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
7260 unsigned Sz = VT.getScalarSizeInBits();
7261
7262 SDValue Tmp, Tmp2, Tmp3;
7263
7264 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
7265 // and finally the i1 pairs.
7266 // TODO: We can easily support i4/i2 legal types if any target ever does.
7267 if (Sz >= 8 && isPowerOf2_32(Sz)) {
7268 // Create the masks - repeating the pattern every byte.
7269 APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0));
7270 APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC));
7271 APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA));
7272 APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F));
7273 APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33));
7274 APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55));
7275
7276 // BSWAP if the type is wider than a single byte.
7277 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
7278
7279 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
7280 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
7281 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
7282 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT));
7283 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
7284 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7285
7286 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
7287 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
7288 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
7289 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT));
7290 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
7291 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7292
7293 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
7294 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
7295 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
7296 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT));
7297 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
7298 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7299 return Tmp;
7300 }
7301
7302 Tmp = DAG.getConstant(0, dl, VT);
7303 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
7304 if (I < J)
7305 Tmp2 =
7306 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
7307 else
7308 Tmp2 =
7309 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
7310
7311 APInt Shift(Sz, 1);
7312 Shift <<= J;
7313 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
7314 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
7315 }
7316
7317 return Tmp;
7318}
7319
7320std::pair<SDValue, SDValue>
7321TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
7322 SelectionDAG &DAG) const {
7323 SDLoc SL(LD);
7324 SDValue Chain = LD->getChain();
7325 SDValue BasePTR = LD->getBasePtr();
7326 EVT SrcVT = LD->getMemoryVT();
7327 EVT DstVT = LD->getValueType(0);
7328 ISD::LoadExtType ExtType = LD->getExtensionType();
7329
7330 if (SrcVT.isScalableVector())
7331 report_fatal_error("Cannot scalarize scalable vector loads");
7332
7333 unsigned NumElem = SrcVT.getVectorNumElements();
7334
7335 EVT SrcEltVT = SrcVT.getScalarType();
7336 EVT DstEltVT = DstVT.getScalarType();
7337
7338 // A vector must always be stored in memory as-is, i.e. without any padding
7339 // between the elements, since various code depend on it, e.g. in the
7340 // handling of a bitcast of a vector type to int, which may be done with a
7341 // vector store followed by an integer load. A vector that does not have
7342 // elements that are byte-sized must therefore be stored as an integer
7343 // built out of the extracted vector elements.
7344 if (!SrcEltVT.isByteSized()) {
7345 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
7346 EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);
7347
7348 unsigned NumSrcBits = SrcVT.getSizeInBits();
7349 EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);
7350
7351 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
7352 SDValue SrcEltBitMask = DAG.getConstant(
7353 APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);
7354
7355 // Load the whole vector and avoid masking off the top bits as it makes
7356 // the codegen worse.
7357 SDValue Load =
7358 DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,
7359 LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),
7360 LD->getMemOperand()->getFlags(), LD->getAAInfo());
7361
7362 SmallVector<SDValue, 8> Vals;
7363 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7364 unsigned ShiftIntoIdx =
7365 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
7366 SDValue ShiftAmount =
7367 DAG.getShiftAmountConstant(ShiftIntoIdx * SrcEltVT.getSizeInBits(),
7368 LoadVT, SL, /*LegalTypes=*/false);
7369 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
7370 SDValue Elt =
7371 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
7372 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);
7373
7374 if (ExtType != ISD::NON_EXTLOAD) {
7375 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
7376 Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);
7377 }
7378
7379 Vals.push_back(Scalar);
7380 }
7381
7382 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
7383 return std::make_pair(Value, Load.getValue(1));
7384 }
7385
7386 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
7387 assert(SrcEltVT.isByteSized())((void)0);
7388
7389 SmallVector<SDValue, 8> Vals;
7390 SmallVector<SDValue, 8> LoadChains;
7391
7392 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7393 SDValue ScalarLoad =
7394 DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
7395 LD->getPointerInfo().getWithOffset(Idx * Stride),
7396 SrcEltVT, LD->getOriginalAlign(),
7397 LD->getMemOperand()->getFlags(), LD->getAAInfo());
7398
7399 BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::Fixed(Stride));
7400
7401 Vals.push_back(ScalarLoad.getValue(0));
7402 LoadChains.push_back(ScalarLoad.getValue(1));
7403 }
7404
7405 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
7406 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
7407
7408 return std::make_pair(Value, NewChain);
7409}
7410
7411SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
7412 SelectionDAG &DAG) const {
7413 SDLoc SL(ST);
7414
7415 SDValue Chain = ST->getChain();
7416 SDValue BasePtr = ST->getBasePtr();
7417 SDValue Value = ST->getValue();
7418 EVT StVT = ST->getMemoryVT();
7419
7420 if (StVT.isScalableVector())
7421 report_fatal_error("Cannot scalarize scalable vector stores");
7422
7423 // The type of the data we want to save
7424 EVT RegVT = Value.getValueType();
7425 EVT RegSclVT = RegVT.getScalarType();
7426
7427 // The type of data as saved in memory.
7428 EVT MemSclVT = StVT.getScalarType();
7429
7430 unsigned NumElem = StVT.getVectorNumElements();
7431
7432 // A vector must always be stored in memory as-is, i.e. without any padding
7433 // between the elements, since various code depend on it, e.g. in the
7434 // handling of a bitcast of a vector type to int, which may be done with a
7435 // vector store followed by an integer load. A vector that does not have
7436 // elements that are byte-sized must therefore be stored as an integer
7437 // built out of the extracted vector elements.
7438 if (!MemSclVT.isByteSized()) {
7439 unsigned NumBits = StVT.getSizeInBits();
7440 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
7441
7442 SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
7443
7444 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7445 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
7446 DAG.getVectorIdxConstant(Idx, SL));
7447 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
7448 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
7449 unsigned ShiftIntoIdx =
7450 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
7451 SDValue ShiftAmount =
7452 DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
7453 SDValue ShiftedElt =
7454 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
7455 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
7456 }
7457
7458 return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
7459 ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
7460 ST->getAAInfo());
7461 }
7462
7463 // Store Stride in bytes
7464 unsigned Stride = MemSclVT.getSizeInBits() / 8;
7465 assert(Stride && "Zero stride!")((void)0);
7466 // Extract each of the elements from the original vector and save them into
7467 // memory individually.
7468 SmallVector<SDValue, 8> Stores;
7469 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7470 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
7471 DAG.getVectorIdxConstant(Idx, SL));
7472
7473 SDValue Ptr =
7474 DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Idx * Stride));
7475
7476 // This scalar TruncStore may be illegal, but we legalize it later.
7477 SDValue Store = DAG.getTruncStore(
7478 Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
7479 MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
7480 ST->getAAInfo());
7481
7482 Stores.push_back(Store);
7483 }
7484
7485 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
7486}
7487
7488std::pair<SDValue, SDValue>
7489TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
7490 assert(LD->getAddressingMode() == ISD::UNINDEXED &&((void)0)
7491 "unaligned indexed loads not implemented!")((void)0);
7492 SDValue Chain = LD->getChain();
7493 SDValue Ptr = LD->getBasePtr();
7494 EVT VT = LD->getValueType(0);
7495 EVT LoadedVT = LD->getMemoryVT();
7496 SDLoc dl(LD);
7497 auto &MF = DAG.getMachineFunction();
7498
7499 if (VT.isFloatingPoint() || VT.isVector()) {
7500 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
7501 if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
7502 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
7503 LoadedVT.isVector()) {
7504 // Scalarize the load and let the individual components be handled.
7505 return scalarizeVectorLoad(LD, DAG);
7506 }
7507
7508 // Expand to a (misaligned) integer load of the same size,
7509 // then bitconvert to floating point or vector.
7510 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
7511 LD->getMemOperand());
7512 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
7513 if (LoadedVT != VT)
7514 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
7515 ISD::ANY_EXTEND, dl, VT, Result);
7516
7517 return std::make_pair(Result, newLoad.getValue(1));
7518 }
7519
7520 // Copy the value to a (aligned) stack slot using (unaligned) integer
7521 // loads and stores, then do a (aligned) load from the stack slot.
7522 MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
7523 unsigned LoadedBytes = LoadedVT.getStoreSize();
7524 unsigned RegBytes = RegVT.getSizeInBits() / 8;
7525 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
7526
7527 // Make sure the stack slot is also aligned for the register type.
7528 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
7529 auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
7530 SmallVector<SDValue, 8> Stores;
7531 SDValue StackPtr = StackBase;
7532 unsigned Offset = 0;
7533
7534 EVT PtrVT = Ptr.getValueType();
7535 EVT StackPtrVT = StackPtr.getValueType();
7536
7537 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
7538 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
7539
7540 // Do all but one copies using the full register width.
7541 for (unsigned i = 1; i < NumRegs; i++) {
7542 // Load one integer register's worth from the original location.
7543 SDValue Load = DAG.getLoad(
7544 RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
7545 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
7546 LD->getAAInfo());
7547 // Follow the load with a store to the stack slot. Remember the store.
7548 Stores.push_back(DAG.getStore(
7549 Load.getValue(1), dl, Load, StackPtr,
7550 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));
7551 // Increment the pointers.
7552 Offset += RegBytes;
7553
7554 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
7555 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
7556 }
7557
7558 // The last copy may be partial. Do an extending load.
7559 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
7560 8 * (LoadedBytes - Offset));
7561 SDValue Load =
7562 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
7563 LD->getPointerInfo().getWithOffset(Offset), MemVT,
7564 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
7565 LD->getAAInfo());
7566 // Follow the load with a store to the stack slot. Remember the store.
7567 // On big-endian machines this requires a truncating store to ensure
7568 // that the bits end up in the right place.
7569 Stores.push_back(DAG.getTruncStore(
7570 Load.getValue(1), dl, Load, StackPtr,
7571 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));
7572
7573 // The order of the stores doesn't matter - say it with a TokenFactor.
7574 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7575
7576 // Finally, perform the original load only redirected to the stack slot.
7577 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
7578 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),
7579 LoadedVT);
7580
7581 // Callers expect a MERGE_VALUES node.
7582 return std::make_pair(Load, TF);
7583 }
7584
7585 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&((void)0)
7586 "Unaligned load of unsupported type.")((void)0);
7587
7588 // Compute the new VT that is half the size of the old one. This is an
7589 // integer MVT.
7590 unsigned NumBits = LoadedVT.getSizeInBits();
7591 EVT NewLoadedVT;
7592 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
7593 NumBits >>= 1;
7594
7595 Align Alignment = LD->getOriginalAlign();
7596 unsigned IncrementSize = NumBits / 8;
7597 ISD::LoadExtType HiExtType = LD->getExtensionType();
7598
7599 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
7600 if (HiExtType == ISD::NON_EXTLOAD)
7601 HiExtType = ISD::ZEXTLOAD;
7602
7603 // Load the value in two parts
7604 SDValue Lo, Hi;
7605 if (DAG.getDataLayout().isLittleEndian()) {
7606 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
7607 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7608 LD->getAAInfo());
7609
7610 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7611 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
7612 LD->getPointerInfo().getWithOffset(IncrementSize),
7613 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7614 LD->getAAInfo());
7615 } else {
7616 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
7617 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7618 LD->getAAInfo());
7619
7620 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7621 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
7622 LD->getPointerInfo().getWithOffset(IncrementSize),
7623 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7624 LD->getAAInfo());
7625 }
7626
7627 // aggregate the two parts
7628 SDValue ShiftAmount =
7629 DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),
7630 DAG.getDataLayout()));
7631 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
7632 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
7633
7634 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
7635 Hi.getValue(1));
7636
7637 return std::make_pair(Result, TF);
7638}
7639
7640SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
7641 SelectionDAG &DAG) const {
7642 assert(ST->getAddressingMode() == ISD::UNINDEXED &&((void)0)
7643 "unaligned indexed stores not implemented!")((void)0);
7644 SDValue Chain = ST->getChain();
7645 SDValue Ptr = ST->getBasePtr();
7646 SDValue Val = ST->getValue();
7647 EVT VT = Val.getValueType();
7648 Align Alignment = ST->getOriginalAlign();
7649 auto &MF = DAG.getMachineFunction();
7650 EVT StoreMemVT = ST->getMemoryVT();
7651
7652 SDLoc dl(ST);
7653 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
7654 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
7655 if (isTypeLegal(intVT)) {
7656 if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
7657 StoreMemVT.isVector()) {
7658 // Scalarize the store and let the individual components be handled.
7659 SDValue Result = scalarizeVectorStore(ST, DAG);
7660 return Result;
7661 }
7662 // Expand to a bitconvert of the value to the integer type of the
7663 // same size, then a (misaligned) int store.
7664 // FIXME: Does not handle truncating floating point stores!
7665 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
7666 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
7667 Alignment, ST->getMemOperand()->getFlags());
7668 return Result;
7669 }
7670 // Do a (aligned) store to a stack slot, then copy from the stack slot
7671 // to the final destination using (unaligned) integer loads and stores.
7672 MVT RegVT = getRegisterType(
7673 *DAG.getContext(),
7674 EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));
7675 EVT PtrVT = Ptr.getValueType();
7676 unsigned StoredBytes = StoreMemVT.getStoreSize();
7677 unsigned RegBytes = RegVT.getSizeInBits() / 8;
7678 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
7679
7680 // Make sure the stack slot is also aligned for the register type.
7681 SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
7682 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
7683
7684 // Perform the original store, only redirected to the stack slot.
7685 SDValue Store = DAG.getTruncStore(
7686 Chain, dl, Val, StackPtr,
7687 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);
7688
7689 EVT StackPtrVT = StackPtr.getValueType();
7690
7691 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
7692 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
7693 SmallVector<SDValue, 8> Stores;
7694 unsigned Offset = 0;
7695
7696 // Do all but one copies using the full register width.
7697 for (unsigned i = 1; i < NumRegs; i++) {
7698 // Load one integer register's worth from the stack slot.
7699 SDValue Load = DAG.getLoad(
7700 RegVT, dl, Store, StackPtr,
7701 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));
7702 // Store it to the final location. Remember the store.
7703 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
7704 ST->getPointerInfo().getWithOffset(Offset),
7705 ST->getOriginalAlign(),
7706 ST->getMemOperand()->getFlags()));
7707 // Increment the pointers.
7708 Offset += RegBytes;
7709 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
7710 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
7711 }
7712
7713 // The last store may be partial. Do a truncating store. On big-endian
7714 // machines this requires an extending load from the stack slot to ensure
7715 // that the bits are in the right place.
7716 EVT LoadMemVT =
7717 EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));
7718
7719 // Load from the stack slot.
7720 SDValue Load = DAG.getExtLoad(
7721 ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
7722 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);
7723
7724 Stores.push_back(
7725 DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
7726 ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,
7727 ST->getOriginalAlign(),
7728 ST->getMemOperand()->getFlags(), ST->getAAInfo()));
7729 // The order of the stores doesn't matter - say it with a TokenFactor.
7730 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7731 return Result;
7732 }
7733
7734 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&((void)0)
7735 "Unaligned store of unknown type.")((void)0);
7736 // Get the half-size VT
7737 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());
7738 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
7739 unsigned IncrementSize = NumBits / 8;
7740
7741 // Divide the stored value in two parts.
7742 SDValue ShiftAmount = DAG.getConstant(
7743 NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout()));
7744 SDValue Lo = Val;
7745 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
7746
7747 // Store the two parts
7748 SDValue Store1, Store2;
7749 Store1 = DAG.getTruncStore(Chain, dl,
7750 DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
7751 Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
7752 ST->getMemOperand()->getFlags());
7753
7754 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7755 Store2 = DAG.getTruncStore(
7756 Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
7757 ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
7758 ST->getMemOperand()->getFlags(), ST->getAAInfo());
7759
7760 SDValue Result =
7761 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
7762 return Result;
7763}
7764
7765SDValue
7766TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
7767 const SDLoc &DL, EVT DataVT,
7768 SelectionDAG &DAG,
7769 bool IsCompressedMemory) const {
7770 SDValue Increment;
7771 EVT AddrVT = Addr.getValueType();
7772 EVT MaskVT = Mask.getValueType();
7773 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&((void)0)
7774 "Incompatible types of Data and Mask")((void)0);
7775 if (IsCompressedMemory) {
7776 if (DataVT.isScalableVector())
7777 report_fatal_error(
7778 "Cannot currently handle compressed memory with scalable vectors");
7779 // Incrementing the pointer according to number of '1's in the mask.
7780 EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
7781 SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
7782 if (MaskIntVT.getSizeInBits() < 32) {
7783 MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
7784 MaskIntVT = MVT::i32;
7785 }
7786
7787 // Count '1's with POPCNT.
7788 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
7789 Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
7790 // Scale is an element size in bytes.
7791 SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
7792 AddrVT);
7793 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
7794 } else if (DataVT.isScalableVector()) {
7795 Increment = DAG.getVScale(DL, AddrVT,
7796 APInt(AddrVT.getFixedSizeInBits(),
7797 DataVT.getStoreSize().getKnownMinSize()));
7798 } else
7799 Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);
7800
7801 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
7802}
7803
7804static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
7805 EVT VecVT, const SDLoc &dl,
7806 unsigned NumSubElts) {
7807 if (!VecVT.isScalableVector() && isa<ConstantSDNode>(Idx))
7808 return Idx;
7809
7810 EVT IdxVT = Idx.getValueType();
7811 unsigned NElts = VecVT.getVectorMinNumElements();
7812 if (VecVT.isScalableVector()) {
7813 // If this is a constant index and we know the value plus the number of the
7814 // elements in the subvector minus one is less than the minimum number of
7815 // elements then it's safe to return Idx.
7816 if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))
7817 if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
7818 return Idx;
7819 SDValue VS =
7820 DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));
7821 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
7822 SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS,
7823 DAG.getConstant(NumSubElts, dl, IdxVT));
7824 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);
7825 }
7826 if (isPowerOf2_32(NElts) && NumSubElts == 1) {
7827 APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));
7828 return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
7829 DAG.getConstant(Imm, dl, IdxVT));
7830 }
7831 unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
7832 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
7833 DAG.getConstant(MaxIndex, dl, IdxVT));
7834}
7835
7836SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,
7837 SDValue VecPtr, EVT VecVT,
7838 SDValue Index) const {
7839 return getVectorSubVecPointer(
7840 DAG, VecPtr, VecVT,
7841 EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(), 1),
7842 Index);
7843}
7844
7845SDValue TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG,
7846 SDValue VecPtr, EVT VecVT,
7847 EVT SubVecVT,
7848 SDValue Index) const {
7849 SDLoc dl(Index);
7850 // Make sure the index type is big enough to compute in.
7851 Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());
7852
7853 EVT EltVT = VecVT.getVectorElementType();
7854
7855 // Calculate the element offset and add it to the pointer.
7856 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
7857 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&((void)0)
7858 "Converting bits to bytes lost precision")((void)0);
7859
7860 // Scalable vectors don't need clamping as these are checked at compile time
7861 if (SubVecVT.isFixedLengthVector()) {
7862 assert(SubVecVT.getVectorElementType() == EltVT &&((void)0)
7863 "Sub-vector must be a fixed vector with matching element type")((void)0);
7864 Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,
7865 SubVecVT.getVectorNumElements());
7866 }
7867
7868 EVT IdxVT = Index.getValueType();
7869
7870 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
7871 DAG.getConstant(EltSize, dl, IdxVT));
7872 return DAG.getMemBasePlusOffset(VecPtr, Index, dl);
7873}
7874
7875//===----------------------------------------------------------------------===//
7876// Implementation of Emulated TLS Model
7877//===----------------------------------------------------------------------===//
7878
7879SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
7880 SelectionDAG &DAG) const {
7881 // Access to address of TLS varialbe xyz is lowered to a function call:
7882 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
7883 EVT PtrVT = getPointerTy(DAG.getDataLayout());
7884 PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
7885 SDLoc dl(GA);
7886
7887 ArgListTy Args;
7888 ArgListEntry Entry;
7889 std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
7890 Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
7891 StringRef EmuTlsVarName(NameString);
7892 GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
7893 assert(EmuTlsVar && "Cannot find EmuTlsVar ")((void)0);
7894 Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
7895 Entry.Ty = VoidPtrType;
7896 Args.push_back(Entry);
7897
7898 SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
7899
7900 TargetLowering::CallLoweringInfo CLI(DAG);
7901 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
7902 CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
7903 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
7904
7905 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
7906 // At last for X86 targets, maybe good for other targets too?
7907 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7908 MFI.setAdjustsStack(true); // Is this only for X86 target?
7909 MFI.setHasCalls(true);
7910
7911 assert((GA->getOffset() == 0) &&((void)0)
7912 "Emulated TLS must have zero offset in GlobalAddressSDNode")((void)0);
7913 return CallResult.first;
7914}
7915
7916SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
7917 SelectionDAG &DAG) const {
7918 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.")((void)0);
7919 if (!isCtlzFast())
7920 return SDValue();
7921 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
7922 SDLoc dl(Op);
7923 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
7924 if (C->isNullValue() && CC == ISD::SETEQ) {
7925 EVT VT = Op.getOperand(0).getValueType();
7926 SDValue Zext = Op.getOperand(0);
7927 if (VT.bitsLT(MVT::i32)) {
7928 VT = MVT::i32;
7929 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
7930 }
7931 unsigned Log2b = Log2_32(VT.getSizeInBits());
7932 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
7933 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
7934 DAG.getConstant(Log2b, dl, MVT::i32));
7935 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
7936 }
7937 }
7938 return SDValue();
7939}
7940
7941// Convert redundant addressing modes (e.g. scaling is redundant
7942// when accessing bytes).
7943ISD::MemIndexType
7944TargetLowering::getCanonicalIndexType(ISD::MemIndexType IndexType, EVT MemVT,
7945 SDValue Offsets) const {
7946 bool IsScaledIndex =
7947 (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::UNSIGNED_SCALED);
7948 bool IsSignedIndex =
7949 (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::SIGNED_UNSCALED);
7950
7951 // Scaling is unimportant for bytes, canonicalize to unscaled.
7952 if (IsScaledIndex && MemVT.getScalarType() == MVT::i8) {
7953 IsScaledIndex = false;
7954 IndexType = IsSignedIndex ? ISD::SIGNED_UNSCALED : ISD::UNSIGNED_UNSCALED;
7955 }
7956
7957 return IndexType;
7958}
7959
7960SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
7961 SDValue Op0 = Node->getOperand(0);
7962 SDValue Op1 = Node->getOperand(1);
7963 EVT VT = Op0.getValueType();
7964 unsigned Opcode = Node->getOpcode();
7965 SDLoc DL(Node);
7966
7967 // umin(x,y) -> sub(x,usubsat(x,y))
7968 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&
7969 isOperationLegal(ISD::USUBSAT, VT)) {
7970 return DAG.getNode(ISD::SUB, DL, VT, Op0,
7971 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));
7972 }
7973
7974 // umax(x,y) -> add(x,usubsat(y,x))
7975 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&
7976 isOperationLegal(ISD::USUBSAT, VT)) {
7977 return DAG.getNode(ISD::ADD, DL, VT, Op0,
7978 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));
7979 }
7980
7981 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
7982 ISD::CondCode CC;
7983 switch (Opcode) {
7984 default: llvm_unreachable("How did we get here?")__builtin_unreachable();
7985 case ISD::SMAX: CC = ISD::SETGT; break;
7986 case ISD::SMIN: CC = ISD::SETLT; break;
7987 case ISD::UMAX: CC = ISD::SETUGT; break;
7988 case ISD::UMIN: CC = ISD::SETULT; break;
7989 }
7990
7991 // FIXME: Should really try to split the vector in case it's legal on a
7992 // subvector.
7993 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
7994 return DAG.UnrollVectorOp(Node);
7995
7996 SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
7997 return DAG.getSelect(DL, VT, Cond, Op0, Op1);
7998}
7999
8000SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
8001 unsigned Opcode = Node->getOpcode();
8002 SDValue LHS = Node->getOperand(0);
8003 SDValue RHS = Node->getOperand(1);
8004 EVT VT = LHS.getValueType();
8005 SDLoc dl(Node);
8006
8007 assert(VT == RHS.getValueType() && "Expected operands to be the same type")((void)0);
8008 assert(VT.isInteger() && "Expected operands to be integers")((void)0);
8009
8010 // usub.sat(a, b) -> umax(a, b) - b
8011 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
8012 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
8013 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
8014 }
8015
8016 // uadd.sat(a, b) -> umin(a, ~b) + b
8017 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
8018 SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
8019 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
8020 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
8021 }
8022
8023 unsigned OverflowOp;
8024 switch (Opcode) {
8025 case ISD::SADDSAT:
8026 OverflowOp = ISD::SADDO;
8027 break;
8028 case ISD::UADDSAT:
8029 OverflowOp = ISD::UADDO;
8030 break;
8031 case ISD::SSUBSAT:
8032 OverflowOp = ISD::SSUBO;
8033 break;
8034 case ISD::USUBSAT:
8035 OverflowOp = ISD::USUBO;
8036 break;
8037 default:
8038 llvm_unreachable("Expected method to receive signed or unsigned saturation "__builtin_unreachable()
8039 "addition or subtraction node.")__builtin_unreachable();
8040 }
8041
8042 // FIXME: Should really try to split the vector in case it's legal on a
8043 // subvector.
8044 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
8045 return DAG.UnrollVectorOp(Node);
8046
8047 unsigned BitWidth = LHS.getScalarValueSizeInBits();
8048 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8049 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8050 SDValue SumDiff = Result.getValue(0);
8051 SDValue Overflow = Result.getValue(1);
8052 SDValue Zero = DAG.getConstant(0, dl, VT);
8053 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
8054
8055 if (Opcode == ISD::UADDSAT) {
8056 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
8057 // (LHS + RHS) | OverflowMask
8058 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
8059 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
8060 }
8061 // Overflow ? 0xffff.... : (LHS + RHS)
8062 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);
8063 }
8064
8065 if (Opcode == ISD::USUBSAT) {
8066 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
8067 // (LHS - RHS) & ~OverflowMask
8068 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
8069 SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
8070 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
8071 }
8072 // Overflow ? 0 : (LHS - RHS)
8073 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);
8074 }
8075
8076 // SatMax -> Overflow && SumDiff < 0
8077 // SatMin -> Overflow && SumDiff >= 0
8078 APInt MinVal = APInt::getSignedMinValue(BitWidth);
8079 APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
8080 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
8081 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8082 SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT);
8083 Result = DAG.getSelect(dl, VT, SumNeg, SatMax, SatMin);
8084 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);
8085}
8086
8087SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
8088 unsigned Opcode = Node->getOpcode();
8089 bool IsSigned = Opcode == ISD::SSHLSAT;
8090 SDValue LHS = Node->getOperand(0);
8091 SDValue RHS = Node->getOperand(1);
8092 EVT VT = LHS.getValueType();
8093 SDLoc dl(Node);
8094
8095 assert((Node->getOpcode() == ISD::SSHLSAT ||((void)0)
8096 Node->getOpcode() == ISD::USHLSAT) &&((void)0)
8097 "Expected a SHLSAT opcode")((void)0);
8098 assert(VT == RHS.getValueType() && "Expected operands to be the same type")((void)0);
8099 assert(VT.isInteger() && "Expected operands to be integers")((void)0);
8100
8101 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
8102
8103 unsigned BW = VT.getScalarSizeInBits();
8104 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);
8105 SDValue Orig =
8106 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);
8107
8108 SDValue SatVal;
8109 if (IsSigned) {
8110 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);
8111 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);
8112 SatVal = DAG.getSelectCC(dl, LHS, DAG.getConstant(0, dl, VT),
8113 SatMin, SatMax, ISD::SETLT);
8114 } else {
8115 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);
8116 }
8117 Result = DAG.getSelectCC(dl, LHS, Orig, SatVal, Result, ISD::SETNE);
8118
8119 return Result;
8120}
8121
8122SDValue
8123TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
8124 assert((Node->getOpcode() == ISD::SMULFIX ||((void)0)
8125 Node->getOpcode() == ISD::UMULFIX ||((void)0)
8126 Node->getOpcode() == ISD::SMULFIXSAT ||((void)0)
8127 Node->getOpcode() == ISD::UMULFIXSAT) &&((void)0)
8128 "Expected a fixed point multiplication opcode")((void)0);
8129
8130 SDLoc dl(Node);
8131 SDValue LHS = Node->getOperand(0);
8132 SDValue RHS = Node->getOperand(1);
8133 EVT VT = LHS.getValueType();
8134 unsigned Scale = Node->getConstantOperandVal(2);
8135 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
8136 Node->getOpcode() == ISD::UMULFIXSAT);
8137 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
8138 Node->getOpcode() == ISD::SMULFIXSAT);
8139 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8140 unsigned VTSize = VT.getScalarSizeInBits();
8141
8142 if (!Scale) {
8143 // [us]mul.fix(a, b, 0) -> mul(a, b)
8144 if (!Saturating) {
8145 if (isOperationLegalOrCustom(ISD::MUL, VT))
8146 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8147 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
8148 SDValue Result =
8149 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8150 SDValue Product = Result.getValue(0);
8151 SDValue Overflow = Result.getValue(1);
8152 SDValue Zero = DAG.getConstant(0, dl, VT);
8153
8154 APInt MinVal = APInt::getSignedMinValue(VTSize);
8155 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
8156 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
8157 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8158 // Xor the inputs, if resulting sign bit is 0 the product will be
8159 // positive, else negative.
8160 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
8161 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
8162 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
8163 return DAG.getSelect(dl, VT, Overflow, Result, Product);
8164 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
8165 SDValue Result =
8166 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8167 SDValue Product = Result.getValue(0);
8168 SDValue Overflow = Result.getValue(1);
8169
8170 APInt MaxVal = APInt::getMaxValue(VTSize);
8171 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8172 return DAG.getSelect(dl, VT, Overflow, SatMax, Product);
8173 }
8174 }
8175
8176 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&((void)0)
8177 "Expected scale to be less than the number of bits if signed or at "((void)0)
8178 "most the number of bits if unsigned.")((void)0);
8179 assert(LHS.getValueType() == RHS.getValueType() &&((void)0)
8180 "Expected both operands to be the same type")((void)0);
8181
8182 // Get the upper and lower bits of the result.
8183 SDValue Lo, Hi;
8184 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
8185 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
8186 if (isOperationLegalOrCustom(LoHiOp, VT)) {
8187 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
8188 Lo = Result.getValue(0);
8189 Hi = Result.getValue(1);
8190 } else if (isOperationLegalOrCustom(HiOp, VT)) {
8191 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8192 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
8193 } else if (VT.isVector()) {
8194 return SDValue();
8195 } else {
8196 report_fatal_error("Unable to expand fixed point multiplication.");
8197 }
8198
8199 if (Scale == VTSize)
8200 // Result is just the top half since we'd be shifting by the width of the
8201 // operand. Overflow impossible so this works for both UMULFIX and
8202 // UMULFIXSAT.
8203 return Hi;
8204
8205 // The result will need to be shifted right by the scale since both operands
8206 // are scaled. The result is given to us in 2 halves, so we only want part of
8207 // both in the result.
8208 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
8209 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
8210 DAG.getConstant(Scale, dl, ShiftTy));
8211 if (!Saturating)
8212 return Result;
8213
8214 if (!Signed) {
8215 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
8216 // widened multiplication) aren't all zeroes.
8217
8218 // Saturate to max if ((Hi >> Scale) != 0),
8219 // which is the same as if (Hi > ((1 << Scale) - 1))
8220 APInt MaxVal = APInt::getMaxValue(VTSize);
8221 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
8222 dl, VT);
8223 Result = DAG.getSelectCC(dl, Hi, LowMask,
8224 DAG.getConstant(MaxVal, dl, VT), Result,
8225 ISD::SETUGT);
8226
8227 return Result;
8228 }
8229
8230 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
8231 // widened multiplication) aren't all ones or all zeroes.
8232
8233 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
8234 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);
8235
8236 if (Scale == 0) {
8237 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
8238 DAG.getConstant(VTSize - 1, dl, ShiftTy));
8239 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
8240 // Saturated to SatMin if wide product is negative, and SatMax if wide
8241 // product is positive ...
8242 SDValue Zero = DAG.getConstant(0, dl, VT);
8243 SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
8244 ISD::SETLT);
8245 // ... but only if we overflowed.
8246 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
8247 }
8248
8249 // We handled Scale==0 above so all the bits to examine is in Hi.
8250
8251 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
8252 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
8253 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
8254 dl, VT);
8255 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
8256 // Saturate to min if (Hi >> (Scale - 1)) < -1),
8257 // which is the same as if (HI < (-1 << (Scale - 1))
8258 SDValue HighMask =
8259 DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),
8260 dl, VT);
8261 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
8262 return Result;
8263}
8264
8265SDValue
8266TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
8267 SDValue LHS, SDValue RHS,
8268 unsigned Scale, SelectionDAG &DAG) const {
8269 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||((void)0)
8270 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&((void)0)
8271 "Expected a fixed point division opcode")((void)0);
8272
8273 EVT VT = LHS.getValueType();
8274 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
8275 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
8276 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8277
8278 // If there is enough room in the type to upscale the LHS or downscale the
8279 // RHS before the division, we can perform it in this type without having to
8280 // resize. For signed operations, the LHS headroom is the number of
8281 // redundant sign bits, and for unsigned ones it is the number of zeroes.
8282 // The headroom for the RHS is the number of trailing zeroes.
8283 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1
8284 : DAG.computeKnownBits(LHS).countMinLeadingZeros();
8285 unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();
8286
8287 // For signed saturating operations, we need to be able to detect true integer
8288 // division overflow; that is, when you have MIN / -EPS. However, this
8289 // is undefined behavior and if we emit divisions that could take such
8290 // values it may cause undesired behavior (arithmetic exceptions on x86, for
8291 // example).
8292 // Avoid this by requiring an extra bit so that we never get this case.
8293 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
8294 // signed saturating division, we need to emit a whopping 32-bit division.
8295 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
8296 return SDValue();
8297
8298 unsigned LHSShift = std::min(LHSLead, Scale);
8299 unsigned RHSShift = Scale - LHSShift;
8300
8301 // At this point, we know that if we shift the LHS up by LHSShift and the
8302 // RHS down by RHSShift, we can emit a regular division with a final scaling
8303 // factor of Scale.
8304
8305 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
8306 if (LHSShift)
8307 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
8308 DAG.getConstant(LHSShift, dl, ShiftTy));
8309 if (RHSShift)
8310 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
8311 DAG.getConstant(RHSShift, dl, ShiftTy));
8312
8313 SDValue Quot;
8314 if (Signed) {
8315 // For signed operations, if the resulting quotient is negative and the
8316 // remainder is nonzero, subtract 1 from the quotient to round towards
8317 // negative infinity.
8318 SDValue Rem;
8319 // FIXME: Ideally we would always produce an SDIVREM here, but if the
8320 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
8321 // we couldn't just form a libcall, but the type legalizer doesn't do it.
8322 if (isTypeLegal(VT) &&
8323 isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
8324 Quot = DAG.getNode(ISD::SDIVREM, dl,
8325 DAG.getVTList(VT, VT),
8326 LHS, RHS);
8327 Rem = Quot.getValue(1);
8328 Quot = Quot.getValue(0);
8329 } else {
8330 Quot = DAG.getNode(ISD::SDIV, dl, VT,
8331 LHS, RHS);
8332 Rem = DAG.getNode(ISD::SREM, dl, VT,
8333 LHS, RHS);
8334 }
8335 SDValue Zero = DAG.getConstant(0, dl, VT);
8336 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
8337 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
8338 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
8339 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
8340 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
8341 DAG.getConstant(1, dl, VT));
8342 Quot = DAG.getSelect(dl, VT,
8343 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
8344 Sub1, Quot);
8345 } else
8346 Quot = DAG.getNode(ISD::UDIV, dl, VT,
8347 LHS, RHS);
8348
8349 return Quot;
8350}
8351
8352void TargetLowering::expandUADDSUBO(
8353 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
8354 SDLoc dl(Node);
8355 SDValue LHS = Node->getOperand(0);
8356 SDValue RHS = Node->getOperand(1);
8357 bool IsAdd = Node->getOpcode() == ISD::UADDO;
8358
8359 // If ADD/SUBCARRY is legal, use that instead.
8360 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
8361 if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
8362 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
8363 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
8364 { LHS, RHS, CarryIn });
8365 Result = SDValue(NodeCarry.getNode(), 0);
8366 Overflow = SDValue(NodeCarry.getNode(), 1);
8367 return;
8368 }
8369
8370 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
8371 LHS.getValueType(), LHS, RHS);
8372
8373 EVT ResultType = Node->getValueType(1);
8374 EVT SetCCType = getSetCCResultType(
8375 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
8376 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
8377 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
8378 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
8379}
8380
8381void TargetLowering::expandSADDSUBO(
8382 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
8383 SDLoc dl(Node);
8384 SDValue LHS = Node->getOperand(0);
8385 SDValue RHS = Node->getOperand(1);
8386 bool IsAdd = Node->getOpcode() == ISD::SADDO;
8387
8388 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
8389 LHS.getValueType(), LHS, RHS);
8390
8391 EVT ResultType = Node->getValueType(1);
8392 EVT OType = getSetCCResultType(
8393 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
8394
8395 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
8396 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
8397 if (isOperationLegalOrCustom(OpcSat, LHS.getValueType())) {
8398 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
8399 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
8400 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
8401 return;
8402 }
8403
8404 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
8405
8406 // For an addition, the result should be less than one of the operands (LHS)
8407 // if and only if the other operand (RHS) is negative, otherwise there will
8408 // be overflow.
8409 // For a subtraction, the result should be less than one of the operands
8410 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
8411 // otherwise there will be overflow.
8412 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
8413 SDValue ConditionRHS =
8414 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);
8415
8416 Overflow = DAG.getBoolExtOrTrunc(
8417 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
8418 ResultType, ResultType);
8419}
8420
8421bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
8422 SDValue &Overflow, SelectionDAG &DAG) const {
8423 SDLoc dl(Node);
8424 EVT VT = Node->getValueType(0);
8425 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8426 SDValue LHS = Node->getOperand(0);
8427 SDValue RHS = Node->getOperand(1);
8428 bool isSigned = Node->getOpcode() == ISD::SMULO;
8429
8430 // For power-of-two multiplications we can use a simpler shift expansion.
8431 if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
8432 const APInt &C = RHSC->getAPIntValue();
8433 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
8434 if (C.isPowerOf2()) {
8435 // smulo(x, signed_min) is same as umulo(x, signed_min).
8436 bool UseArithShift = isSigned && !C.isMinSignedValue();
8437 EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout());
8438 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);
8439 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
8440 Overflow = DAG.getSetCC(dl, SetCCVT,
8441 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
8442 dl, VT, Result, ShiftAmt),
8443 LHS, ISD::SETNE);
8444 return true;
8445 }
8446 }
8447
8448 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
8449 if (VT.isVector())
8450 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
8451 VT.getVectorNumElements());
8452
8453 SDValue BottomHalf;
8454 SDValue TopHalf;
8455 static const unsigned Ops[2][3] =
8456 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
8457 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
8458 if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
8459 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8460 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
8461 } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
8462 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
8463 RHS);
8464 TopHalf = BottomHalf.getValue(1);
8465 } else if (isTypeLegal(WideVT)) {
8466 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
8467 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
8468 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
8469 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
8470 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,
8471 getShiftAmountTy(WideVT, DAG.getDataLayout()));
8472 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
8473 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
8474 } else {
8475 if (VT.isVector())
8476 return false;
8477
8478 // We can fall back to a libcall with an illegal type for the MUL if we
8479 // have a libcall big enough.
8480 // Also, we can fall back to a division in some cases, but that's a big
8481 // performance hit in the general case.
8482 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
8483 if (WideVT == MVT::i16)
8484 LC = RTLIB::MUL_I16;
8485 else if (WideVT == MVT::i32)
8486 LC = RTLIB::MUL_I32;
8487 else if (WideVT == MVT::i64)
8488 LC = RTLIB::MUL_I64;
8489 else if (WideVT == MVT::i128)
8490 LC = RTLIB::MUL_I128;
8491 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")((void)0);
8492
8493 SDValue HiLHS;
8494 SDValue HiRHS;
8495 if (isSigned) {
8496 // The high part is obtained by SRA'ing all but one of the bits of low
8497 // part.
8498 unsigned LoSize = VT.getFixedSizeInBits();
8499 HiLHS =
8500 DAG.getNode(ISD::SRA, dl, VT, LHS,
8501 DAG.getConstant(LoSize - 1, dl,
8502 getPointerTy(DAG.getDataLayout())));
8503 HiRHS =
8504 DAG.getNode(ISD::SRA, dl, VT, RHS,
8505 DAG.getConstant(LoSize - 1, dl,
8506 getPointerTy(DAG.getDataLayout())));
8507 } else {
8508 HiLHS = DAG.getConstant(0, dl, VT);
8509 HiRHS = DAG.getConstant(0, dl, VT);
8510 }
8511
8512 // Here we're passing the 2 arguments explicitly as 4 arguments that are
8513 // pre-lowered to the correct types. This all depends upon WideVT not
8514 // being a legal type for the architecture and thus has to be split to
8515 // two arguments.
8516 SDValue Ret;
8517 TargetLowering::MakeLibCallOptions CallOptions;
8518 CallOptions.setSExt(isSigned);
8519 CallOptions.setIsPostTypeLegalization(true);
8520 if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
8521 // Halves of WideVT are packed into registers in different order
8522 // depending on platform endianness. This is usually handled by
8523 // the C calling convention, but we can't defer to it in
8524 // the legalizer.
8525 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
8526 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
8527 } else {
8528 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
8529 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
8530 }
8531 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&((void)0)
8532 "Ret value is a collection of constituent nodes holding result.")((void)0);
8533 if (DAG.getDataLayout().isLittleEndian()) {
8534 // Same as above.
8535 BottomHalf = Ret.getOperand(0);
8536 TopHalf = Ret.getOperand(1);
8537 } else {
8538 BottomHalf = Ret.getOperand(1);
8539 TopHalf = Ret.getOperand(0);
8540 }
8541 }
8542
8543 Result = BottomHalf;
8544 if (isSigned) {
8545 SDValue ShiftAmt = DAG.getConstant(
8546 VT.getScalarSizeInBits() - 1, dl,
8547 getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
8548 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
8549 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
8550 } else {
8551 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,
8552 DAG.getConstant(0, dl, VT), ISD::SETNE);
8553 }
8554
8555 // Truncate the result if SetCC returns a larger type than needed.
8556 EVT RType = Node->getValueType(1);
8557 if (RType.bitsLT(Overflow.getValueType()))
8558 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);
8559
8560 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&((void)0)
8561 "Unexpected result type for S/UMULO legalization")((void)0);
8562 return true;
8563}
8564
8565SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
8566 SDLoc dl(Node);
8567 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
8568 SDValue Op = Node->getOperand(0);
8569 EVT VT = Op.getValueType();
8570
8571 if (VT.isScalableVector())
8572 report_fatal_error(
8573 "Expanding reductions for scalable vectors is undefined.");
8574
8575 // Try to use a shuffle reduction for power of two vectors.
8576 if (VT.isPow2VectorType()) {
8577 while (VT.getVectorNumElements() > 1) {
8578 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
8579 if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))
8580 break;
8581
8582 SDValue Lo, Hi;
8583 std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
8584 Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi);
8585 VT = HalfVT;
8586 }
8587 }
8588
8589 EVT EltVT = VT.getVectorElementType();
8590 unsigned NumElts = VT.getVectorNumElements();
8591
8592 SmallVector<SDValue, 8> Ops;
8593 DAG.ExtractVectorElements(Op, Ops, 0, NumElts);
8594
8595 SDValue Res = Ops[0];
8596 for (unsigned i = 1; i < NumElts; i++)
8597 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());
8598
8599 // Result type may be wider than element type.
8600 if (EltVT != Node->getValueType(0))
8601 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
8602 return Res;
8603}
8604
8605SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
8606 SDLoc dl(Node);
8607 SDValue AccOp = Node->getOperand(0);
8608 SDValue VecOp = Node->getOperand(1);
8609 SDNodeFlags Flags = Node->getFlags();
8610
8611 EVT VT = VecOp.getValueType();
8612 EVT EltVT = VT.getVectorElementType();
8613
8614 if (VT.isScalableVector())
8615 report_fatal_error(
8616 "Expanding reductions for scalable vectors is undefined.");
8617
8618 unsigned NumElts = VT.getVectorNumElements();
8619
8620 SmallVector<SDValue, 8> Ops;
8621 DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);
8622
8623 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
8624
8625 SDValue Res = AccOp;
8626 for (unsigned i = 0; i < NumElts; i++)
8627 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);
8628
8629 return Res;
8630}
8631
8632bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
8633 SelectionDAG &DAG) const {
8634 EVT VT = Node->getValueType(0);
8635 SDLoc dl(Node);
8636 bool isSigned = Node->getOpcode() == ISD::SREM;
8637 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
8638 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
8639 SDValue Dividend = Node->getOperand(0);
8640 SDValue Divisor = Node->getOperand(1);
8641 if (isOperationLegalOrCustom(DivRemOpc, VT)) {
8642 SDVTList VTs = DAG.getVTList(VT, VT);
8643 Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
8644 return true;
8645 }
8646 if (isOperationLegalOrCustom(DivOpc, VT)) {
8647 // X % Y -> X-X/Y*Y
8648 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
8649 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
8650 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
8651 return true;
8652 }
8653 return false;
8654}
8655
8656SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
8657 SelectionDAG &DAG) const {
8658 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
8659 SDLoc dl(SDValue(Node, 0));
8660 SDValue Src = Node->getOperand(0);
8661
8662 // DstVT is the result type, while SatVT is the size to which we saturate
8663 EVT SrcVT = Src.getValueType();
8664 EVT DstVT = Node->getValueType(0);
8665
8666 EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
8667 unsigned SatWidth = SatVT.getScalarSizeInBits();
8668 unsigned DstWidth = DstVT.getScalarSizeInBits();
8669 assert(SatWidth <= DstWidth &&((void)0)
8670 "Expected saturation width smaller than result width")((void)0);
8671
8672 // Determine minimum and maximum integer values and their corresponding
8673 // floating-point values.
8674 APInt MinInt, MaxInt;
8675 if (IsSigned) {
8676 MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth);
8677 MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth);
8678 } else {
8679 MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth);
8680 MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth);
8681 }
8682
8683 // We cannot risk emitting FP_TO_XINT nodes with a source VT of f16, as
8684 // libcall emission cannot handle this. Large result types will fail.
8685 if (SrcVT == MVT::f16) {
8686 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);
8687 SrcVT = Src.getValueType();
8688 }
8689
8690 APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));
8691 APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));
8692
8693 APFloat::opStatus MinStatus =
8694 MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
8695 APFloat::opStatus MaxStatus =
8696 MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
8697 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8698 !(MaxStatus & APFloat::opStatus::opInexact);
8699
8700 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
8701 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
8702
8703 // If the integer bounds are exactly representable as floats and min/max are
8704 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
8705 // of comparisons and selects.
8706 bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&
8707 isOperationLegal(ISD::FMAXNUM, SrcVT);
8708 if (AreExactFloatBounds && MinMaxLegal) {
8709 SDValue Clamped = Src;
8710
8711 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8712 Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);
8713 // Clamp by MaxFloat from above. NaN cannot occur.
8714 Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);
8715 // Convert clamped value to integer.
8716 SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
8717 dl, DstVT, Clamped);
8718
8719 // In the unsigned case we're done, because we mapped NaN to MinFloat,
8720 // which will cast to zero.
8721 if (!IsSigned)
8722 return FpToInt;
8723
8724 // Otherwise, select 0 if Src is NaN.
8725 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
8726 return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,
8727 ISD::CondCode::SETUO);
8728 }
8729
8730 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
8731 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
8732
8733 // Result of direct conversion. The assumption here is that the operation is
8734 // non-trapping and it's fine to apply it to an out-of-range value if we
8735 // select it away later.
8736 SDValue FpToInt =
8737 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);
8738
8739 SDValue Select = FpToInt;
8740
8741 // If Src ULT MinFloat, select MinInt. In particular, this also selects
8742 // MinInt if Src is NaN.
8743 Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select,
8744 ISD::CondCode::SETULT);
8745 // If Src OGT MaxFloat, select MaxInt.
8746 Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,
8747 ISD::CondCode::SETOGT);
8748
8749 // In the unsigned case we are done, because we mapped NaN to MinInt, which
8750 // is already zero.
8751 if (!IsSigned)
8752 return Select;
8753
8754 // Otherwise, select 0 if Src is NaN.
8755 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
8756 return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
8757}
8758
8759SDValue TargetLowering::expandVectorSplice(SDNode *Node,
8760 SelectionDAG &DAG) const {
8761 assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!")((void)0);
8762 assert(Node->getValueType(0).isScalableVector() &&((void)0)
8763 "Fixed length vector types expected to use SHUFFLE_VECTOR!")((void)0);
8764
8765 EVT VT = Node->getValueType(0);
8766 SDValue V1 = Node->getOperand(0);
8767 SDValue V2 = Node->getOperand(1);
8768 int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();
8769 SDLoc DL(Node);
8770
8771 // Expand through memory thusly:
8772 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
8773 // Store V1, Ptr
8774 // Store V2, Ptr + sizeof(V1)
8775 // If (Imm < 0)
8776 // TrailingElts = -Imm
8777 // Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt))
8778 // else
8779 // Ptr = Ptr + (Imm * sizeof(VT.Elt))
8780 // Res = Load Ptr
8781
8782 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8783
8784 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
8785 VT.getVectorElementCount() * 2);
8786 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
8787 EVT PtrVT = StackPtr.getValueType();
8788 auto &MF = DAG.getMachineFunction();
8789 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
8790 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
8791
8792 // Store the lo part of CONCAT_VECTORS(V1, V2)
8793 SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo);
8794 // Store the hi part of CONCAT_VECTORS(V1, V2)
8795 SDValue OffsetToV2 = DAG.getVScale(
8796 DL, PtrVT,
8797 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
8798 SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2);
8799 SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo);
8800
8801 if (Imm >= 0) {
8802 // Load back the required element. getVectorElementPointer takes care of
8803 // clamping the index if it's out-of-bounds.
8804 StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));
8805 // Load the spliced result
8806 return DAG.getLoad(VT, DL, StoreV2, StackPtr,
8807 MachinePointerInfo::getUnknownStack(MF));
8808 }
8809
8810 uint64_t TrailingElts = -Imm;
8811
8812 // NOTE: TrailingElts must be clamped so as not to read outside of V1:V2.
8813 TypeSize EltByteSize = VT.getVectorElementType().getStoreSize();
8814 SDValue TrailingBytes =
8815 DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT);
8816
8817 if (TrailingElts > VT.getVectorMinNumElements()) {
8818 SDValue VLBytes = DAG.getVScale(
8819 DL, PtrVT,
8820 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
8821 TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes);
8822 }
8823
8824 // Calculate the start address of the spliced result.
8825 StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);
8826
8827 // Load the spliced result
8828 return DAG.getLoad(VT, DL, StoreV2, StackPtr2,
8829 MachinePointerInfo::getUnknownStack(MF));
8830}
8831
8832bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
8833 SDValue &LHS, SDValue &RHS,
8834 SDValue &CC, bool &NeedInvert,
8835 const SDLoc &dl, SDValue &Chain,
8836 bool IsSignaling) const {
8837 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8838 MVT OpVT = LHS.getSimpleValueType();
8839 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
8840 NeedInvert = false;
8841 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
8842 default:
8843 llvm_unreachable("Unknown condition code action!")__builtin_unreachable();
8844 case TargetLowering::Legal:
8845 // Nothing to do.
8846 break;
8847 case TargetLowering::Expand: {
8848 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
8849 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8850 std::swap(LHS, RHS);
8851 CC = DAG.getCondCode(InvCC);
8852 return true;
8853 }
8854 // Swapping operands didn't work. Try inverting the condition.
8855 bool NeedSwap = false;
8856 InvCC = getSetCCInverse(CCCode, OpVT);
8857 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8858 // If inverting the condition is not enough, try swapping operands
8859 // on top of it.
8860 InvCC = ISD::getSetCCSwappedOperands(InvCC);
8861 NeedSwap = true;
8862 }
8863 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8864 CC = DAG.getCondCode(InvCC);
8865 NeedInvert = true;
8866 if (NeedSwap)
8867 std::swap(LHS, RHS);
8868 return true;
8869 }
8870
8871 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
8872 unsigned Opc = 0;
8873 switch (CCCode) {
8874 default:
8875 llvm_unreachable("Don't know how to expand this condition!")__builtin_unreachable();
8876 case ISD::SETUO:
8877 if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
8878 CC1 = ISD::SETUNE;
8879 CC2 = ISD::SETUNE;
8880 Opc = ISD::OR;
8881 break;
8882 }
8883 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&((void)0)
8884 "If SETUE is expanded, SETOEQ or SETUNE must be legal!")((void)0);
8885 NeedInvert = true;
8886 LLVM_FALLTHROUGH[[gnu::fallthrough]];
8887 case ISD::SETO:
8888 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&((void)0)
8889 "If SETO is expanded, SETOEQ must be legal!")((void)0);
8890 CC1 = ISD::SETOEQ;
8891 CC2 = ISD::SETOEQ;
8892 Opc = ISD::AND;
8893 break;
8894 case ISD::SETONE:
8895 case ISD::SETUEQ:
8896 // If the SETUO or SETO CC isn't legal, we might be able to use
8897 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
8898 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
8899 // the operands.
8900 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
8901 if (!TLI.isCondCodeLegal(CC2, OpVT) &&
8902 (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
8903 TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
8904 CC1 = ISD::SETOGT;
8905 CC2 = ISD::SETOLT;
8906 Opc = ISD::OR;
8907 NeedInvert = ((unsigned)CCCode & 0x8U);
8908 break;
8909 }
8910 LLVM_FALLTHROUGH[[gnu::fallthrough]];
8911 case ISD::SETOEQ:
8912 case ISD::SETOGT:
8913 case ISD::SETOGE:
8914 case ISD::SETOLT:
8915 case ISD::SETOLE:
8916 case ISD::SETUNE:
8917 case ISD::SETUGT:
8918 case ISD::SETUGE:
8919 case ISD::SETULT:
8920 case ISD::SETULE:
8921 // If we are floating point, assign and break, otherwise fall through.
8922 if (!OpVT.isInteger()) {
8923 // We can use the 4th bit to tell if we are the unordered
8924 // or ordered version of the opcode.
8925 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
8926 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
8927 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
8928 break;
8929 }
8930 // Fallthrough if we are unsigned integer.
8931 LLVM_FALLTHROUGH[[gnu::fallthrough]];
8932 case ISD::SETLE:
8933 case ISD::SETGT:
8934 case ISD::SETGE:
8935 case ISD::SETLT:
8936 case ISD::SETNE:
8937 case ISD::SETEQ:
8938 // If all combinations of inverting the condition and swapping operands
8939 // didn't work then we have no means to expand the condition.
8940 llvm_unreachable("Don't know how to expand this condition!")__builtin_unreachable();
8941 }
8942
8943 SDValue SetCC1, SetCC2;
8944 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
8945 // If we aren't the ordered or unorder operation,
8946 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
8947 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
8948 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
8949 } else {
8950 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
8951 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
8952 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
8953 }
8954 if (Chain)
8955 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
8956 SetCC2.getValue(1));
8957 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
8958 RHS = SDValue();
8959 CC = SDValue();
8960 return true;
8961 }
8962 }
8963 return false;
8964}
←

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support/Casting.h

→
1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
10// and dyn_cast_or_null<X>() templates.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_CASTING_H
15#define LLVM_SUPPORT_CASTING_H
16
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/type_traits.h"
19#include <cassert>
20#include <memory>
21#include <type_traits>
22
23namespace llvm {
24
25//===----------------------------------------------------------------------===//
26// isa<x> Support Templates
27//===----------------------------------------------------------------------===//
28
29// Define a template that can be specialized by smart pointers to reflect the
30// fact that they are automatically dereferenced, and are not involved with the
31// template selection process... the default implementation is a noop.
32//
33template<typename From> struct simplify_type {
34 using SimpleType = From; // The real type this represents...
35
36 // An accessor to get the real value...
37 static SimpleType &getSimplifiedValue(From &Val) { return Val; }
38};
39
40template<typename From> struct simplify_type<const From> {
41 using NonConstSimpleType = typename simplify_type<From>::SimpleType;
42 using SimpleType =
43 typename add_const_past_pointer<NonConstSimpleType>::type;
44 using RetType =
45 typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
46
47 static RetType getSimplifiedValue(const From& Val) {
48 return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
49 }
50};
51
52// The core of the implementation of isa<X> is here; To and From should be
53// the names of classes. This template can be specialized to customize the
54// implementation of isa<> without rewriting it from scratch.
55template <typename To, typename From, typename Enabler = void>
56struct isa_impl {
57 static inline bool doit(const From &Val) {
58 return To::classof(&Val);
59 }
60};
61
62/// Always allow upcasts, and perform no dynamic check for them.
63template <typename To, typename From>
64struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> {
65 static inline bool doit(const From &) { return true; }
66};
67
68template <typename To, typename From> struct isa_impl_cl {
69 static inline bool doit(const From &Val) {
70 return isa_impl<To, From>::doit(Val);
71 }
72};
73
74template <typename To, typename From> struct isa_impl_cl<To, const From> {
75 static inline bool doit(const From &Val) {
76 return isa_impl<To, From>::doit(Val);
77 }
78};
79
80template <typename To, typename From>
81struct isa_impl_cl<To, const std::unique_ptr<From>> {
82 static inline bool doit(const std::unique_ptr<From> &Val) {
83 assert(Val && "isa<> used on a null pointer")((void)0);
84 return isa_impl_cl<To, From>::doit(*Val);
85 }
86};
87
88template <typename To, typename From> struct isa_impl_cl<To, From*> {
89 static inline bool doit(const From *Val) {
90 assert(Val && "isa<> used on a null pointer")((void)0);
91 return isa_impl<To, From>::doit(*Val);
92 }
93};
94
95template <typename To, typename From> struct isa_impl_cl<To, From*const> {
96 static inline bool doit(const From *Val) {
97 assert(Val && "isa<> used on a null pointer")((void)0);
98 return isa_impl<To, From>::doit(*Val);
99 }
100};
101
102template <typename To, typename From> struct isa_impl_cl<To, const From*> {
103 static inline bool doit(const From *Val) {
104 assert(Val && "isa<> used on a null pointer")((void)0);
105 return isa_impl<To, From>::doit(*Val);
106 }
107};
108
109template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
110 static inline bool doit(const From *Val) {
111 assert(Val && "isa<> used on a null pointer")((void)0);
112 return isa_impl<To, From>::doit(*Val);
113 }
114};
115
116template<typename To, typename From, typename SimpleFrom>
117struct isa_impl_wrap {
118 // When From != SimplifiedType, we can simplify the type some more by using
119 // the simplify_type template.
120 static bool doit(const From &Val) {
121 return isa_impl_wrap<To, SimpleFrom,
122 typename simplify_type<SimpleFrom>::SimpleType>::doit(
123 simplify_type<const From>::getSimplifiedValue(Val));
124 }
125};
126
127template<typename To, typename FromTy>
128struct isa_impl_wrap<To, FromTy, FromTy> {
129 // When From == SimpleType, we are as simple as we are going to get.
130 static bool doit(const FromTy &Val) {
131 return isa_impl_cl<To,FromTy>::doit(Val);
132 }
133};
134
135// isa<X> - Return true if the parameter to the template is an instance of one
136// of the template type arguments. Used like this:
137//
138// if (isa<Type>(myVal)) { ... }
139// if (isa<Type0, Type1, Type2>(myVal)) { ... }
140//
141template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
142 return isa_impl_wrap<X, const Y,
143 typename simplify_type<const Y>::SimpleType>::doit(Val);
144}
145
146template <typename First, typename Second, typename... Rest, typename Y>
147LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
148 return isa<First>(Val) || isa<Second, Rest...>(Val);
149}
150
151// isa_and_nonnull<X> - Functionally identical to isa, except that a null value
152// is accepted.
153//
154template <typename... X, class Y>
155LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa_and_nonnull(const Y &Val) {
156 if (!Val)
157 return false;
158 return isa<X...>(Val);
159}
160
161//===----------------------------------------------------------------------===//
162// cast<x> Support Templates
163//===----------------------------------------------------------------------===//
164
165template<class To, class From> struct cast_retty;
166
167// Calculate what type the 'cast' function should return, based on a requested
168// type of To and a source type of From.
169template<class To, class From> struct cast_retty_impl {
170 using ret_type = To &; // Normal case, return Ty&
171};
172template<class To, class From> struct cast_retty_impl<To, const From> {
173 using ret_type = const To &; // Normal case, return Ty&
174};
175
176template<class To, class From> struct cast_retty_impl<To, From*> {
177 using ret_type = To *; // Pointer arg case, return Ty*
178};
179
180template<class To, class From> struct cast_retty_impl<To, const From*> {
181 using ret_type = const To *; // Constant pointer arg case, return const Ty*
182};
183
184template<class To, class From> struct cast_retty_impl<To, const From*const> {
185 using ret_type = const To *; // Constant pointer arg case, return const Ty*
186};
187
188template <class To, class From>
189struct cast_retty_impl<To, std::unique_ptr<From>> {
190private:
191 using PointerType = typename cast_retty_impl<To, From *>::ret_type;
192 using ResultType = std::remove_pointer_t<PointerType>;
193
194public:
195 using ret_type = std::unique_ptr<ResultType>;
196};
197
198template<class To, class From, class SimpleFrom>
199struct cast_retty_wrap {
200 // When the simplified type and the from type are not the same, use the type
201 // simplifier to reduce the type, then reuse cast_retty_impl to get the
202 // resultant type.
203 using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
204};
205
206template<class To, class FromTy>
207struct cast_retty_wrap<To, FromTy, FromTy> {
208 // When the simplified type is equal to the from type, use it directly.
209 using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
210};
211
212template<class To, class From>
213struct cast_retty {
214 using ret_type = typename cast_retty_wrap<
215 To, From, typename simplify_type<From>::SimpleType>::ret_type;
216};
217
218// Ensure the non-simple values are converted using the simplify_type template
219// that may be specialized by smart pointers...
220//
221template<class To, class From, class SimpleFrom> struct cast_convert_val {
222 // This is not a simple type, use the template to simplify it...
223 static typename cast_retty<To, From>::ret_type doit(From &Val) {
224 return cast_convert_val<To, SimpleFrom,
59
←
Returning without writing to 'Val.Node'
→
225 typename simplify_type<SimpleFrom>::SimpleType>::doit(
226 simplify_type<From>::getSimplifiedValue(Val));
56
←
Calling 'simplify_type::getSimplifiedValue'
→
58
←
Returning from 'simplify_type::getSimplifiedValue'
→
227 }
228};
229
230template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
231 // This _is_ a simple type, just cast it.
232 static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
233 typename cast_retty<To, FromTy>::ret_type Res2
234 = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
235 return Res2;
236 }
237};
238
239template <class X> struct is_simple_type {
240 static const bool value =
241 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
242};
243
244// cast<X> - Return the argument parameter cast to the specified type. This
245// casting operator asserts that the type is correct, so it does not return null
246// on failure. It does not allow a null argument (use cast_or_null for that).
247// It is typically used like this:
248//
249// cast<Instruction>(myVal)->getParent()
250//
251template <class X, class Y>
252inline std::enable_if_t<!is_simple_type<Y>::value,
253 typename cast_retty<X, const Y>::ret_type>
254cast(const Y &Val) {
255 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((void)0);
256 return cast_convert_val<
257 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
258}
259
260template <class X, class Y>
261inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
262 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((void)0);
263 return cast_convert_val<X, Y,
55
←
Calling 'cast_convert_val::doit'
→
60
←
Returning from 'cast_convert_val::doit'
→
61
←
Returning without writing to 'Val.Node'
→
264 typename simplify_type<Y>::SimpleType>::doit(Val);
265}
266
267template <class X, class Y>
268inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
269 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((void)0);
270 return cast_convert_val<X, Y*,
271 typename simplify_type<Y*>::SimpleType>::doit(Val);
272}
273
274template <class X, class Y>
275inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
276cast(std::unique_ptr<Y> &&Val) {
277 assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")((void)0);
278 using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
279 return ret_type(
280 cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
281 Val.release()));
282}
283
284// cast_or_null<X> - Functionally identical to cast, except that a null value is
285// accepted.
286//
287template <class X, class Y>
288LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
289 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
290cast_or_null(const Y &Val) {
291 if (!Val)
292 return nullptr;
293 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((void)0);
294 return cast<X>(Val);
295}
296
297template <class X, class Y>
298LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
299 typename cast_retty<X, Y>::ret_type>
300cast_or_null(Y &Val) {
301 if (!Val)
302 return nullptr;
303 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((void)0);
304 return cast<X>(Val);
305}
306
307template <class X, class Y>
308LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
309cast_or_null(Y *Val) {
310 if (!Val) return nullptr;
311 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((void)0);
312 return cast<X>(Val);
313}
314
315template <class X, class Y>
316inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
317cast_or_null(std::unique_ptr<Y> &&Val) {
318 if (!Val)
319 return nullptr;
320 return cast<X>(std::move(Val));
321}
322
323// dyn_cast<X> - Return the argument parameter cast to the specified type. This
324// casting operator returns null if the argument is of the wrong type, so it can
325// be used to test for a type as well as cast if successful. This should be
326// used in the context of an if statement like this:
327//
328// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
329//
330
331template <class X, class Y>
332LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
333 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
334dyn_cast(const Y &Val) {
335 return isa<X>(Val) ? cast<X>(Val) : nullptr;
336}
337
338template <class X, class Y>
339LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
340 return isa<X>(Val) ? cast<X>(Val) : nullptr;
15
←
Assuming 'Val' is not a 'GlobalAddressSDNode'
→
16
←
'?' condition is false
→
17
←
Returning without writing to 'Val.Node'
→
21
←
Assuming 'Val' is not a 'ConstantSDNode'
→
22
←
'?' condition is false
→
23
←
Returning without writing to 'Val.Node'
→
27
←
Assuming 'Val' is not a 'BlockAddressSDNode'
→
28
←
'?' condition is false
→
29
←
Returning without writing to 'Val.Node'
→
40
←
Assuming 'Val' is not a 'GlobalAddressSDNode'
→
41
←
'?' condition is false
→
42
←
Returning without writing to 'Val.Node'
→
46
←
Assuming 'Val' is not a 'ConstantSDNode'
→
47
←
'?' condition is false
→
48
←
Returning without writing to 'Val.Node'
→
52
←
Assuming 'Val' is a 'BlockAddressSDNode'
→
53
←
'?' condition is true
→
54
←
Calling 'cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
62
←
Returning from 'cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
63
←
Returning without writing to 'Val.Node'
→
341}
342
343template <class X, class Y>
344LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
345 return isa<X>(Val) ? cast<X>(Val) : nullptr;
346}
347
348// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
349// value is accepted.
350//
351template <class X, class Y>
352LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
353 !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
354dyn_cast_or_null(const Y &Val) {
355 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
356}
357
358template <class X, class Y>
359LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
360 typename cast_retty<X, Y>::ret_type>
361dyn_cast_or_null(Y &Val) {
362 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
363}
364
365template <class X, class Y>
366LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
367dyn_cast_or_null(Y *Val) {
368 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
369}
370
371// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
372// taking ownership of the input pointer iff isa<X>(Val) is true. If the
373// cast is successful, From refers to nullptr on exit and the casted value
374// is returned. If the cast is unsuccessful, the function returns nullptr
375// and From is unchanged.
376template <class X, class Y>
377LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
378 -> decltype(cast<X>(Val)) {
379 if (!isa<X>(Val))
380 return nullptr;
381 return cast<X>(std::move(Val));
382}
383
384template <class X, class Y>
385LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) {
386 return unique_dyn_cast<X, Y>(Val);
387}
388
389// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
390// a null value is accepted.
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
393 -> decltype(cast<X>(Val)) {
394 if (!Val)
395 return nullptr;
396 return unique_dyn_cast<X, Y>(Val);
397}
398
399template <class X, class Y>
400LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val) {
401 return unique_dyn_cast_or_null<X, Y>(Val);
402}
403
404} // end namespace llvm
405
406#endif // LLVM_SUPPORT_CASTING_H
←

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
57
←
Returning without writing to 'Val.Node'
→
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class MaskedLoadStoreSDNode;
513 friend class MaskedGatherScatterSDNode;
514
515 uint16_t : NumMemSDNodeBits;
516
517 // This storage is shared between disparate class hierarchies to hold an
518 // enumeration specific to the class hierarchy in use.
519 // LSBaseSDNode => enum ISD::MemIndexedMode
520 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
521 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
522 uint16_t AddressingMode : 3;
523 };
524 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
525
526 class LoadSDNodeBitfields {
527 friend class LoadSDNode;
528 friend class MaskedLoadSDNode;
529 friend class MaskedGatherSDNode;
530
531 uint16_t : NumLSBaseSDNodeBits;
532
533 uint16_t ExtTy : 2; // enum ISD::LoadExtType
534 uint16_t IsExpanding : 1;
535 };
536
537 class StoreSDNodeBitfields {
538 friend class StoreSDNode;
539 friend class MaskedStoreSDNode;
540 friend class MaskedScatterSDNode;
541
542 uint16_t : NumLSBaseSDNodeBits;
543
544 uint16_t IsTruncating : 1;
545 uint16_t IsCompressing : 1;
546 };
547
548 union {
549 char RawSDNodeBits[sizeof(uint16_t)];
550 SDNodeBitfields SDNodeBits;
551 ConstantSDNodeBitfields ConstantSDNodeBits;
552 MemSDNodeBitfields MemSDNodeBits;
553 LSBaseSDNodeBitfields LSBaseSDNodeBits;
554 LoadSDNodeBitfields LoadSDNodeBits;
555 StoreSDNodeBitfields StoreSDNodeBits;
556 };
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK
560
561 // RawSDNodeBits must cover the entirety of the union. This means that all of
562 // the union's members must have size <= RawSDNodeBits. We write the RHS as
563 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
564 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
565 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
566 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
567 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
568 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
569 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
570
571private:
572 friend class SelectionDAG;
573 // TODO: unfriend HandleSDNode once we fix its operand handling.
574 friend class HandleSDNode;
575
576 /// Unique id per SDNode in the DAG.
577 int NodeId = -1;
578
579 /// The values that are used by this operation.
580 SDUse *OperandList = nullptr;
581
582 /// The types of the values this node defines. SDNode's may
583 /// define multiple values simultaneously.
584 const EVT *ValueList;
585
586 /// List of uses for this SDNode.
587 SDUse *UseList = nullptr;
588
589 /// The number of entries in the Operand/Value list.
590 unsigned short NumOperands = 0;
591 unsigned short NumValues;
592
593 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
594 // original LLVM instructions.
595 // This is used for turning off scheduling, because we'll forgo
596 // the normal scheduling algorithms and output the instructions according to
597 // this ordering.
598 unsigned IROrder;
599
600 /// Source line information.
601 DebugLoc debugLoc;
602
603 /// Return a pointer to the specified value type.
604 static const EVT *getValueTypeList(EVT VT);
605
606 SDNodeFlags Flags;
607
608public:
609 /// Unique and persistent id per SDNode in the DAG.
610 /// Used for debug printing.
611 uint16_t PersistentId;
612
613 //===--------------------------------------------------------------------===//
614 // Accessors
615 //
616
617 /// Return the SelectionDAG opcode value for this node. For
618 /// pre-isel nodes (those for which isMachineOpcode returns false), these
619 /// are the opcode values in the ISD and <target>ISD namespaces. For
620 /// post-isel opcodes, see getMachineOpcode.
621 unsigned getOpcode() const { return (unsigned short)NodeType; }
622
623 /// Test if this node has a target-specific opcode (in the
624 /// \<target\>ISD namespace).
625 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
626
627 /// Test if this node has a target-specific opcode that may raise
628 /// FP exceptions (in the \<target\>ISD namespace and greater than
629 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
630 /// opcode are currently automatically considered to possibly raise
631 /// FP exceptions as well.
632 bool isTargetStrictFPOpcode() const {
633 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
634 }
635
636 /// Test if this node has a target-specific
637 /// memory-referencing opcode (in the \<target\>ISD namespace and
638 /// greater than FIRST_TARGET_MEMORY_OPCODE).
639 bool isTargetMemoryOpcode() const {
640 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
641 }
642
643 /// Return true if the type of the node type undefined.
644 bool isUndef() const { return NodeType == ISD::UNDEF; }
645
646 /// Test if this node is a memory intrinsic (with valid pointer information).
647 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
648 /// non-memory intrinsics (with chains) that are not really instances of
649 /// MemSDNode. For such nodes, we need some extra state to determine the
650 /// proper classof relationship.
651 bool isMemIntrinsic() const {
652 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
653 NodeType == ISD::INTRINSIC_VOID) &&
654 SDNodeBits.IsMemIntrinsic;
655 }
656
657 /// Test if this node is a strict floating point pseudo-op.
658 bool isStrictFPOpcode() {
659 switch (NodeType) {
660 default:
661 return false;
662 case ISD::STRICT_FP16_TO_FP:
663 case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
665 case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
667 return true;
668 }
669 }
670
671 /// Test if this node has a post-isel opcode, directly
672 /// corresponding to a MachineInstr opcode.
673 bool isMachineOpcode() const { return NodeType < 0; }
674
675 /// This may only be called if isMachineOpcode returns
676 /// true. It returns the MachineInstr opcode value that the node's opcode
677 /// corresponds to.
678 unsigned getMachineOpcode() const {
679 assert(isMachineOpcode() && "Not a MachineInstr opcode!")((void)0);
680 return ~NodeType;
681 }
682
683 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
684 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
685
686 bool isDivergent() const { return SDNodeBits.IsDivergent; }
687
688 /// Return true if there are no uses of this node.
689 bool use_empty() const { return UseList == nullptr; }
690
691 /// Return true if there is exactly one use of this node.
692 bool hasOneUse() const { return hasSingleElement(uses()); }
693
694 /// Return the number of uses of this node. This method takes
695 /// time proportional to the number of uses.
696 size_t use_size() const { return std::distance(use_begin(), use_end()); }
697
698 /// Return the unique node id.
699 int getNodeId() const { return NodeId; }
700
701 /// Set unique node id.
702 void setNodeId(int Id) { NodeId = Id; }
703
704 /// Return the node ordering.
705 unsigned getIROrder() const { return IROrder; }
706
707 /// Set the node ordering.
708 void setIROrder(unsigned Order) { IROrder = Order; }
709
710 /// Return the source location info.
711 const DebugLoc &getDebugLoc() const { return debugLoc; }
712
713 /// Set source location info. Try to avoid this, putting
714 /// it in the constructor is preferable.
715 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
716
717 /// This class provides iterator support for SDUse
718 /// operands that use a specific SDNode.
719 class use_iterator {
720 friend class SDNode;
721
722 SDUse *Op = nullptr;
723
724 explicit use_iterator(SDUse *op) : Op(op) {}
725
726 public:
727 using iterator_category = std::forward_iterator_tag;
728 using value_type = SDUse;
729 using difference_type = std::ptrdiff_t;
730 using pointer = value_type *;
731 using reference = value_type &;
732
733 use_iterator() = default;
734 use_iterator(const use_iterator &I) : Op(I.Op) {}
735
736 bool operator==(const use_iterator &x) const {
737 return Op == x.Op;
738 }
739 bool operator!=(const use_iterator &x) const {
740 return !operator==(x);
741 }
742
743 /// Return true if this iterator is at the end of uses list.
744 bool atEnd() const { return Op == nullptr; }
745
746 // Iterator traversal: forward iteration only.
747 use_iterator &operator++() { // Preincrement
748 assert(Op && "Cannot increment end iterator!")((void)0);
749 Op = Op->getNext();
750 return *this;
751 }
752
753 use_iterator operator++(int) { // Postincrement
754 use_iterator tmp = *this; ++*this; return tmp;
755 }
756
757 /// Retrieve a pointer to the current user node.
758 SDNode *operator*() const {
759 assert(Op && "Cannot dereference end iterator!")((void)0);
760 return Op->getUser();
761 }
762
763 SDNode *operator->() const { return operator*(); }
764
765 SDUse &getUse() const { return *Op; }
766
767 /// Retrieve the operand # of this use in its user.
768 unsigned getOperandNo() const {
769 assert(Op && "Cannot dereference end iterator!")((void)0);
770 return (unsigned)(Op - Op->getUser()->OperandList);
771 }
772 };
773
774 /// Provide iteration support to walk over all uses of an SDNode.
775 use_iterator use_begin() const {
776 return use_iterator(UseList);
777 }
778
779 static use_iterator use_end() { return use_iterator(nullptr); }
780
781 inline iterator_range<use_iterator> uses() {
782 return make_range(use_begin(), use_end());
783 }
784 inline iterator_range<use_iterator> uses() const {
785 return make_range(use_begin(), use_end());
786 }
787
788 /// Return true if there are exactly NUSES uses of the indicated value.
789 /// This method ignores uses of other values defined by this operation.
790 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
791
792 /// Return true if there are any use of the indicated value.
793 /// This method ignores uses of other values defined by this operation.
794 bool hasAnyUseOfValue(unsigned Value) const;
795
796 /// Return true if this node is the only use of N.
797 bool isOnlyUserOf(const SDNode *N) const;
798
799 /// Return true if this node is an operand of N.
800 bool isOperandOf(const SDNode *N) const;
801
802 /// Return true if this node is a predecessor of N.
803 /// NOTE: Implemented on top of hasPredecessor and every bit as
804 /// expensive. Use carefully.
805 bool isPredecessorOf(const SDNode *N) const {
806 return N->hasPredecessor(this);
807 }
808
809 /// Return true if N is a predecessor of this node.
810 /// N is either an operand of this node, or can be reached by recursively
811 /// traversing up the operands.
812 /// NOTE: This is an expensive method. Use it carefully.
813 bool hasPredecessor(const SDNode *N) const;
814
815 /// Returns true if N is a predecessor of any node in Worklist. This
816 /// helper keeps Visited and Worklist sets externally to allow unions
817 /// searches to be performed in parallel, caching of results across
818 /// queries and incremental addition to Worklist. Stops early if N is
819 /// found but will resume. Remember to clear Visited and Worklists
820 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
821 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
822 /// topologically ordered (Operands have strictly smaller node id) and search
823 /// can be pruned leveraging this.
824 static bool hasPredecessorHelper(const SDNode *N,
825 SmallPtrSetImpl<const SDNode *> &Visited,
826 SmallVectorImpl<const SDNode *> &Worklist,
827 unsigned int MaxSteps = 0,
828 bool TopologicalPrune = false) {
829 SmallVector<const SDNode *, 8> DeferredNodes;
830 if (Visited.count(N))
831 return true;
832
833 // Node Id's are assigned in three places: As a topological
834 // ordering (> 0), during legalization (results in values set to
835 // 0), new nodes (set to -1). If N has a topolgical id then we
836 // know that all nodes with ids smaller than it cannot be
837 // successors and we need not check them. Filter out all node
838 // that can't be matches. We add them to the worklist before exit
839 // in case of multiple calls. Note that during selection the topological id
840 // may be violated if a node's predecessor is selected before it. We mark
841 // this at selection negating the id of unselected successors and
842 // restricting topological pruning to positive ids.
843
844 int NId = N->getNodeId();
845 // If we Invalidated the Id, reconstruct original NId.
846 if (NId < -1)
847 NId = -(NId + 1);
848
849 bool Found = false;
850 while (!Worklist.empty()) {
851 const SDNode *M = Worklist.pop_back_val();
852 int MId = M->getNodeId();
853 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
854 (MId > 0) && (MId < NId)) {
855 DeferredNodes.push_back(M);
856 continue;
857 }
858 for (const SDValue &OpV : M->op_values()) {
859 SDNode *Op = OpV.getNode();
860 if (Visited.insert(Op).second)
861 Worklist.push_back(Op);
862 if (Op == N)
863 Found = true;
864 }
865 if (Found)
866 break;
867 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
868 break;
869 }
870 // Push deferred nodes back on worklist.
871 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
872 // If we bailed early, conservatively return found.
873 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
874 return true;
875 return Found;
876 }
877
878 /// Return true if all the users of N are contained in Nodes.
879 /// NOTE: Requires at least one match, but doesn't require them all.
880 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
881
882 /// Return the number of values used by this operation.
883 unsigned getNumOperands() const { return NumOperands; }
884
885 /// Return the maximum number of operands that a SDNode can hold.
886 static constexpr size_t getMaxNumOperands() {
887 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
888 }
889
890 /// Helper method returns the integer value of a ConstantSDNode operand.
891 inline uint64_t getConstantOperandVal(unsigned Num) const;
892
893 /// Helper method returns the APInt of a ConstantSDNode operand.
894 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
895
896 const SDValue &getOperand(unsigned Num) const {
897 assert(Num < NumOperands && "Invalid child # of SDNode!")((void)0);
898 return OperandList[Num];
899 }
900
901 using op_iterator = SDUse *;
902
903 op_iterator op_begin() const { return OperandList; }
904 op_iterator op_end() const { return OperandList+NumOperands; }
905 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
906
907 /// Iterator for directly iterating over the operand SDValue's.
908 struct value_op_iterator
909 : iterator_adaptor_base<value_op_iterator, op_iterator,
910 std::random_access_iterator_tag, SDValue,
911 ptrdiff_t, value_op_iterator *,
912 value_op_iterator *> {
913 explicit value_op_iterator(SDUse *U = nullptr)
914 : iterator_adaptor_base(U) {}
915
916 const SDValue &operator*() const { return I->get(); }
917 };
918
919 iterator_range<value_op_iterator> op_values() const {
920 return make_range(value_op_iterator(op_begin()),
921 value_op_iterator(op_end()));
922 }
923
924 SDVTList getVTList() const {
925 SDVTList X = { ValueList, NumValues };
926 return X;
927 }
928
929 /// If this node has a glue operand, return the node
930 /// to which the glue operand points. Otherwise return NULL.
931 SDNode *getGluedNode() const {
932 if (getNumOperands() != 0 &&
933 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
934 return getOperand(getNumOperands()-1).getNode();
935 return nullptr;
936 }
937
938 /// If this node has a glue value with a user, return
939 /// the user (there is at most one). Otherwise return NULL.
940 SDNode *getGluedUser() const {
941 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
942 if (UI.getUse().get().getValueType() == MVT::Glue)
943 return *UI;
944 return nullptr;
945 }
946
947 SDNodeFlags getFlags() const { return Flags; }
948 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
949
950 /// Clear any flags in this node that aren't also set in Flags.
951 /// If Flags is not in a defined state then this has no effect.
952 void intersectFlagsWith(const SDNodeFlags Flags);
953
954 /// Return the number of values defined/returned by this operator.
955 unsigned getNumValues() const { return NumValues; }
956
957 /// Return the type of a specified result.
958 EVT getValueType(unsigned ResNo) const {
959 assert(ResNo < NumValues && "Illegal result number!")((void)0);
960 return ValueList[ResNo];
961 }
962
963 /// Return the type of a specified result as a simple type.
964 MVT getSimpleValueType(unsigned ResNo) const {
965 return getValueType(ResNo).getSimpleVT();
966 }
967
968 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
969 ///
970 /// If the value type is a scalable vector type, the scalable property will
971 /// be set and the runtime size will be a positive integer multiple of the
972 /// base size.
973 TypeSize getValueSizeInBits(unsigned ResNo) const {
974 return getValueType(ResNo).getSizeInBits();
975 }
976
977 using value_iterator = const EVT *;
978
979 value_iterator value_begin() const { return ValueList; }
980 value_iterator value_end() const { return ValueList+NumValues; }
981 iterator_range<value_iterator> values() const {
982 return llvm::make_range(value_begin(), value_end());
983 }
984
985 /// Return the opcode of this operation for printing.
986 std::string getOperationName(const SelectionDAG *G = nullptr) const;
987 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
988 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
989 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
990 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
991 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
992
993 /// Print a SelectionDAG node and all children down to
994 /// the leaves. The given SelectionDAG allows target-specific nodes
995 /// to be printed in human-readable form. Unlike printr, this will
996 /// print the whole DAG, including children that appear multiple
997 /// times.
998 ///
999 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and children up to
1002 /// depth "depth." The given SelectionDAG allows target-specific
1003 /// nodes to be printed in human-readable form. Unlike printr, this
1004 /// will print children that appear multiple times wherever they are
1005 /// used.
1006 ///
1007 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1008 unsigned depth = 100) const;
1009
1010 /// Dump this node, for debugging.
1011 void dump() const;
1012
1013 /// Dump (recursively) this node and its use-def subgraph.
1014 void dumpr() const;
1015
1016 /// Dump this node, for debugging.
1017 /// The given SelectionDAG allows target-specific nodes to be printed
1018 /// in human-readable form.
1019 void dump(const SelectionDAG *G) const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 /// The given SelectionDAG allows target-specific nodes to be printed
1023 /// in human-readable form.
1024 void dumpr(const SelectionDAG *G) const;
1025
1026 /// printrFull to dbgs(). The given SelectionDAG allows
1027 /// target-specific nodes to be printed in human-readable form.
1028 /// Unlike dumpr, this will print the whole DAG, including children
1029 /// that appear multiple times.
1030 void dumprFull(const SelectionDAG *G = nullptr) const;
1031
1032 /// printrWithDepth to dbgs(). The given
1033 /// SelectionDAG allows target-specific nodes to be printed in
1034 /// human-readable form. Unlike dumpr, this will print children
1035 /// that appear multiple times wherever they are used.
1036 ///
1037 void dumprWithDepth(const SelectionDAG *G = nullptr,
1038 unsigned depth = 100) const;
1039
1040 /// Gather unique data for the node.
1041 void Profile(FoldingSetNodeID &ID) const;
1042
1043 /// This method should only be used by the SDUse class.
1044 void addUse(SDUse &U) { U.addToList(&UseList); }
1045
1046protected:
1047 static SDVTList getSDVTList(EVT VT) {
1048 SDVTList Ret = { getValueTypeList(VT), 1 };
1049 return Ret;
1050 }
1051
1052 /// Create an SDNode.
1053 ///
1054 /// SDNodes are created without any operands, and never own the operand
1055 /// storage. To add operands, see SelectionDAG::createOperands.
1056 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1057 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1058 IROrder(Order), debugLoc(std::move(dl)) {
1059 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1060 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((void)0);
1061 assert(NumValues == VTs.NumVTs &&((void)0)
1062 "NumValues wasn't wide enough for its operands!")((void)0);
1063 }
1064
1065 /// Release the operands and set this node to have zero operands.
1066 void DropOperands();
1067};
1068
1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
1080 DebugLoc DL;
1081 int IROrder = 0;
1082
1083public:
1084 SDLoc() = default;
1085 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1086 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1087 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1088 assert(Order >= 0 && "bad IROrder")((void)0);
1089 if (I)
1090 DL = I->getDebugLoc();
1091 }
1092
1093 unsigned getIROrder() const { return IROrder; }
1094 const DebugLoc &getDebugLoc() const { return DL; }
1095};
1096
1097// Define inline functions from the SDValue class.
1098
1099inline SDValue::SDValue(SDNode *node, unsigned resno)
1100 : Node(node), ResNo(resno) {
1101 // Explicitly check for !ResNo to avoid use-after-free, because there are
1102 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1103 // combines.
1104 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&((void)0)
1105 "Invalid result number for the given node!")((void)0);
1106 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((void)0);
1107}
1108
1109inline unsigned SDValue::getOpcode() const {
1110 return Node->getOpcode();
68
←
Called C++ object pointer is null
1111}
1112
1113inline EVT SDValue::getValueType() const {
1114 return Node->getValueType(ResNo);
1115}
1116
1117inline unsigned SDValue::getNumOperands() const {
1118 return Node->getNumOperands();
1119}
1120
1121inline const SDValue &SDValue::getOperand(unsigned i) const {
1122 return Node->getOperand(i);
1123}
1124
1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1126 return Node->getConstantOperandVal(i);
1127}
1128
1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1130 return Node->getConstantOperandAPInt(i);
1131}
1132
1133inline bool SDValue::isTargetOpcode() const {
1134 return Node->isTargetOpcode();
1135}
1136
1137inline bool SDValue::isTargetMemoryOpcode() const {
1138 return Node->isTargetMemoryOpcode();
1139}
1140
1141inline bool SDValue::isMachineOpcode() const {
1142 return Node->isMachineOpcode();
1143}
1144
1145inline unsigned SDValue::getMachineOpcode() const {
1146 return Node->getMachineOpcode();
1147}
1148
1149inline bool SDValue::isUndef() const {
1150 return Node->isUndef();
1151}
1152
1153inline bool SDValue::use_empty() const {
1154 return !Node->hasAnyUseOfValue(ResNo);
1155}
1156
1157inline bool SDValue::hasOneUse() const {
1158 return Node->hasNUsesOfValue(1, ResNo);
1159}
1160
1161inline const DebugLoc &SDValue::getDebugLoc() const {
1162 return Node->getDebugLoc();
1163}
1164
1165inline void SDValue::dump() const {
1166 return Node->dump();
1167}
1168
1169inline void SDValue::dump(const SelectionDAG *G) const {
1170 return Node->dump(G);
1171}
1172
1173inline void SDValue::dumpr() const {
1174 return Node->dumpr();
1175}
1176
1177inline void SDValue::dumpr(const SelectionDAG *G) const {
1178 return Node->dumpr(G);
1179}
1180
1181// Define inline functions from the SDUse class.
1182
1183inline void SDUse::set(const SDValue &V) {
1184 if (Val.getNode()) removeFromList();
1185 Val = V;
1186 if (V.getNode()) V.getNode()->addUse(*this);
1187}
1188
1189inline void SDUse::setInitial(const SDValue &V) {
1190 Val = V;
1191 V.getNode()->addUse(*this);
1192}
1193
1194inline void SDUse::setNode(SDNode *N) {
1195 if (Val.getNode()) removeFromList();
1196 Val.setNode(N);
1197 if (N) N->addUse(*this);
1198}
1199
1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand. This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
1205 SDUse Op;
1206
1207public:
1208 explicit HandleSDNode(SDValue X)
1209 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1210 // HandleSDNodes are never inserted into the DAG, so they won't be
1211 // auto-numbered. Use ID 65535 as a sentinel.
1212 PersistentId = 0xffff;
1213
1214 // Manually set up the operand list. This node type is special in that it's
1215 // always stack allocated and SelectionDAG does not manage its operands.
1216 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1217 // be so special.
1218 Op.setUser(this);
1219 Op.setInitial(X);
1220 NumOperands = 1;
1221 OperandList = &Op;
1222 }
1223 ~HandleSDNode();
1224
1225 const SDValue &getValue() const { return Op; }
1226};
1227
1228class AddrSpaceCastSDNode : public SDNode {
1229private:
1230 unsigned SrcAddrSpace;
1231 unsigned DestAddrSpace;
1232
1233public:
1234 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1235 unsigned SrcAS, unsigned DestAS);
1236
1237 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1238 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1239
1240 static bool classof(const SDNode *N) {
1241 return N->getOpcode() == ISD::ADDRSPACECAST;
1242 }
1243};
1244
1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
1248 // VT of in-memory value.
1249 EVT MemoryVT;
1250
1251protected:
1252 /// Memory reference information.
1253 MachineMemOperand *MMO;
1254
1255public:
1256 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1257 EVT memvt, MachineMemOperand *MMO);
1258
1259 bool readMem() const { return MMO->isLoad(); }
1260 bool writeMem() const { return MMO->isStore(); }
1261
1262 /// Returns alignment and volatility of the memory access
1263 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1264 Align getAlign() const { return MMO->getAlign(); }
1265 // FIXME: Remove once transition to getAlign is over.
1266 unsigned getAlignment() const { return MMO->getAlign().value(); }
1267
1268 /// Return the SubclassData value, without HasDebugValue. This contains an
1269 /// encoding of the volatile flag, as well as bits used by subclasses. This
1270 /// function should only be used to compute a FoldingSetNodeID value.
1271 /// The HasDebugValue bit is masked out because CSE map needs to match
1272 /// nodes with debug info with nodes without debug info. Same is about
1273 /// isDivergent bit.
1274 unsigned getRawSubclassData() const {
1275 uint16_t Data;
1276 union {
1277 char RawSDNodeBits[sizeof(uint16_t)];
1278 SDNodeBitfields SDNodeBits;
1279 };
1280 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1281 SDNodeBits.HasDebugValue = 0;
1282 SDNodeBits.IsDivergent = false;
1283 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1284 return Data;
1285 }
1286
1287 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1288 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1289 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1290 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1291
1292 // Returns the offset from the location of the access.
1293 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1294
1295 /// Returns the AA info that describes the dereference.
1296 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1297
1298 /// Returns the Ranges that describes the dereference.
1299 const MDNode *getRanges() const { return MMO->getRanges(); }
1300
1301 /// Returns the synchronization scope ID for this memory operation.
1302 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1303
1304 /// Return the atomic ordering requirements for this memory operation. For
1305 /// cmpxchg atomic operations, return the atomic ordering requirements when
1306 /// store occurs.
1307 AtomicOrdering getSuccessOrdering() const {
1308 return MMO->getSuccessOrdering();
1309 }
1310
1311 /// Return a single atomic ordering that is at least as strong as both the
1312 /// success and failure orderings for an atomic operation. (For operations
1313 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1314 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1315
1316 /// Return true if the memory operation ordering is Unordered or higher.
1317 bool isAtomic() const { return MMO->isAtomic(); }
1318
1319 /// Returns true if the memory operation doesn't imply any ordering
1320 /// constraints on surrounding memory operations beyond the normal memory
1321 /// aliasing rules.
1322 bool isUnordered() const { return MMO->isUnordered(); }
1323
1324 /// Returns true if the memory operation is neither atomic or volatile.
1325 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1326
1327 /// Return the type of the in-memory value.
1328 EVT getMemoryVT() const { return MemoryVT; }
1329
1330 /// Return a MachineMemOperand object describing the memory
1331 /// reference performed by operation.
1332 MachineMemOperand *getMemOperand() const { return MMO; }
1333
1334 const MachinePointerInfo &getPointerInfo() const {
1335 return MMO->getPointerInfo();
1336 }
1337
1338 /// Return the address space for the associated pointer
1339 unsigned getAddressSpace() const {
1340 return getPointerInfo().getAddrSpace();
1341 }
1342
1343 /// Update this MemSDNode's MachineMemOperand information
1344 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1345 /// This must only be used when the new alignment applies to all users of
1346 /// this MachineMemOperand.
1347 void refineAlignment(const MachineMemOperand *NewMMO) {
1348 MMO->refineAlignment(NewMMO);
1349 }
1350
1351 const SDValue &getChain() const { return getOperand(0); }
1352
1353 const SDValue &getBasePtr() const {
1354 switch (getOpcode()) {
1355 case ISD::STORE:
1356 case ISD::MSTORE:
1357 return getOperand(2);
1358 case ISD::MGATHER:
1359 case ISD::MSCATTER:
1360 return getOperand(3);
1361 default:
1362 return getOperand(1);
1363 }
1364 }
1365
1366 // Methods to support isa and dyn_cast
1367 static bool classof(const SDNode *N) {
1368 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1369 // with either an intrinsic or a target opcode.
1370 switch (N->getOpcode()) {
1371 case ISD::LOAD:
1372 case ISD::STORE:
1373 case ISD::PREFETCH:
1374 case ISD::ATOMIC_CMP_SWAP:
1375 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1376 case ISD::ATOMIC_SWAP:
1377 case ISD::ATOMIC_LOAD_ADD:
1378 case ISD::ATOMIC_LOAD_SUB:
1379 case ISD::ATOMIC_LOAD_AND:
1380 case ISD::ATOMIC_LOAD_CLR:
1381 case ISD::ATOMIC_LOAD_OR:
1382 case ISD::ATOMIC_LOAD_XOR:
1383 case ISD::ATOMIC_LOAD_NAND:
1384 case ISD::ATOMIC_LOAD_MIN:
1385 case ISD::ATOMIC_LOAD_MAX:
1386 case ISD::ATOMIC_LOAD_UMIN:
1387 case ISD::ATOMIC_LOAD_UMAX:
1388 case ISD::ATOMIC_LOAD_FADD:
1389 case ISD::ATOMIC_LOAD_FSUB:
1390 case ISD::ATOMIC_LOAD:
1391 case ISD::ATOMIC_STORE:
1392 case ISD::MLOAD:
1393 case ISD::MSTORE:
1394 case ISD::MGATHER:
1395 case ISD::MSCATTER:
1396 return true;
1397 default:
1398 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1399 }
1400 }
1401};
1402
1403/// This is an SDNode representing atomic operations.
1404class AtomicSDNode : public MemSDNode {
1405public:
1406 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1407 EVT MemVT, MachineMemOperand *MMO)
1408 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1409 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||((void)0)
1410 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")((void)0);
1411 }
1412
1413 const SDValue &getBasePtr() const { return getOperand(1); }
1414 const SDValue &getVal() const { return getOperand(2); }
1415
1416 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1417 /// otherwise.
1418 bool isCompareAndSwap() const {
1419 unsigned Op = getOpcode();
1420 return Op == ISD::ATOMIC_CMP_SWAP ||
1421 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1422 }
1423
1424 /// For cmpxchg atomic operations, return the atomic ordering requirements
1425 /// when store does not occur.
1426 AtomicOrdering getFailureOrdering() const {
1427 assert(isCompareAndSwap() && "Must be cmpxchg operation")((void)0);
1428 return MMO->getFailureOrdering();
1429 }
1430
1431 // Methods to support isa and dyn_cast
1432 static bool classof(const SDNode *N) {
1433 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1434 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1435 N->getOpcode() == ISD::ATOMIC_SWAP ||
1436 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1437 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1438 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1439 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1440 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1441 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1442 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1443 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1444 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1445 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1446 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1447 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1448 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1449 N->getOpcode() == ISD::ATOMIC_LOAD ||
1450 N->getOpcode() == ISD::ATOMIC_STORE;
1451 }
1452};
1453
1454/// This SDNode is used for target intrinsics that touch
1455/// memory and need an associated MachineMemOperand. Its opcode may be
1456/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1457/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1458class MemIntrinsicSDNode : public MemSDNode {
1459public:
1460 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1461 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1462 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1463 SDNodeBits.IsMemIntrinsic = true;
1464 }
1465
1466 // Methods to support isa and dyn_cast
1467 static bool classof(const SDNode *N) {
1468 // We lower some target intrinsics to their target opcode
1469 // early a node with a target opcode can be of this class
1470 return N->isMemIntrinsic() ||
1471 N->getOpcode() == ISD::PREFETCH ||
1472 N->isTargetMemoryOpcode();
1473 }
1474};
1475
1476/// This SDNode is used to implement the code generator
1477/// support for the llvm IR shufflevector instruction. It combines elements
1478/// from two input vectors into a new input vector, with the selection and
1479/// ordering of elements determined by an array of integers, referred to as
1480/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1481/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1482/// An index of -1 is treated as undef, such that the code generator may put
1483/// any value in the corresponding element of the result.
1484class ShuffleVectorSDNode : public SDNode {
1485 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1486 // is freed when the SelectionDAG object is destroyed.
1487 const int *Mask;
1488
1489protected:
1490 friend class SelectionDAG;
1491
1492 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1493 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1494
1495public:
1496 ArrayRef<int> getMask() const {
1497 EVT VT = getValueType(0);
1498 return makeArrayRef(Mask, VT.getVectorNumElements());
1499 }
1500
1501 int getMaskElt(unsigned Idx) const {
1502 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")((void)0);
1503 return Mask[Idx];
1504 }
1505
1506 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1507
1508 int getSplatIndex() const {
1509 assert(isSplat() && "Cannot get splat index for non-splat!")((void)0);
1510 EVT VT = getValueType(0);
1511 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1512 if (Mask[i] >= 0)
1513 return Mask[i];
1514
1515 // We can choose any index value here and be correct because all elements
1516 // are undefined. Return 0 for better potential for callers to simplify.
1517 return 0;
1518 }
1519
1520 static bool isSplatMask(const int *Mask, EVT VT);
1521
1522 /// Change values in a shuffle permute mask assuming
1523 /// the two vector operands have swapped position.
1524 static void commuteMask(MutableArrayRef<int> Mask) {
1525 unsigned NumElems = Mask.size();
1526 for (unsigned i = 0; i != NumElems; ++i) {
1527 int idx = Mask[i];
1528 if (idx < 0)
1529 continue;
1530 else if (idx < (int)NumElems)
1531 Mask[i] = idx + NumElems;
1532 else
1533 Mask[i] = idx - NumElems;
1534 }
1535 }
1536
1537 static bool classof(const SDNode *N) {
1538 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1539 }
1540};
1541
1542class ConstantSDNode : public SDNode {
1543 friend class SelectionDAG;
1544
1545 const ConstantInt *Value;
1546
1547 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1548 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1549 getSDVTList(VT)),
1550 Value(val) {
1551 ConstantSDNodeBits.IsOpaque = isOpaque;
1552 }
1553
1554public:
1555 const ConstantInt *getConstantIntValue() const { return Value; }
1556 const APInt &getAPIntValue() const { return Value->getValue(); }
1557 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1558 int64_t getSExtValue() const { return Value->getSExtValue(); }
1559 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX0xffffffffffffffffULL) {
1560 return Value->getLimitedValue(Limit);
1561 }
1562 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1563 Align getAlignValue() const { return Value->getAlignValue(); }
1564
1565 bool isOne() const { return Value->isOne(); }
1566 bool isNullValue() const { return Value->isZero(); }
1567 bool isAllOnesValue() const { return Value->isMinusOne(); }
1568 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1569 bool isMinSignedValue() const { return Value->isMinValue(true); }
1570
1571 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1572
1573 static bool classof(const SDNode *N) {
1574 return N->getOpcode() == ISD::Constant ||
1575 N->getOpcode() == ISD::TargetConstant;
1576 }
1577};
1578
1579uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1580 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1581}
1582
1583const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1584 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1585}
1586
1587class ConstantFPSDNode : public SDNode {
1588 friend class SelectionDAG;
1589
1590 const ConstantFP *Value;
1591
1592 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1593 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1594 DebugLoc(), getSDVTList(VT)),
1595 Value(val) {}
1596
1597public:
1598 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1599 const ConstantFP *getConstantFPValue() const { return Value; }
1600
1601 /// Return true if the value is positive or negative zero.
1602 bool isZero() const { return Value->isZero(); }
1603
1604 /// Return true if the value is a NaN.
1605 bool isNaN() const { return Value->isNaN(); }
1606
1607 /// Return true if the value is an infinity
1608 bool isInfinity() const { return Value->isInfinity(); }
1609
1610 /// Return true if the value is negative.
1611 bool isNegative() const { return Value->isNegative(); }
1612
1613 /// We don't rely on operator== working on double values, as
1614 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1615 /// As such, this method can be used to do an exact bit-for-bit comparison of
1616 /// two floating point values.
1617
1618 /// We leave the version with the double argument here because it's just so
1619 /// convenient to write "2.0" and the like. Without this function we'd
1620 /// have to duplicate its logic everywhere it's called.
1621 bool isExactlyValue(double V) const {
1622 return Value->getValueAPF().isExactlyValue(V);
1623 }
1624 bool isExactlyValue(const APFloat& V) const;
1625
1626 static bool isValueValidForType(EVT VT, const APFloat& Val);
1627
1628 static bool classof(const SDNode *N) {
1629 return N->getOpcode() == ISD::ConstantFP ||
1630 N->getOpcode() == ISD::TargetConstantFP;
1631 }
1632};
1633
1634/// Returns true if \p V is a constant integer zero.
1635bool isNullConstant(SDValue V);
1636
1637/// Returns true if \p V is an FP constant with a value of positive zero.
1638bool isNullFPConstant(SDValue V);
1639
1640/// Returns true if \p V is an integer constant with all bits set.
1641bool isAllOnesConstant(SDValue V);
1642
1643/// Returns true if \p V is a constant integer one.
1644bool isOneConstant(SDValue V);
1645
1646/// Return the non-bitcasted source operand of \p V if it exists.
1647/// If \p V is not a bitcasted value, it is returned as-is.
1648SDValue peekThroughBitcasts(SDValue V);
1649
1650/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1651/// If \p V is not a bitcasted one-use value, it is returned as-is.
1652SDValue peekThroughOneUseBitcasts(SDValue V);
1653
1654/// Return the non-extracted vector source operand of \p V if it exists.
1655/// If \p V is not an extracted subvector, it is returned as-is.
1656SDValue peekThroughExtractSubvectors(SDValue V);
1657
1658/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1659/// constant is canonicalized to be operand 1.
1660bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1661
1662/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1663ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1664 bool AllowTruncation = false);
1665
1666/// Returns the SDNode if it is a demanded constant splat BuildVector or
1667/// constant int.
1668ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1669 bool AllowUndefs = false,
1670 bool AllowTruncation = false);
1671
1672/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1673ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1674
1675/// Returns the SDNode if it is a demanded constant splat BuildVector or
1676/// constant float.
1677ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1678 bool AllowUndefs = false);
1679
1680/// Return true if the value is a constant 0 integer or a splatted vector of
1681/// a constant 0 integer (with no undefs by default).
1682/// Build vector implicit truncation is not an issue for null values.
1683bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1684
1685/// Return true if the value is a constant 1 integer or a splatted vector of a
1686/// constant 1 integer (with no undefs).
1687/// Does not permit build vector implicit truncation.
1688bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1689
1690/// Return true if the value is a constant -1 integer or a splatted vector of a
1691/// constant -1 integer (with no undefs).
1692/// Does not permit build vector implicit truncation.
1693bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1694
1695/// Return true if \p V is either a integer or FP constant.
1696inline bool isIntOrFPConstant(SDValue V) {
1697 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1698}
1699
1700class GlobalAddressSDNode : public SDNode {
1701 friend class SelectionDAG;
1702
1703 const GlobalValue *TheGlobal;
1704 int64_t Offset;
1705 unsigned TargetFlags;
1706
1707 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1708 const GlobalValue *GA, EVT VT, int64_t o,
1709 unsigned TF);
1710
1711public:
1712 const GlobalValue *getGlobal() const { return TheGlobal; }
1713 int64_t getOffset() const { return Offset; }
1714 unsigned getTargetFlags() const { return TargetFlags; }
1715 // Return the address space this GlobalAddress belongs to.
1716 unsigned getAddressSpace() const;
1717
1718 static bool classof(const SDNode *N) {
1719 return N->getOpcode() == ISD::GlobalAddress ||
1720 N->getOpcode() == ISD::TargetGlobalAddress ||
1721 N->getOpcode() == ISD::GlobalTLSAddress ||
1722 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1723 }
1724};
1725
1726class FrameIndexSDNode : public SDNode {
1727 friend class SelectionDAG;
1728
1729 int FI;
1730
1731 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1732 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1733 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1734 }
1735
1736public:
1737 int getIndex() const { return FI; }
1738
1739 static bool classof(const SDNode *N) {
1740 return N->getOpcode() == ISD::FrameIndex ||
1741 N->getOpcode() == ISD::TargetFrameIndex;
1742 }
1743};
1744
1745/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1746/// the offet and size that are started/ended in the underlying FrameIndex.
1747class LifetimeSDNode : public SDNode {
1748 friend class SelectionDAG;
1749 int64_t Size;
1750 int64_t Offset; // -1 if offset is unknown.
1751
1752 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1753 SDVTList VTs, int64_t Size, int64_t Offset)
1754 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1755public:
1756 int64_t getFrameIndex() const {
1757 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1758 }
1759
1760 bool hasOffset() const { return Offset >= 0; }
1761 int64_t getOffset() const {
1762 assert(hasOffset() && "offset is unknown")((void)0);
1763 return Offset;
1764 }
1765 int64_t getSize() const {
1766 assert(hasOffset() && "offset is unknown")((void)0);
1767 return Size;
1768 }
1769
1770 // Methods to support isa and dyn_cast
1771 static bool classof(const SDNode *N) {
1772 return N->getOpcode() == ISD::LIFETIME_START ||
1773 N->getOpcode() == ISD::LIFETIME_END;
1774 }
1775};
1776
1777/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1778/// the index of the basic block being probed. A pseudo probe serves as a place
1779/// holder and will be removed at the end of compilation. It does not have any
1780/// operand because we do not want the instruction selection to deal with any.
1781class PseudoProbeSDNode : public SDNode {
1782 friend class SelectionDAG;
1783 uint64_t Guid;
1784 uint64_t Index;
1785 uint32_t Attributes;
1786
1787 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1788 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1789 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1790 Attributes(Attr) {}
1791
1792public:
1793 uint64_t getGuid() const { return Guid; }
1794 uint64_t getIndex() const { return Index; }
1795 uint32_t getAttributes() const { return Attributes; }
1796
1797 // Methods to support isa and dyn_cast
1798 static bool classof(const SDNode *N) {
1799 return N->getOpcode() == ISD::PSEUDO_PROBE;
1800 }
1801};
1802
1803class JumpTableSDNode : public SDNode {
1804 friend class SelectionDAG;
1805
1806 int JTI;
1807 unsigned TargetFlags;
1808
1809 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1810 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1811 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1812 }
1813
1814public:
1815 int getIndex() const { return JTI; }
1816 unsigned getTargetFlags() const { return TargetFlags; }
1817
1818 static bool classof(const SDNode *N) {
1819 return N->getOpcode() == ISD::JumpTable ||
1820 N->getOpcode() == ISD::TargetJumpTable;
1821 }
1822};
1823
1824class ConstantPoolSDNode : public SDNode {
1825 friend class SelectionDAG;
1826
1827 union {
1828 const Constant *ConstVal;
1829 MachineConstantPoolValue *MachineCPVal;
1830 } Val;
1831 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1832 Align Alignment; // Minimum alignment requirement of CP.
1833 unsigned TargetFlags;
1834
1835 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1836 Align Alignment, unsigned TF)
1837 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1838 DebugLoc(), getSDVTList(VT)),
1839 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1840 assert(Offset >= 0 && "Offset is too large")((void)0);
1841 Val.ConstVal = c;
1842 }
1843
1844 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1845 Align Alignment, unsigned TF)
1846 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1847 DebugLoc(), getSDVTList(VT)),
1848 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1849 assert(Offset >= 0 && "Offset is too large")((void)0);
1850 Val.MachineCPVal = v;
1851 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1852 }
1853
1854public:
1855 bool isMachineConstantPoolEntry() const {
1856 return Offset < 0;
1857 }
1858
1859 const Constant *getConstVal() const {
1860 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")((void)0);
1861 return Val.ConstVal;
1862 }
1863
1864 MachineConstantPoolValue *getMachineCPVal() const {
1865 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")((void)0);
1866 return Val.MachineCPVal;
1867 }
1868
1869 int getOffset() const {
1870 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1871 }
1872
1873 // Return the alignment of this constant pool object, which is either 0 (for
1874 // default alignment) or the desired value.
1875 Align getAlign() const { return Alignment; }
1876 unsigned getTargetFlags() const { return TargetFlags; }
1877
1878 Type *getType() const;
1879
1880 static bool classof(const SDNode *N) {
1881 return N->getOpcode() == ISD::ConstantPool ||
1882 N->getOpcode() == ISD::TargetConstantPool;
1883 }
1884};
1885
1886/// Completely target-dependent object reference.
1887class TargetIndexSDNode : public SDNode {
1888 friend class SelectionDAG;
1889
1890 unsigned TargetFlags;
1891 int Index;
1892 int64_t Offset;
1893
1894public:
1895 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1896 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1897 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1898
1899 unsigned getTargetFlags() const { return TargetFlags; }
1900 int getIndex() const { return Index; }
1901 int64_t getOffset() const { return Offset; }
1902
1903 static bool classof(const SDNode *N) {
1904 return N->getOpcode() == ISD::TargetIndex;
1905 }
1906};
1907
1908class BasicBlockSDNode : public SDNode {
1909 friend class SelectionDAG;
1910
1911 MachineBasicBlock *MBB;
1912
1913 /// Debug info is meaningful and potentially useful here, but we create
1914 /// blocks out of order when they're jumped to, which makes it a bit
1915 /// harder. Let's see if we need it first.
1916 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1917 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1918 {}
1919
1920public:
1921 MachineBasicBlock *getBasicBlock() const { return MBB; }
1922
1923 static bool classof(const SDNode *N) {
1924 return N->getOpcode() == ISD::BasicBlock;
1925 }
1926};
1927
1928/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1929class BuildVectorSDNode : public SDNode {
1930public:
1931 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1932 explicit BuildVectorSDNode() = delete;
1933
1934 /// Check if this is a constant splat, and if so, find the
1935 /// smallest element size that splats the vector. If MinSplatBits is
1936 /// nonzero, the element size must be at least that large. Note that the
1937 /// splat element may be the entire vector (i.e., a one element vector).
1938 /// Returns the splat element value in SplatValue. Any undefined bits in
1939 /// that value are zero, and the corresponding bits in the SplatUndef mask
1940 /// are set. The SplatBitSize value is set to the splat element size in
1941 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1942 /// undefined. isBigEndian describes the endianness of the target.
1943 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1944 unsigned &SplatBitSize, bool &HasAnyUndefs,
1945 unsigned MinSplatBits = 0,
1946 bool isBigEndian = false) const;
1947
1948 /// Returns the demanded splatted value or a null value if this is not a
1949 /// splat.
1950 ///
1951 /// The DemandedElts mask indicates the elements that must be in the splat.
1952 /// If passed a non-null UndefElements bitvector, it will resize it to match
1953 /// the vector width and set the bits where elements are undef.
1954 SDValue getSplatValue(const APInt &DemandedElts,
1955 BitVector *UndefElements = nullptr) const;
1956
1957 /// Returns the splatted value or a null value if this is not a splat.
1958 ///
1959 /// If passed a non-null UndefElements bitvector, it will resize it to match
1960 /// the vector width and set the bits where elements are undef.
1961 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1962
1963 /// Find the shortest repeating sequence of values in the build vector.
1964 ///
1965 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1966 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1967 ///
1968 /// Currently this must be a power-of-2 build vector.
1969 /// The DemandedElts mask indicates the elements that must be present,
1970 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1971 /// non-null UndefElements bitvector, it will resize it to match the original
1972 /// vector width and set the bits where elements are undef. If result is
1973 /// false, Sequence will be empty.
1974 bool getRepeatedSequence(const APInt &DemandedElts,
1975 SmallVectorImpl<SDValue> &Sequence,
1976 BitVector *UndefElements = nullptr) const;
1977
1978 /// Find the shortest repeating sequence of values in the build vector.
1979 ///
1980 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1981 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1982 ///
1983 /// Currently this must be a power-of-2 build vector.
1984 /// If passed a non-null UndefElements bitvector, it will resize it to match
1985 /// the original vector width and set the bits where elements are undef.
1986 /// If result is false, Sequence will be empty.
1987 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
1988 BitVector *UndefElements = nullptr) const;
1989
1990 /// Returns the demanded splatted constant or null if this is not a constant
1991 /// splat.
1992 ///
1993 /// The DemandedElts mask indicates the elements that must be in the splat.
1994 /// If passed a non-null UndefElements bitvector, it will resize it to match
1995 /// the vector width and set the bits where elements are undef.
1996 ConstantSDNode *
1997 getConstantSplatNode(const APInt &DemandedElts,
1998 BitVector *UndefElements = nullptr) const;
1999
2000 /// Returns the splatted constant or null if this is not a constant
2001 /// splat.
2002 ///
2003 /// If passed a non-null UndefElements bitvector, it will resize it to match
2004 /// the vector width and set the bits where elements are undef.
2005 ConstantSDNode *
2006 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2007
2008 /// Returns the demanded splatted constant FP or null if this is not a
2009 /// constant FP splat.
2010 ///
2011 /// The DemandedElts mask indicates the elements that must be in the splat.
2012 /// If passed a non-null UndefElements bitvector, it will resize it to match
2013 /// the vector width and set the bits where elements are undef.
2014 ConstantFPSDNode *
2015 getConstantFPSplatNode(const APInt &DemandedElts,
2016 BitVector *UndefElements = nullptr) const;
2017
2018 /// Returns the splatted constant FP or null if this is not a constant
2019 /// FP splat.
2020 ///
2021 /// If passed a non-null UndefElements bitvector, it will resize it to match
2022 /// the vector width and set the bits where elements are undef.
2023 ConstantFPSDNode *
2024 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2025
2026 /// If this is a constant FP splat and the splatted constant FP is an
2027 /// exact power or 2, return the log base 2 integer value. Otherwise,
2028 /// return -1.
2029 ///
2030 /// The BitWidth specifies the necessary bit precision.
2031 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2032 uint32_t BitWidth) const;
2033
2034 bool isConstant() const;
2035
2036 static bool classof(const SDNode *N) {
2037 return N->getOpcode() == ISD::BUILD_VECTOR;
2038 }
2039};
2040
2041/// An SDNode that holds an arbitrary LLVM IR Value. This is
2042/// used when the SelectionDAG needs to make a simple reference to something
2043/// in the LLVM IR representation.
2044///
2045class SrcValueSDNode : public SDNode {
2046 friend class SelectionDAG;
2047
2048 const Value *V;
2049
2050 /// Create a SrcValue for a general value.
2051 explicit SrcValueSDNode(const Value *v)
2052 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2053
2054public:
2055 /// Return the contained Value.
2056 const Value *getValue() const { return V; }
2057
2058 static bool classof(const SDNode *N) {
2059 return N->getOpcode() == ISD::SRCVALUE;
2060 }
2061};
2062
2063class MDNodeSDNode : public SDNode {
2064 friend class SelectionDAG;
2065
2066 const MDNode *MD;
2067
2068 explicit MDNodeSDNode(const MDNode *md)
2069 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2070 {}
2071
2072public:
2073 const MDNode *getMD() const { return MD; }
2074
2075 static bool classof(const SDNode *N) {
2076 return N->getOpcode() == ISD::MDNODE_SDNODE;
2077 }
2078};
2079
2080class RegisterSDNode : public SDNode {
2081 friend class SelectionDAG;
2082
2083 Register Reg;
2084
2085 RegisterSDNode(Register reg, EVT VT)
2086 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2087
2088public:
2089 Register getReg() const { return Reg; }
2090
2091 static bool classof(const SDNode *N) {
2092 return N->getOpcode() == ISD::Register;
2093 }
2094};
2095
2096class RegisterMaskSDNode : public SDNode {
2097 friend class SelectionDAG;
2098
2099 // The memory for RegMask is not owned by the node.
2100 const uint32_t *RegMask;
2101
2102 RegisterMaskSDNode(const uint32_t *mask)
2103 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2104 RegMask(mask) {}
2105
2106public:
2107 const uint32_t *getRegMask() const { return RegMask; }
2108
2109 static bool classof(const SDNode *N) {
2110 return N->getOpcode() == ISD::RegisterMask;
2111 }
2112};
2113
2114class BlockAddressSDNode : public SDNode {
2115 friend class SelectionDAG;
2116
2117 const BlockAddress *BA;
2118 int64_t Offset;
2119 unsigned TargetFlags;
2120
2121 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2122 int64_t o, unsigned Flags)
2123 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2124 BA(ba), Offset(o), TargetFlags(Flags) {}
2125
2126public:
2127 const BlockAddress *getBlockAddress() const { return BA; }
2128 int64_t getOffset() const { return Offset; }
2129 unsigned getTargetFlags() const { return TargetFlags; }
2130
2131 static bool classof(const SDNode *N) {
2132 return N->getOpcode() == ISD::BlockAddress ||
2133 N->getOpcode() == ISD::TargetBlockAddress;
2134 }
2135};
2136
2137class LabelSDNode : public SDNode {
2138 friend class SelectionDAG;
2139
2140 MCSymbol *Label;
2141
2142 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2143 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2144 assert(LabelSDNode::classof(this) && "not a label opcode")((void)0);
2145 }
2146
2147public:
2148 MCSymbol *getLabel() const { return Label; }
2149
2150 static bool classof(const SDNode *N) {
2151 return N->getOpcode() == ISD::EH_LABEL ||
2152 N->getOpcode() == ISD::ANNOTATION_LABEL;
2153 }
2154};
2155
2156class ExternalSymbolSDNode : public SDNode {
2157 friend class SelectionDAG;
2158
2159 const char *Symbol;
2160 unsigned TargetFlags;
2161
2162 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2163 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2164 DebugLoc(), getSDVTList(VT)),
2165 Symbol(Sym), TargetFlags(TF) {}
2166
2167public:
2168 const char *getSymbol() const { return Symbol; }
2169 unsigned getTargetFlags() const { return TargetFlags; }
2170
2171 static bool classof(const SDNode *N) {
2172 return N->getOpcode() == ISD::ExternalSymbol ||
2173 N->getOpcode() == ISD::TargetExternalSymbol;
2174 }
2175};
2176
2177class MCSymbolSDNode : public SDNode {
2178 friend class SelectionDAG;
2179
2180 MCSymbol *Symbol;
2181
2182 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2183 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2184
2185public:
2186 MCSymbol *getMCSymbol() const { return Symbol; }
2187
2188 static bool classof(const SDNode *N) {
2189 return N->getOpcode() == ISD::MCSymbol;
2190 }
2191};
2192
2193class CondCodeSDNode : public SDNode {
2194 friend class SelectionDAG;
2195
2196 ISD::CondCode Condition;
2197
2198 explicit CondCodeSDNode(ISD::CondCode Cond)
2199 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2200 Condition(Cond) {}
2201
2202public:
2203 ISD::CondCode get() const { return Condition; }
2204
2205 static bool classof(const SDNode *N) {
2206 return N->getOpcode() == ISD::CONDCODE;
2207 }
2208};
2209
2210/// This class is used to represent EVT's, which are used
2211/// to parameterize some operations.
2212class VTSDNode : public SDNode {
2213 friend class SelectionDAG;
2214
2215 EVT ValueType;
2216
2217 explicit VTSDNode(EVT VT)
2218 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2219 ValueType(VT) {}
2220
2221public:
2222 EVT getVT() const { return ValueType; }
2223
2224 static bool classof(const SDNode *N) {
2225 return N->getOpcode() == ISD::VALUETYPE;
2226 }
2227};
2228
2229/// Base class for LoadSDNode and StoreSDNode
2230class LSBaseSDNode : public MemSDNode {
2231public:
2232 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2233 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2234 MachineMemOperand *MMO)
2235 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2236 LSBaseSDNodeBits.AddressingMode = AM;
2237 assert(getAddressingMode() == AM && "Value truncated")((void)0);
2238 }
2239
2240 const SDValue &getOffset() const {
2241 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2242 }
2243
2244 /// Return the addressing mode for this load or store:
2245 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2246 ISD::MemIndexedMode getAddressingMode() const {
2247 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2248 }
2249
2250 /// Return true if this is a pre/post inc/dec load/store.
2251 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2252
2253 /// Return true if this is NOT a pre/post inc/dec load/store.
2254 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2255
2256 static bool classof(const SDNode *N) {
2257 return N->getOpcode() == ISD::LOAD ||
2258 N->getOpcode() == ISD::STORE;
2259 }
2260};
2261
2262/// This class is used to represent ISD::LOAD nodes.
2263class LoadSDNode : public LSBaseSDNode {
2264 friend class SelectionDAG;
2265
2266 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2267 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2268 MachineMemOperand *MMO)
2269 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2270 LoadSDNodeBits.ExtTy = ETy;
2271 assert(readMem() && "Load MachineMemOperand is not a load!")((void)0);
2272 assert(!writeMem() && "Load MachineMemOperand is a store!")((void)0);
2273 }
2274
2275public:
2276 /// Return whether this is a plain node,
2277 /// or one of the varieties of value-extending loads.
2278 ISD::LoadExtType getExtensionType() const {
2279 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2280 }
2281
2282 const SDValue &getBasePtr() const { return getOperand(1); }
2283 const SDValue &getOffset() const { return getOperand(2); }
2284
2285 static bool classof(const SDNode *N) {
2286 return N->getOpcode() == ISD::LOAD;
2287 }
2288};
2289
2290/// This class is used to represent ISD::STORE nodes.
2291class StoreSDNode : public LSBaseSDNode {
2292 friend class SelectionDAG;
2293
2294 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2295 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2296 MachineMemOperand *MMO)
2297 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2298 StoreSDNodeBits.IsTruncating = isTrunc;
2299 assert(!readMem() && "Store MachineMemOperand is a load!")((void)0);
2300 assert(writeMem() && "Store MachineMemOperand is not a store!")((void)0);
2301 }
2302
2303public:
2304 /// Return true if the op does a truncation before store.
2305 /// For integers this is the same as doing a TRUNCATE and storing the result.
2306 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2307 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2308 void setTruncatingStore(bool Truncating) {
2309 StoreSDNodeBits.IsTruncating = Truncating;
2310 }
2311
2312 const SDValue &getValue() const { return getOperand(1); }
2313 const SDValue &getBasePtr() const { return getOperand(2); }
2314 const SDValue &getOffset() const { return getOperand(3); }
2315
2316 static bool classof(const SDNode *N) {
2317 return N->getOpcode() == ISD::STORE;
2318 }
2319};
2320
2321/// This base class is used to represent MLOAD and MSTORE nodes
2322class MaskedLoadStoreSDNode : public MemSDNode {
2323public:
2324 friend class SelectionDAG;
2325
2326 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2327 const DebugLoc &dl, SDVTList VTs,
2328 ISD::MemIndexedMode AM, EVT MemVT,
2329 MachineMemOperand *MMO)
2330 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2331 LSBaseSDNodeBits.AddressingMode = AM;
2332 assert(getAddressingMode() == AM && "Value truncated")((void)0);
2333 }
2334
2335 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2336 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2337 // Mask is a vector of i1 elements
2338 const SDValue &getOffset() const {
2339 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2340 }
2341 const SDValue &getMask() const {
2342 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2343 }
2344
2345 /// Return the addressing mode for this load or store:
2346 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2347 ISD::MemIndexedMode getAddressingMode() const {
2348 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2349 }
2350
2351 /// Return true if this is a pre/post inc/dec load/store.
2352 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2353
2354 /// Return true if this is NOT a pre/post inc/dec load/store.
2355 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2356
2357 static bool classof(const SDNode *N) {
2358 return N->getOpcode() == ISD::MLOAD ||
2359 N->getOpcode() == ISD::MSTORE;
2360 }
2361};
2362
2363/// This class is used to represent an MLOAD node
2364class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2365public:
2366 friend class SelectionDAG;
2367
2368 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2369 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2370 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2371 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2372 LoadSDNodeBits.ExtTy = ETy;
2373 LoadSDNodeBits.IsExpanding = IsExpanding;
2374 }
2375
2376 ISD::LoadExtType getExtensionType() const {
2377 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2378 }
2379
2380 const SDValue &getBasePtr() const { return getOperand(1); }
2381 const SDValue &getOffset() const { return getOperand(2); }
2382 const SDValue &getMask() const { return getOperand(3); }
2383 const SDValue &getPassThru() const { return getOperand(4); }
2384
2385 static bool classof(const SDNode *N) {
2386 return N->getOpcode() == ISD::MLOAD;
2387 }
2388
2389 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2390};
2391
2392/// This class is used to represent an MSTORE node
2393class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2394public:
2395 friend class SelectionDAG;
2396
2397 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2398 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2399 EVT MemVT, MachineMemOperand *MMO)
2400 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2401 StoreSDNodeBits.IsTruncating = isTrunc;
2402 StoreSDNodeBits.IsCompressing = isCompressing;
2403 }
2404
2405 /// Return true if the op does a truncation before store.
2406 /// For integers this is the same as doing a TRUNCATE and storing the result.
2407 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2408 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2409
2410 /// Returns true if the op does a compression to the vector before storing.
2411 /// The node contiguously stores the active elements (integers or floats)
2412 /// in src (those with their respective bit set in writemask k) to unaligned
2413 /// memory at base_addr.
2414 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2415
2416 const SDValue &getValue() const { return getOperand(1); }
2417 const SDValue &getBasePtr() const { return getOperand(2); }
2418 const SDValue &getOffset() const { return getOperand(3); }
2419 const SDValue &getMask() const { return getOperand(4); }
2420
2421 static bool classof(const SDNode *N) {
2422 return N->getOpcode() == ISD::MSTORE;
2423 }
2424};
2425
2426/// This is a base class used to represent
2427/// MGATHER and MSCATTER nodes
2428///
2429class MaskedGatherScatterSDNode : public MemSDNode {
2430public:
2431 friend class SelectionDAG;
2432
2433 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2434 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2435 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2436 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2437 LSBaseSDNodeBits.AddressingMode = IndexType;
2438 assert(getIndexType() == IndexType && "Value truncated")((void)0);
2439 }
2440
2441 /// How is Index applied to BasePtr when computing addresses.
2442 ISD::MemIndexType getIndexType() const {
2443 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2444 }
2445 void setIndexType(ISD::MemIndexType IndexType) {
2446 LSBaseSDNodeBits.AddressingMode = IndexType;
2447 }
2448 bool isIndexScaled() const {
2449 return (getIndexType() == ISD::SIGNED_SCALED) ||
2450 (getIndexType() == ISD::UNSIGNED_SCALED);
2451 }
2452 bool isIndexSigned() const {
2453 return (getIndexType() == ISD::SIGNED_SCALED) ||
2454 (getIndexType() == ISD::SIGNED_UNSCALED);
2455 }
2456
2457 // In the both nodes address is Op1, mask is Op2:
2458 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2459 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2460 // Mask is a vector of i1 elements
2461 const SDValue &getBasePtr() const { return getOperand(3); }
2462 const SDValue &getIndex() const { return getOperand(4); }
2463 const SDValue &getMask() const { return getOperand(2); }
2464 const SDValue &getScale() const { return getOperand(5); }
2465
2466 static bool classof(const SDNode *N) {
2467 return N->getOpcode() == ISD::MGATHER ||
2468 N->getOpcode() == ISD::MSCATTER;
2469 }
2470};
2471
2472/// This class is used to represent an MGATHER node
2473///
2474class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2475public:
2476 friend class SelectionDAG;
2477
2478 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2479 EVT MemVT, MachineMemOperand *MMO,
2480 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2481 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2482 IndexType) {
2483 LoadSDNodeBits.ExtTy = ETy;
2484 }
2485
2486 const SDValue &getPassThru() const { return getOperand(1); }
2487
2488 ISD::LoadExtType getExtensionType() const {
2489 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2490 }
2491
2492 static bool classof(const SDNode *N) {
2493 return N->getOpcode() == ISD::MGATHER;
2494 }
2495};
2496
2497/// This class is used to represent an MSCATTER node
2498///
2499class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2500public:
2501 friend class SelectionDAG;
2502
2503 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2504 EVT MemVT, MachineMemOperand *MMO,
2505 ISD::MemIndexType IndexType, bool IsTrunc)
2506 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2507 IndexType) {
2508 StoreSDNodeBits.IsTruncating = IsTrunc;
2509 }
2510
2511 /// Return true if the op does a truncation before store.
2512 /// For integers this is the same as doing a TRUNCATE and storing the result.
2513 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2514 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2515
2516 const SDValue &getValue() const { return getOperand(1); }
2517
2518 static bool classof(const SDNode *N) {
2519 return N->getOpcode() == ISD::MSCATTER;
2520 }
2521};
2522
2523/// An SDNode that represents everything that will be needed
2524/// to construct a MachineInstr. These nodes are created during the
2525/// instruction selection proper phase.
2526///
2527/// Note that the only supported way to set the `memoperands` is by calling the
2528/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2529/// inside the DAG rather than in the node.
2530class MachineSDNode : public SDNode {
2531private:
2532 friend class SelectionDAG;
2533
2534 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2535 : SDNode(Opc, Order, DL, VTs) {}
2536
2537 // We use a pointer union between a single `MachineMemOperand` pointer and
2538 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2539 // the number of these is zero, the single pointer variant used when the
2540 // number is one, and the array is used for larger numbers.
2541 //
2542 // The array is allocated via the `SelectionDAG`'s allocator and so will
2543 // always live until the DAG is cleaned up and doesn't require ownership here.
2544 //
2545 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2546 // subclasses aren't managed in a conforming C++ manner. See the comments on
2547 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2548 // constraint here is that these don't manage memory with their constructor or
2549 // destructor and can be initialized to a good state even if they start off
2550 // uninitialized.
2551 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2552
2553 // Note that this could be folded into the above `MemRefs` member if doing so
2554 // is advantageous at some point. We don't need to store this in most cases.
2555 // However, at the moment this doesn't appear to make the allocation any
2556 // smaller and makes the code somewhat simpler to read.
2557 int NumMemRefs = 0;
2558
2559public:
2560 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2561
2562 ArrayRef<MachineMemOperand *> memoperands() const {
2563 // Special case the common cases.
2564 if (NumMemRefs == 0)
2565 return {};
2566 if (NumMemRefs == 1)
2567 return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);
2568
2569 // Otherwise we have an actual array.
2570 return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
2571 }
2572 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2573 mmo_iterator memoperands_end() const { return memoperands().end(); }
2574 bool memoperands_empty() const { return memoperands().empty(); }
2575
2576 /// Clear out the memory reference descriptor list.
2577 void clearMemRefs() {
2578 MemRefs = nullptr;
2579 NumMemRefs = 0;
2580 }
2581
2582 static bool classof(const SDNode *N) {
2583 return N->isMachineOpcode();
2584 }
2585};
2586
2587/// An SDNode that records if a register contains a value that is guaranteed to
2588/// be aligned accordingly.
2589class AssertAlignSDNode : public SDNode {
2590 Align Alignment;
2591
2592public:
2593 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
2594 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
2595
2596 Align getAlign() const { return Alignment; }
2597
2598 static bool classof(const SDNode *N) {
2599 return N->getOpcode() == ISD::AssertAlign;
2600 }
2601};
2602
2603class SDNodeIterator {
2604 const SDNode *Node;
2605 unsigned Operand;
2606
2607 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
2608
2609public:
2610 using iterator_category = std::forward_iterator_tag;
2611 using value_type = SDNode;
2612 using difference_type = std::ptrdiff_t;
2613 using pointer = value_type *;
2614 using reference = value_type &;
2615
2616 bool operator==(const SDNodeIterator& x) const {
2617 return Operand == x.Operand;
2618 }
2619 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
2620
2621 pointer operator*() const {
2622 return Node->getOperand(Operand).getNode();
2623 }
2624 pointer operator->() const { return operator*(); }
2625
2626 SDNodeIterator& operator++() { // Preincrement
2627 ++Operand;
2628 return *this;
2629 }
2630 SDNodeIterator operator++(int) { // Postincrement
2631 SDNodeIterator tmp = *this; ++*this; return tmp;
2632 }
2633 size_t operator-(SDNodeIterator Other) const {
2634 assert(Node == Other.Node &&((void)0)
2635 "Cannot compare iterators of two different nodes!")((void)0);
2636 return Operand - Other.Operand;
2637 }
2638
2639 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
2640 static SDNodeIterator end (const SDNode *N) {
2641 return SDNodeIterator(N, N->getNumOperands());
2642 }
2643
2644 unsigned getOperand() const { return Operand; }
2645 const SDNode *getNode() const { return Node; }
2646};
2647
2648template <> struct GraphTraits<SDNode*> {
2649 using NodeRef = SDNode *;
2650 using ChildIteratorType = SDNodeIterator;
2651
2652 static NodeRef getEntryNode(SDNode *N) { return N; }
2653
2654 static ChildIteratorType child_begin(NodeRef N) {
2655 return SDNodeIterator::begin(N);
2656 }
2657
2658 static ChildIteratorType child_end(NodeRef N) {
2659 return SDNodeIterator::end(N);
2660 }
2661};
2662
2663/// A representation of the largest SDNode, for use in sizeof().
2664///
2665/// This needs to be a union because the largest node differs on 32 bit systems
2666/// with 4 and 8 byte pointer alignment, respectively.
2667using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
2668 BlockAddressSDNode,
2669 GlobalAddressSDNode,
2670 PseudoProbeSDNode>;
2671
2672/// The SDNode class with the greatest alignment requirement.
2673using MostAlignedSDNode = GlobalAddressSDNode;
2674
2675namespace ISD {
2676
2677 /// Returns true if the specified node is a non-extending and unindexed load.
2678 inline bool isNormalLoad(const SDNode *N) {
2679 const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
2680 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
2681 Ld->getAddressingMode() == ISD::UNINDEXED;
2682 }
2683
2684 /// Returns true if the specified node is a non-extending load.
2685 inline bool isNON_EXTLoad(const SDNode *N) {
2686 return isa<LoadSDNode>(N) &&
2687 cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
2688 }
2689
2690 /// Returns true if the specified node is a EXTLOAD.
2691 inline bool isEXTLoad(const SDNode *N) {
2692 return isa<LoadSDNode>(N) &&
2693 cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
2694 }
2695
2696 /// Returns true if the specified node is a SEXTLOAD.
2697 inline bool isSEXTLoad(const SDNode *N) {
2698 return isa<LoadSDNode>(N) &&
2699 cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
2700 }
2701
2702 /// Returns true if the specified node is a ZEXTLOAD.
2703 inline bool isZEXTLoad(const SDNode *N) {
2704 return isa<LoadSDNode>(N) &&
2705 cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
2706 }
2707
2708 /// Returns true if the specified node is an unindexed load.
2709 inline bool isUNINDEXEDLoad(const SDNode *N) {
2710 return isa<LoadSDNode>(N) &&
2711 cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2712 }
2713
2714 /// Returns true if the specified node is a non-truncating
2715 /// and unindexed store.
2716 inline bool isNormalStore(const SDNode *N) {
2717 const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
2718 return St && !St->isTruncatingStore() &&
2719 St->getAddressingMode() == ISD::UNINDEXED;
2720 }
2721
2722 /// Returns true if the specified node is an unindexed store.
2723 inline bool isUNINDEXEDStore(const SDNode *N) {
2724 return isa<StoreSDNode>(N) &&
2725 cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2726 }
2727
2728 /// Attempt to match a unary predicate against a scalar/splat constant or
2729 /// every element of a constant BUILD_VECTOR.
2730 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2731 bool matchUnaryPredicate(SDValue Op,
2732 std::function<bool(ConstantSDNode *)> Match,
2733 bool AllowUndefs = false);
2734
2735 /// Attempt to match a binary predicate against a pair of scalar/splat
2736 /// constants or every element of a pair of constant BUILD_VECTORs.
2737 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2738 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
2739 bool matchBinaryPredicate(
2740 SDValue LHS, SDValue RHS,
2741 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
2742 bool AllowUndefs = false, bool AllowTypeMismatch = false);
2743
2744 /// Returns true if the specified value is the overflow result from one
2745 /// of the overflow intrinsic nodes.
2746 inline bool isOverflowIntrOpRes(SDValue Op) {
2747 unsigned Opc = Op.getOpcode();
2748 return (Op.getResNo() == 1 &&
2749 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2750 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2751 }
2752
2753} // end namespace ISD
2754
2755} // end namespace llvm
2756
2757#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H