Bug Summary

File:src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1110, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

/usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

→
1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Target/TargetLoweringObjectFile.h"
33#include "llvm/Target/TargetMachine.h"
34#include <cctype>
35using namespace llvm;
36
37/// NOTE: The TargetMachine owns TLOF.
38TargetLowering::TargetLowering(const TargetMachine &tm)
39 : TargetLoweringBase(tm) {}
40
41const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
42 return nullptr;
43}
44
45bool TargetLowering::isPositionIndependent() const {
46 return getTargetMachine().isPositionIndependent();
47}
48
49/// Check whether a given call node is in tail position within its function. If
50/// so, it sets Chain to the input chain of the tail call.
51bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
52 SDValue &Chain) const {
53 const Function &F = DAG.getMachineFunction().getFunction();
54
55 // First, check if tail calls have been disabled in this function.
56 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
57 return false;
58
59 // Conservatively require the attributes of the call to match those of
60 // the return. Ignore following attributes because they don't affect the
61 // call sequence.
62 AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
63 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
64 Attribute::DereferenceableOrNull, Attribute::NoAlias,
65 Attribute::NonNull})
66 CallerAttrs.removeAttribute(Attr);
67
68 if (CallerAttrs.hasAttributes())
69 return false;
70
71 // It's not safe to eliminate the sign / zero extension of the return value.
72 if (CallerAttrs.contains(Attribute::ZExt) ||
73 CallerAttrs.contains(Attribute::SExt))
74 return false;
75
76 // Check if the only use is a function return node.
77 return isUsedByReturnOnly(Node, Chain);
78}
79
80bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
81 const uint32_t *CallerPreservedMask,
82 const SmallVectorImpl<CCValAssign> &ArgLocs,
83 const SmallVectorImpl<SDValue> &OutVals) const {
84 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
85 const CCValAssign &ArgLoc = ArgLocs[I];
86 if (!ArgLoc.isRegLoc())
87 continue;
88 MCRegister Reg = ArgLoc.getLocReg();
89 // Only look at callee saved registers.
90 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
91 continue;
92 // Check that we pass the value used for the caller.
93 // (We look for a CopyFromReg reading a virtual register that is used
94 // for the function live-in value of register Reg)
95 SDValue Value = OutVals[I];
96 if (Value->getOpcode() != ISD::CopyFromReg)
97 return false;
98 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
99 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
100 return false;
101 }
102 return true;
103}
104
105/// Set CallLoweringInfo attribute flags based on a call instruction
106/// and called function attributes.
107void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
108 unsigned ArgIdx) {
109 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
110 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
111 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
112 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
113 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
114 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
115 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
116 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
117 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
118 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
119 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
120 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
121 Alignment = Call->getParamStackAlign(ArgIdx);
122 IndirectType = nullptr;
123 assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&((void)0)
124 "multiple ABI attributes?")((void)0);
125 if (IsByVal) {
126 IndirectType = Call->getParamByValType(ArgIdx);
127 if (!Alignment)
128 Alignment = Call->getParamAlign(ArgIdx);
129 }
130 if (IsPreallocated)
131 IndirectType = Call->getParamPreallocatedType(ArgIdx);
132 if (IsInAlloca)
133 IndirectType = Call->getParamInAllocaType(ArgIdx);
134}
135
136/// Generate a libcall taking the given operands as arguments and returning a
137/// result of type RetVT.
138std::pair<SDValue, SDValue>
139TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
140 ArrayRef<SDValue> Ops,
141 MakeLibCallOptions CallOptions,
142 const SDLoc &dl,
143 SDValue InChain) const {
144 if (!InChain)
145 InChain = DAG.getEntryNode();
146
147 TargetLowering::ArgListTy Args;
148 Args.reserve(Ops.size());
149
150 TargetLowering::ArgListEntry Entry;
151 for (unsigned i = 0; i < Ops.size(); ++i) {
152 SDValue NewOp = Ops[i];
153 Entry.Node = NewOp;
154 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
155 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
156 CallOptions.IsSExt);
157 Entry.IsZExt = !Entry.IsSExt;
158
159 if (CallOptions.IsSoften &&
160 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
161 Entry.IsSExt = Entry.IsZExt = false;
162 }
163 Args.push_back(Entry);
164 }
165
166 if (LC == RTLIB::UNKNOWN_LIBCALL)
167 report_fatal_error("Unsupported library call operation!");
168 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
169 getPointerTy(DAG.getDataLayout()));
170
171 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
172 TargetLowering::CallLoweringInfo CLI(DAG);
173 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
174 bool zeroExtend = !signExtend;
175
176 if (CallOptions.IsSoften &&
177 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
178 signExtend = zeroExtend = false;
179 }
180
181 CLI.setDebugLoc(dl)
182 .setChain(InChain)
183 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
184 .setNoReturn(CallOptions.DoesNotReturn)
185 .setDiscardResult(!CallOptions.IsReturnValueUsed)
186 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
187 .setSExtResult(signExtend)
188 .setZExtResult(zeroExtend);
189 return LowerCallTo(CLI);
190}
191
192bool TargetLowering::findOptimalMemOpLowering(
193 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
194 unsigned SrcAS, const AttributeList &FuncAttributes) const {
195 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
196 return false;
197
198 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
199
200 if (VT == MVT::Other) {
201 // Use the largest integer type whose alignment constraints are satisfied.
202 // We only need to check DstAlign here as SrcAlign is always greater or
203 // equal to DstAlign (or zero).
204 VT = MVT::i64;
205 if (Op.isFixedDstAlign())
206 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
207 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
208 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
209 assert(VT.isInteger())((void)0);
210
211 // Find the largest legal integer type.
212 MVT LVT = MVT::i64;
213 while (!isTypeLegal(LVT))
214 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
215 assert(LVT.isInteger())((void)0);
216
217 // If the type we've chosen is larger than the largest legal integer type
218 // then use that instead.
219 if (VT.bitsGT(LVT))
220 VT = LVT;
221 }
222
223 unsigned NumMemOps = 0;
224 uint64_t Size = Op.size();
225 while (Size) {
226 unsigned VTSize = VT.getSizeInBits() / 8;
227 while (VTSize > Size) {
228 // For now, only use non-vector load / store's for the left-over pieces.
229 EVT NewVT = VT;
230 unsigned NewVTSize;
231
232 bool Found = false;
233 if (VT.isVector() || VT.isFloatingPoint()) {
234 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
235 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
236 isSafeMemOpType(NewVT.getSimpleVT()))
237 Found = true;
238 else if (NewVT == MVT::i64 &&
239 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
240 isSafeMemOpType(MVT::f64)) {
241 // i64 is usually not legal on 32-bit targets, but f64 may be.
242 NewVT = MVT::f64;
243 Found = true;
244 }
245 }
246
247 if (!Found) {
248 do {
249 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
250 if (NewVT == MVT::i8)
251 break;
252 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
253 }
254 NewVTSize = NewVT.getSizeInBits() / 8;
255
256 // If the new VT cannot cover all of the remaining bits, then consider
257 // issuing a (or a pair of) unaligned and overlapping load / store.
258 bool Fast;
259 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
260 allowsMisalignedMemoryAccesses(
261 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
262 MachineMemOperand::MONone, &Fast) &&
263 Fast)
264 VTSize = Size;
265 else {
266 VT = NewVT;
267 VTSize = NewVTSize;
268 }
269 }
270
271 if (++NumMemOps > Limit)
272 return false;
273
274 MemOps.push_back(VT);
275 Size -= VTSize;
276 }
277
278 return true;
279}
280
281/// Soften the operands of a comparison. This code is shared among BR_CC,
282/// SELECT_CC, and SETCC handlers.
283void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
284 SDValue &NewLHS, SDValue &NewRHS,
285 ISD::CondCode &CCCode,
286 const SDLoc &dl, const SDValue OldLHS,
287 const SDValue OldRHS) const {
288 SDValue Chain;
289 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
290 OldRHS, Chain);
291}
292
293void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
294 SDValue &NewLHS, SDValue &NewRHS,
295 ISD::CondCode &CCCode,
296 const SDLoc &dl, const SDValue OldLHS,
297 const SDValue OldRHS,
298 SDValue &Chain,
299 bool IsSignaling) const {
300 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
301 // not supporting it. We can update this code when libgcc provides such
302 // functions.
303
304 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)((void)0)
305 && "Unsupported setcc type!")((void)0);
306
307 // Expand into one or more soft-fp libcall(s).
308 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
309 bool ShouldInvertCC = false;
310 switch (CCCode) {
311 case ISD::SETEQ:
312 case ISD::SETOEQ:
313 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
314 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
315 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
316 break;
317 case ISD::SETNE:
318 case ISD::SETUNE:
319 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
320 (VT == MVT::f64) ? RTLIB::UNE_F64 :
321 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
322 break;
323 case ISD::SETGE:
324 case ISD::SETOGE:
325 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
326 (VT == MVT::f64) ? RTLIB::OGE_F64 :
327 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
328 break;
329 case ISD::SETLT:
330 case ISD::SETOLT:
331 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
332 (VT == MVT::f64) ? RTLIB::OLT_F64 :
333 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
334 break;
335 case ISD::SETLE:
336 case ISD::SETOLE:
337 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
338 (VT == MVT::f64) ? RTLIB::OLE_F64 :
339 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
340 break;
341 case ISD::SETGT:
342 case ISD::SETOGT:
343 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
344 (VT == MVT::f64) ? RTLIB::OGT_F64 :
345 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
346 break;
347 case ISD::SETO:
348 ShouldInvertCC = true;
349 LLVM_FALLTHROUGH[[gnu::fallthrough]];
350 case ISD::SETUO:
351 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
352 (VT == MVT::f64) ? RTLIB::UO_F64 :
353 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
354 break;
355 case ISD::SETONE:
356 // SETONE = O && UNE
357 ShouldInvertCC = true;
358 LLVM_FALLTHROUGH[[gnu::fallthrough]];
359 case ISD::SETUEQ:
360 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
361 (VT == MVT::f64) ? RTLIB::UO_F64 :
362 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
364 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
365 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
366 break;
367 default:
368 // Invert CC for unordered comparisons
369 ShouldInvertCC = true;
370 switch (CCCode) {
371 case ISD::SETULT:
372 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
373 (VT == MVT::f64) ? RTLIB::OGE_F64 :
374 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
375 break;
376 case ISD::SETULE:
377 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
378 (VT == MVT::f64) ? RTLIB::OGT_F64 :
379 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
380 break;
381 case ISD::SETUGT:
382 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
383 (VT == MVT::f64) ? RTLIB::OLE_F64 :
384 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
385 break;
386 case ISD::SETUGE:
387 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
388 (VT == MVT::f64) ? RTLIB::OLT_F64 :
389 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
390 break;
391 default: llvm_unreachable("Do not know how to soften this setcc!")__builtin_unreachable();
392 }
393 }
394
395 // Use the target specific return value for comparions lib calls.
396 EVT RetVT = getCmpLibcallReturnType();
397 SDValue Ops[2] = {NewLHS, NewRHS};
398 TargetLowering::MakeLibCallOptions CallOptions;
399 EVT OpsVT[2] = { OldLHS.getValueType(),
400 OldRHS.getValueType() };
401 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
402 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
403 NewLHS = Call.first;
404 NewRHS = DAG.getConstant(0, dl, RetVT);
405
406 CCCode = getCmpLibcallCC(LC1);
407 if (ShouldInvertCC) {
408 assert(RetVT.isInteger())((void)0);
409 CCCode = getSetCCInverse(CCCode, RetVT);
410 }
411
412 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
413 // Update Chain.
414 Chain = Call.second;
415 } else {
416 EVT SetCCVT =
417 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
418 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
419 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
420 CCCode = getCmpLibcallCC(LC2);
421 if (ShouldInvertCC)
422 CCCode = getSetCCInverse(CCCode, RetVT);
423 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
424 if (Chain)
425 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
426 Call2.second);
427 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
428 Tmp.getValueType(), Tmp, NewLHS);
429 NewRHS = SDValue();
430 }
431}
432
433/// Return the entry encoding for a jump table in the current function. The
434/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
435unsigned TargetLowering::getJumpTableEncoding() const {
436 // In non-pic modes, just use the address of a block.
437 if (!isPositionIndependent())
438 return MachineJumpTableInfo::EK_BlockAddress;
439
440 // In PIC mode, if the target supports a GPRel32 directive, use it.
441 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
442 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
443
444 // Otherwise, use a label difference.
445 return MachineJumpTableInfo::EK_LabelDifference32;
446}
447
448SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
449 SelectionDAG &DAG) const {
450 // If our PIC model is GP relative, use the global offset table as the base.
451 unsigned JTEncoding = getJumpTableEncoding();
452
453 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
454 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
455 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
456
457 return Table;
458}
459
460/// This returns the relocation base for the given PIC jumptable, the same as
461/// getPICJumpTableRelocBase, but as an MCExpr.
462const MCExpr *
463TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
464 unsigned JTI,MCContext &Ctx) const{
465 // The normal PIC reloc base is the label at the start of the jump table.
466 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
467}
468
469bool
470TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
471 const TargetMachine &TM = getTargetMachine();
472 const GlobalValue *GV = GA->getGlobal();
473
474 // If the address is not even local to this DSO we will have to load it from
475 // a got and then add the offset.
476 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
477 return false;
478
479 // If the code is position independent we will have to add a base register.
480 if (isPositionIndependent())
481 return false;
482
483 // Otherwise we can do it.
484 return true;
485}
486
487//===----------------------------------------------------------------------===//
488// Optimization Methods
489//===----------------------------------------------------------------------===//
490
491/// If the specified instruction has a constant integer operand and there are
492/// bits set in that constant that are not demanded, then clear those bits and
493/// return true.
494bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
495 const APInt &DemandedBits,
496 const APInt &DemandedElts,
497 TargetLoweringOpt &TLO) const {
498 SDLoc DL(Op);
499 unsigned Opcode = Op.getOpcode();
500
501 // Do target-specific constant optimization.
502 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
503 return TLO.New.getNode();
504
505 // FIXME: ISD::SELECT, ISD::SELECT_CC
506 switch (Opcode) {
507 default:
508 break;
509 case ISD::XOR:
510 case ISD::AND:
511 case ISD::OR: {
512 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
513 if (!Op1C || Op1C->isOpaque())
514 return false;
515
516 // If this is a 'not' op, don't touch it because that's a canonical form.
517 const APInt &C = Op1C->getAPIntValue();
518 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
519 return false;
520
521 if (!C.isSubsetOf(DemandedBits)) {
522 EVT VT = Op.getValueType();
523 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
524 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
525 return TLO.CombineTo(Op, NewOp);
526 }
527
528 break;
529 }
530 }
531
532 return false;
533}
534
535bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
536 const APInt &DemandedBits,
537 TargetLoweringOpt &TLO) const {
538 EVT VT = Op.getValueType();
539 APInt DemandedElts = VT.isVector()
540 ? APInt::getAllOnesValue(VT.getVectorNumElements())
541 : APInt(1, 1);
542 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
543}
544
545/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
546/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
547/// generalized for targets with other types of implicit widening casts.
548bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
549 const APInt &Demanded,
550 TargetLoweringOpt &TLO) const {
551 assert(Op.getNumOperands() == 2 &&((void)0)
552 "ShrinkDemandedOp only supports binary operators!")((void)0);
553 assert(Op.getNode()->getNumValues() == 1 &&((void)0)
554 "ShrinkDemandedOp only supports nodes with one result!")((void)0);
555
556 SelectionDAG &DAG = TLO.DAG;
557 SDLoc dl(Op);
558
559 // Early return, as this function cannot handle vector types.
560 if (Op.getValueType().isVector())
561 return false;
562
563 // Don't do this if the node has another user, which may require the
564 // full value.
565 if (!Op.getNode()->hasOneUse())
566 return false;
567
568 // Search for the smallest integer type with free casts to and from
569 // Op's type. For expedience, just check power-of-2 integer types.
570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
571 unsigned DemandedSize = Demanded.getActiveBits();
572 unsigned SmallVTBits = DemandedSize;
573 if (!isPowerOf2_32(SmallVTBits))
574 SmallVTBits = NextPowerOf2(SmallVTBits);
575 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
576 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
577 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
578 TLI.isZExtFree(SmallVT, Op.getValueType())) {
579 // We found a type with free casts.
580 SDValue X = DAG.getNode(
581 Op.getOpcode(), dl, SmallVT,
582 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
583 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
584 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")((void)0);
585 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
586 return TLO.CombineTo(Op, Z);
587 }
588 }
589 return false;
590}
591
592bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
593 DAGCombinerInfo &DCI) const {
594 SelectionDAG &DAG = DCI.DAG;
595 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
596 !DCI.isBeforeLegalizeOps());
597 KnownBits Known;
598
599 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
600 if (Simplified) {
601 DCI.AddToWorklist(Op.getNode());
602 DCI.CommitTargetLoweringOpt(TLO);
603 }
604 return Simplified;
605}
606
607bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
608 KnownBits &Known,
609 TargetLoweringOpt &TLO,
610 unsigned Depth,
611 bool AssumeSingleUse) const {
612 EVT VT = Op.getValueType();
613
614 // TODO: We can probably do more work on calculating the known bits and
615 // simplifying the operations for scalable vectors, but for now we just
616 // bail out.
617 if (VT.isScalableVector()) {
618 // Pretend we don't know anything for now.
619 Known = KnownBits(DemandedBits.getBitWidth());
620 return false;
621 }
622
623 APInt DemandedElts = VT.isVector()
624 ? APInt::getAllOnesValue(VT.getVectorNumElements())
625 : APInt(1, 1);
626 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
627 AssumeSingleUse);
628}
629
630// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
631// TODO: Under what circumstances can we create nodes? Constant folding?
632SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
633 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
634 SelectionDAG &DAG, unsigned Depth) const {
635 // Limit search depth.
636 if (Depth >= SelectionDAG::MaxRecursionDepth)
637 return SDValue();
638
639 // Ignore UNDEFs.
640 if (Op.isUndef())
641 return SDValue();
642
643 // Not demanding any bits/elts from Op.
644 if (DemandedBits == 0 || DemandedElts == 0)
645 return DAG.getUNDEF(Op.getValueType());
646
647 unsigned NumElts = DemandedElts.getBitWidth();
648 unsigned BitWidth = DemandedBits.getBitWidth();
649 KnownBits LHSKnown, RHSKnown;
650 switch (Op.getOpcode()) {
651 case ISD::BITCAST: {
652 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
653 EVT SrcVT = Src.getValueType();
654 EVT DstVT = Op.getValueType();
655 if (SrcVT == DstVT)
656 return Src;
657
658 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
659 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
660 if (NumSrcEltBits == NumDstEltBits)
661 if (SDValue V = SimplifyMultipleUseDemandedBits(
662 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
663 return DAG.getBitcast(DstVT, V);
664
665 // TODO - bigendian once we have test coverage.
666 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
667 DAG.getDataLayout().isLittleEndian()) {
668 unsigned Scale = NumDstEltBits / NumSrcEltBits;
669 unsigned NumSrcElts = SrcVT.getVectorNumElements();
670 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
671 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
672 for (unsigned i = 0; i != Scale; ++i) {
673 unsigned Offset = i * NumSrcEltBits;
674 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
675 if (!Sub.isNullValue()) {
676 DemandedSrcBits |= Sub;
677 for (unsigned j = 0; j != NumElts; ++j)
678 if (DemandedElts[j])
679 DemandedSrcElts.setBit((j * Scale) + i);
680 }
681 }
682
683 if (SDValue V = SimplifyMultipleUseDemandedBits(
684 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
685 return DAG.getBitcast(DstVT, V);
686 }
687
688 // TODO - bigendian once we have test coverage.
689 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
690 DAG.getDataLayout().isLittleEndian()) {
691 unsigned Scale = NumSrcEltBits / NumDstEltBits;
692 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
693 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
694 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
695 for (unsigned i = 0; i != NumElts; ++i)
696 if (DemandedElts[i]) {
697 unsigned Offset = (i % Scale) * NumDstEltBits;
698 DemandedSrcBits.insertBits(DemandedBits, Offset);
699 DemandedSrcElts.setBit(i / Scale);
700 }
701
702 if (SDValue V = SimplifyMultipleUseDemandedBits(
703 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
704 return DAG.getBitcast(DstVT, V);
705 }
706
707 break;
708 }
709 case ISD::AND: {
710 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
711 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
712
713 // If all of the demanded bits are known 1 on one side, return the other.
714 // These bits cannot contribute to the result of the 'and' in this
715 // context.
716 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
717 return Op.getOperand(0);
718 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
719 return Op.getOperand(1);
720 break;
721 }
722 case ISD::OR: {
723 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
724 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
725
726 // If all of the demanded bits are known zero on one side, return the
727 // other. These bits cannot contribute to the result of the 'or' in this
728 // context.
729 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
730 return Op.getOperand(0);
731 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
732 return Op.getOperand(1);
733 break;
734 }
735 case ISD::XOR: {
736 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
737 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
738
739 // If all of the demanded bits are known zero on one side, return the
740 // other.
741 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
742 return Op.getOperand(0);
743 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
744 return Op.getOperand(1);
745 break;
746 }
747 case ISD::SHL: {
748 // If we are only demanding sign bits then we can use the shift source
749 // directly.
750 if (const APInt *MaxSA =
751 DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
752 SDValue Op0 = Op.getOperand(0);
753 unsigned ShAmt = MaxSA->getZExtValue();
754 unsigned NumSignBits =
755 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
756 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
757 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
758 return Op0;
759 }
760 break;
761 }
762 case ISD::SETCC: {
763 SDValue Op0 = Op.getOperand(0);
764 SDValue Op1 = Op.getOperand(1);
765 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
766 // If (1) we only need the sign-bit, (2) the setcc operands are the same
767 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
768 // -1, we may be able to bypass the setcc.
769 if (DemandedBits.isSignMask() &&
770 Op0.getScalarValueSizeInBits() == BitWidth &&
771 getBooleanContents(Op0.getValueType()) ==
772 BooleanContent::ZeroOrNegativeOneBooleanContent) {
773 // If we're testing X < 0, then this compare isn't needed - just use X!
774 // FIXME: We're limiting to integer types here, but this should also work
775 // if we don't care about FP signed-zero. The use of SETLT with FP means
776 // that we don't care about NaNs.
777 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
778 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
779 return Op0;
780 }
781 break;
782 }
783 case ISD::SIGN_EXTEND_INREG: {
784 // If none of the extended bits are demanded, eliminate the sextinreg.
785 SDValue Op0 = Op.getOperand(0);
786 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
787 unsigned ExBits = ExVT.getScalarSizeInBits();
788 if (DemandedBits.getActiveBits() <= ExBits)
789 return Op0;
790 // If the input is already sign extended, just drop the extension.
791 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
792 if (NumSignBits >= (BitWidth - ExBits + 1))
793 return Op0;
794 break;
795 }
796 case ISD::ANY_EXTEND_VECTOR_INREG:
797 case ISD::SIGN_EXTEND_VECTOR_INREG:
798 case ISD::ZERO_EXTEND_VECTOR_INREG: {
799 // If we only want the lowest element and none of extended bits, then we can
800 // return the bitcasted source vector.
801 SDValue Src = Op.getOperand(0);
802 EVT SrcVT = Src.getValueType();
803 EVT DstVT = Op.getValueType();
804 if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
805 DAG.getDataLayout().isLittleEndian() &&
806 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
807 return DAG.getBitcast(DstVT, Src);
808 }
809 break;
810 }
811 case ISD::INSERT_VECTOR_ELT: {
812 // If we don't demand the inserted element, return the base vector.
813 SDValue Vec = Op.getOperand(0);
814 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
815 EVT VecVT = Vec.getValueType();
816 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
817 !DemandedElts[CIdx->getZExtValue()])
818 return Vec;
819 break;
820 }
821 case ISD::INSERT_SUBVECTOR: {
822 // If we don't demand the inserted subvector, return the base vector.
823 SDValue Vec = Op.getOperand(0);
824 SDValue Sub = Op.getOperand(1);
825 uint64_t Idx = Op.getConstantOperandVal(2);
826 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
827 if (DemandedElts.extractBits(NumSubElts, Idx) == 0)
828 return Vec;
829 break;
830 }
831 case ISD::VECTOR_SHUFFLE: {
832 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
833
834 // If all the demanded elts are from one operand and are inline,
835 // then we can use the operand directly.
836 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
837 for (unsigned i = 0; i != NumElts; ++i) {
838 int M = ShuffleMask[i];
839 if (M < 0 || !DemandedElts[i])
840 continue;
841 AllUndef = false;
842 IdentityLHS &= (M == (int)i);
843 IdentityRHS &= ((M - NumElts) == i);
844 }
845
846 if (AllUndef)
847 return DAG.getUNDEF(Op.getValueType());
848 if (IdentityLHS)
849 return Op.getOperand(0);
850 if (IdentityRHS)
851 return Op.getOperand(1);
852 break;
853 }
854 default:
855 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
856 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
857 Op, DemandedBits, DemandedElts, DAG, Depth))
858 return V;
859 break;
860 }
861 return SDValue();
862}
863
864SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
865 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
866 unsigned Depth) const {
867 EVT VT = Op.getValueType();
868 APInt DemandedElts = VT.isVector()
869 ? APInt::getAllOnesValue(VT.getVectorNumElements())
870 : APInt(1, 1);
871 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
872 Depth);
873}
874
875SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
876 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
877 unsigned Depth) const {
878 APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());
879 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
880 Depth);
881}
882
883/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
884/// result of Op are ever used downstream. If we can use this information to
885/// simplify Op, create a new simplified DAG node and return true, returning the
886/// original and new nodes in Old and New. Otherwise, analyze the expression and
887/// return a mask of Known bits for the expression (used to simplify the
888/// caller). The Known bits may only be accurate for those bits in the
889/// OriginalDemandedBits and OriginalDemandedElts.
890bool TargetLowering::SimplifyDemandedBits(
891 SDValue Op, const APInt &OriginalDemandedBits,
892 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
893 unsigned Depth, bool AssumeSingleUse) const {
894 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
895 assert(Op.getScalarValueSizeInBits() == BitWidth &&((void)0)
896 "Mask size mismatches value type size!")((void)0);
897
898 // Don't know anything.
899 Known = KnownBits(BitWidth);
900
901 // TODO: We can probably do more work on calculating the known bits and
902 // simplifying the operations for scalable vectors, but for now we just
903 // bail out.
904 if (Op.getValueType().isScalableVector())
905 return false;
906
907 unsigned NumElts = OriginalDemandedElts.getBitWidth();
908 assert((!Op.getValueType().isVector() ||((void)0)
909 NumElts == Op.getValueType().getVectorNumElements()) &&((void)0)
910 "Unexpected vector size")((void)0);
911
912 APInt DemandedBits = OriginalDemandedBits;
913 APInt DemandedElts = OriginalDemandedElts;
914 SDLoc dl(Op);
915 auto &DL = TLO.DAG.getDataLayout();
916
917 // Undef operand.
918 if (Op.isUndef())
919 return false;
920
921 if (Op.getOpcode() == ISD::Constant) {
922 // We know all of the bits for a constant!
923 Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
924 return false;
925 }
926
927 if (Op.getOpcode() == ISD::ConstantFP) {
928 // We know all of the bits for a floating point constant!
929 Known = KnownBits::makeConstant(
930 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
931 return false;
932 }
933
934 // Other users may use these bits.
935 EVT VT = Op.getValueType();
936 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
937 if (Depth != 0) {
938 // If not at the root, Just compute the Known bits to
939 // simplify things downstream.
940 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
941 return false;
942 }
943 // If this is the root being simplified, allow it to have multiple uses,
944 // just set the DemandedBits/Elts to all bits.
945 DemandedBits = APInt::getAllOnesValue(BitWidth);
946 DemandedElts = APInt::getAllOnesValue(NumElts);
947 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
948 // Not demanding any bits/elts from Op.
949 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
950 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
951 // Limit search depth.
952 return false;
953 }
954
955 KnownBits Known2;
956 switch (Op.getOpcode()) {
957 case ISD::TargetConstant:
958 llvm_unreachable("Can't simplify this node")__builtin_unreachable();
959 case ISD::SCALAR_TO_VECTOR: {
960 if (!DemandedElts[0])
961 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
962
963 KnownBits SrcKnown;
964 SDValue Src = Op.getOperand(0);
965 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
966 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
967 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
968 return true;
969
970 // Upper elements are undef, so only get the knownbits if we just demand
971 // the bottom element.
972 if (DemandedElts == 1)
973 Known = SrcKnown.anyextOrTrunc(BitWidth);
974 break;
975 }
976 case ISD::BUILD_VECTOR:
977 // Collect the known bits that are shared by every demanded element.
978 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
979 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
980 return false; // Don't fall through, will infinitely loop.
981 case ISD::LOAD: {
982 auto *LD = cast<LoadSDNode>(Op);
983 if (getTargetConstantFromLoad(LD)) {
984 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
985 return false; // Don't fall through, will infinitely loop.
986 }
987 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
988 // If this is a ZEXTLoad and we are looking at the loaded value.
989 EVT MemVT = LD->getMemoryVT();
990 unsigned MemBits = MemVT.getScalarSizeInBits();
991 Known.Zero.setBitsFrom(MemBits);
992 return false; // Don't fall through, will infinitely loop.
993 }
994 break;
995 }
996 case ISD::INSERT_VECTOR_ELT: {
997 SDValue Vec = Op.getOperand(0);
998 SDValue Scl = Op.getOperand(1);
999 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1000 EVT VecVT = Vec.getValueType();
1001
1002 // If index isn't constant, assume we need all vector elements AND the
1003 // inserted element.
1004 APInt DemandedVecElts(DemandedElts);
1005 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1006 unsigned Idx = CIdx->getZExtValue();
1007 DemandedVecElts.clearBit(Idx);
1008
1009 // Inserted element is not required.
1010 if (!DemandedElts[Idx])
1011 return TLO.CombineTo(Op, Vec);
1012 }
1013
1014 KnownBits KnownScl;
1015 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1016 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1017 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1018 return true;
1019
1020 Known = KnownScl.anyextOrTrunc(BitWidth);
1021
1022 KnownBits KnownVec;
1023 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1024 Depth + 1))
1025 return true;
1026
1027 if (!!DemandedVecElts)
1028 Known = KnownBits::commonBits(Known, KnownVec);
1029
1030 return false;
1031 }
1032 case ISD::INSERT_SUBVECTOR: {
1033 // Demand any elements from the subvector and the remainder from the src its
1034 // inserted into.
1035 SDValue Src = Op.getOperand(0);
1036 SDValue Sub = Op.getOperand(1);
1037 uint64_t Idx = Op.getConstantOperandVal(2);
1038 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1039 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1040 APInt DemandedSrcElts = DemandedElts;
1041 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
1042
1043 KnownBits KnownSub, KnownSrc;
1044 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1045 Depth + 1))
1046 return true;
1047 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1048 Depth + 1))
1049 return true;
1050
1051 Known.Zero.setAllBits();
1052 Known.One.setAllBits();
1053 if (!!DemandedSubElts)
1054 Known = KnownBits::commonBits(Known, KnownSub);
1055 if (!!DemandedSrcElts)
1056 Known = KnownBits::commonBits(Known, KnownSrc);
1057
1058 // Attempt to avoid multi-use src if we don't need anything from it.
1059 if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||
1060 !DemandedSrcElts.isAllOnesValue()) {
1061 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1062 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1063 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1064 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1065 if (NewSub || NewSrc) {
1066 NewSub = NewSub ? NewSub : Sub;
1067 NewSrc = NewSrc ? NewSrc : Src;
1068 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1069 Op.getOperand(2));
1070 return TLO.CombineTo(Op, NewOp);
1071 }
1072 }
1073 break;
1074 }
1075 case ISD::EXTRACT_SUBVECTOR: {
1076 // Offset the demanded elts by the subvector index.
1077 SDValue Src = Op.getOperand(0);
1078 if (Src.getValueType().isScalableVector())
1079 break;
1080 uint64_t Idx = Op.getConstantOperandVal(1);
1081 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1082 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1083
1084 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1085 Depth + 1))
1086 return true;
1087
1088 // Attempt to avoid multi-use src if we don't need anything from it.
1089 if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {
1090 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1091 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1092 if (DemandedSrc) {
1093 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1094 Op.getOperand(1));
1095 return TLO.CombineTo(Op, NewOp);
1096 }
1097 }
1098 break;
1099 }
1100 case ISD::CONCAT_VECTORS: {
1101 Known.Zero.setAllBits();
1102 Known.One.setAllBits();
1103 EVT SubVT = Op.getOperand(0).getValueType();
1104 unsigned NumSubVecs = Op.getNumOperands();
1105 unsigned NumSubElts = SubVT.getVectorNumElements();
1106 for (unsigned i = 0; i != NumSubVecs; ++i) {
1107 APInt DemandedSubElts =
1108 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1109 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1110 Known2, TLO, Depth + 1))
1111 return true;
1112 // Known bits are shared by every demanded subvector element.
1113 if (!!DemandedSubElts)
1114 Known = KnownBits::commonBits(Known, Known2);
1115 }
1116 break;
1117 }
1118 case ISD::VECTOR_SHUFFLE: {
1119 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1120
1121 // Collect demanded elements from shuffle operands..
1122 APInt DemandedLHS(NumElts, 0);
1123 APInt DemandedRHS(NumElts, 0);
1124 for (unsigned i = 0; i != NumElts; ++i) {
1125 if (!DemandedElts[i])
1126 continue;
1127 int M = ShuffleMask[i];
1128 if (M < 0) {
1129 // For UNDEF elements, we don't know anything about the common state of
1130 // the shuffle result.
1131 DemandedLHS.clearAllBits();
1132 DemandedRHS.clearAllBits();
1133 break;
1134 }
1135 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((void)0);
1136 if (M < (int)NumElts)
1137 DemandedLHS.setBit(M);
1138 else
1139 DemandedRHS.setBit(M - NumElts);
1140 }
1141
1142 if (!!DemandedLHS || !!DemandedRHS) {
1143 SDValue Op0 = Op.getOperand(0);
1144 SDValue Op1 = Op.getOperand(1);
1145
1146 Known.Zero.setAllBits();
1147 Known.One.setAllBits();
1148 if (!!DemandedLHS) {
1149 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1150 Depth + 1))
1151 return true;
1152 Known = KnownBits::commonBits(Known, Known2);
1153 }
1154 if (!!DemandedRHS) {
1155 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1156 Depth + 1))
1157 return true;
1158 Known = KnownBits::commonBits(Known, Known2);
1159 }
1160
1161 // Attempt to avoid multi-use ops if we don't need anything from them.
1162 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1163 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1164 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1165 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1166 if (DemandedOp0 || DemandedOp1) {
1167 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1168 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1169 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1170 return TLO.CombineTo(Op, NewOp);
1171 }
1172 }
1173 break;
1174 }
1175 case ISD::AND: {
1176 SDValue Op0 = Op.getOperand(0);
1177 SDValue Op1 = Op.getOperand(1);
1178
1179 // If the RHS is a constant, check to see if the LHS would be zero without
1180 // using the bits from the RHS. Below, we use knowledge about the RHS to
1181 // simplify the LHS, here we're using information from the LHS to simplify
1182 // the RHS.
1183 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1184 // Do not increment Depth here; that can cause an infinite loop.
1185 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1186 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1187 if ((LHSKnown.Zero & DemandedBits) ==
1188 (~RHSC->getAPIntValue() & DemandedBits))
1189 return TLO.CombineTo(Op, Op0);
1190
1191 // If any of the set bits in the RHS are known zero on the LHS, shrink
1192 // the constant.
1193 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1194 DemandedElts, TLO))
1195 return true;
1196
1197 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1198 // constant, but if this 'and' is only clearing bits that were just set by
1199 // the xor, then this 'and' can be eliminated by shrinking the mask of
1200 // the xor. For example, for a 32-bit X:
1201 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1202 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1203 LHSKnown.One == ~RHSC->getAPIntValue()) {
1204 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1205 return TLO.CombineTo(Op, Xor);
1206 }
1207 }
1208
1209 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1210 Depth + 1))
1211 return true;
1212 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1213 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1214 Known2, TLO, Depth + 1))
1215 return true;
1216 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1217
1218 // Attempt to avoid multi-use ops if we don't need anything from them.
1219 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1220 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1221 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1222 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1223 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1224 if (DemandedOp0 || DemandedOp1) {
1225 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1226 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1227 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1228 return TLO.CombineTo(Op, NewOp);
1229 }
1230 }
1231
1232 // If all of the demanded bits are known one on one side, return the other.
1233 // These bits cannot contribute to the result of the 'and'.
1234 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1235 return TLO.CombineTo(Op, Op0);
1236 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1237 return TLO.CombineTo(Op, Op1);
1238 // If all of the demanded bits in the inputs are known zeros, return zero.
1239 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1240 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1241 // If the RHS is a constant, see if we can simplify it.
1242 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1243 TLO))
1244 return true;
1245 // If the operation can be done in a smaller type, do so.
1246 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1247 return true;
1248
1249 Known &= Known2;
1250 break;
1251 }
1252 case ISD::OR: {
1253 SDValue Op0 = Op.getOperand(0);
1254 SDValue Op1 = Op.getOperand(1);
1255
1256 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1257 Depth + 1))
1258 return true;
1259 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1260 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1261 Known2, TLO, Depth + 1))
1262 return true;
1263 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1264
1265 // Attempt to avoid multi-use ops if we don't need anything from them.
1266 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1267 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1268 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1269 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1270 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1271 if (DemandedOp0 || DemandedOp1) {
1272 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1273 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1274 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1275 return TLO.CombineTo(Op, NewOp);
1276 }
1277 }
1278
1279 // If all of the demanded bits are known zero on one side, return the other.
1280 // These bits cannot contribute to the result of the 'or'.
1281 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1282 return TLO.CombineTo(Op, Op0);
1283 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1284 return TLO.CombineTo(Op, Op1);
1285 // If the RHS is a constant, see if we can simplify it.
1286 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1287 return true;
1288 // If the operation can be done in a smaller type, do so.
1289 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1290 return true;
1291
1292 Known |= Known2;
1293 break;
1294 }
1295 case ISD::XOR: {
1296 SDValue Op0 = Op.getOperand(0);
1297 SDValue Op1 = Op.getOperand(1);
1298
1299 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1300 Depth + 1))
1301 return true;
1302 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1303 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1304 Depth + 1))
1305 return true;
1306 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1307
1308 // Attempt to avoid multi-use ops if we don't need anything from them.
1309 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1310 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1311 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1312 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1313 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1314 if (DemandedOp0 || DemandedOp1) {
1315 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1316 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1317 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1318 return TLO.CombineTo(Op, NewOp);
1319 }
1320 }
1321
1322 // If all of the demanded bits are known zero on one side, return the other.
1323 // These bits cannot contribute to the result of the 'xor'.
1324 if (DemandedBits.isSubsetOf(Known.Zero))
1325 return TLO.CombineTo(Op, Op0);
1326 if (DemandedBits.isSubsetOf(Known2.Zero))
1327 return TLO.CombineTo(Op, Op1);
1328 // If the operation can be done in a smaller type, do so.
1329 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1330 return true;
1331
1332 // If all of the unknown bits are known to be zero on one side or the other
1333 // turn this into an *inclusive* or.
1334 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1335 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1336 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1337
1338 ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1339 if (C) {
1340 // If one side is a constant, and all of the set bits in the constant are
1341 // also known set on the other side, turn this into an AND, as we know
1342 // the bits will be cleared.
1343 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1344 // NB: it is okay if more bits are known than are requested
1345 if (C->getAPIntValue() == Known2.One) {
1346 SDValue ANDC =
1347 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1348 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1349 }
1350
1351 // If the RHS is a constant, see if we can change it. Don't alter a -1
1352 // constant because that's a 'not' op, and that is better for combining
1353 // and codegen.
1354 if (!C->isAllOnesValue() &&
1355 DemandedBits.isSubsetOf(C->getAPIntValue())) {
1356 // We're flipping all demanded bits. Flip the undemanded bits too.
1357 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1358 return TLO.CombineTo(Op, New);
1359 }
1360 }
1361
1362 // If we can't turn this into a 'not', try to shrink the constant.
1363 if (!C || !C->isAllOnesValue())
1364 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1365 return true;
1366
1367 Known ^= Known2;
1368 break;
1369 }
1370 case ISD::SELECT:
1371 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1372 Depth + 1))
1373 return true;
1374 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1375 Depth + 1))
1376 return true;
1377 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1378 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1379
1380 // If the operands are constants, see if we can simplify them.
1381 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1382 return true;
1383
1384 // Only known if known in both the LHS and RHS.
1385 Known = KnownBits::commonBits(Known, Known2);
1386 break;
1387 case ISD::SELECT_CC:
1388 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1389 Depth + 1))
1390 return true;
1391 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1392 Depth + 1))
1393 return true;
1394 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1395 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((void)0);
1396
1397 // If the operands are constants, see if we can simplify them.
1398 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1399 return true;
1400
1401 // Only known if known in both the LHS and RHS.
1402 Known = KnownBits::commonBits(Known, Known2);
1403 break;
1404 case ISD::SETCC: {
1405 SDValue Op0 = Op.getOperand(0);
1406 SDValue Op1 = Op.getOperand(1);
1407 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1408 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1409 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1410 // -1, we may be able to bypass the setcc.
1411 if (DemandedBits.isSignMask() &&
1412 Op0.getScalarValueSizeInBits() == BitWidth &&
1413 getBooleanContents(Op0.getValueType()) ==
1414 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1415 // If we're testing X < 0, then this compare isn't needed - just use X!
1416 // FIXME: We're limiting to integer types here, but this should also work
1417 // if we don't care about FP signed-zero. The use of SETLT with FP means
1418 // that we don't care about NaNs.
1419 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1420 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1421 return TLO.CombineTo(Op, Op0);
1422
1423 // TODO: Should we check for other forms of sign-bit comparisons?
1424 // Examples: X <= -1, X >= 0
1425 }
1426 if (getBooleanContents(Op0.getValueType()) ==
1427 TargetLowering::ZeroOrOneBooleanContent &&
1428 BitWidth > 1)
1429 Known.Zero.setBitsFrom(1);
1430 break;
1431 }
1432 case ISD::SHL: {
1433 SDValue Op0 = Op.getOperand(0);
1434 SDValue Op1 = Op.getOperand(1);
1435 EVT ShiftVT = Op1.getValueType();
1436
1437 if (const APInt *SA =
1438 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1439 unsigned ShAmt = SA->getZExtValue();
1440 if (ShAmt == 0)
1441 return TLO.CombineTo(Op, Op0);
1442
1443 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1444 // single shift. We can do this if the bottom bits (which are shifted
1445 // out) are never demanded.
1446 // TODO - support non-uniform vector amounts.
1447 if (Op0.getOpcode() == ISD::SRL) {
1448 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1449 if (const APInt *SA2 =
1450 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1451 unsigned C1 = SA2->getZExtValue();
1452 unsigned Opc = ISD::SHL;
1453 int Diff = ShAmt - C1;
1454 if (Diff < 0) {
1455 Diff = -Diff;
1456 Opc = ISD::SRL;
1457 }
1458 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1459 return TLO.CombineTo(
1460 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1461 }
1462 }
1463 }
1464
1465 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1466 // are not demanded. This will likely allow the anyext to be folded away.
1467 // TODO - support non-uniform vector amounts.
1468 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1469 SDValue InnerOp = Op0.getOperand(0);
1470 EVT InnerVT = InnerOp.getValueType();
1471 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1472 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1473 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1474 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1475 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1476 ShTy = InnerVT;
1477 SDValue NarrowShl =
1478 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1479 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1480 return TLO.CombineTo(
1481 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1482 }
1483
1484 // Repeat the SHL optimization above in cases where an extension
1485 // intervenes: (shl (anyext (shr x, c1)), c2) to
1486 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1487 // aren't demanded (as above) and that the shifted upper c1 bits of
1488 // x aren't demanded.
1489 // TODO - support non-uniform vector amounts.
1490 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1491 InnerOp.hasOneUse()) {
1492 if (const APInt *SA2 =
1493 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1494 unsigned InnerShAmt = SA2->getZExtValue();
1495 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1496 DemandedBits.getActiveBits() <=
1497 (InnerBits - InnerShAmt + ShAmt) &&
1498 DemandedBits.countTrailingZeros() >= ShAmt) {
1499 SDValue NewSA =
1500 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1501 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1502 InnerOp.getOperand(0));
1503 return TLO.CombineTo(
1504 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1505 }
1506 }
1507 }
1508 }
1509
1510 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1511 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1512 Depth + 1))
1513 return true;
1514 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1515 Known.Zero <<= ShAmt;
1516 Known.One <<= ShAmt;
1517 // low bits known zero.
1518 Known.Zero.setLowBits(ShAmt);
1519
1520 // Try shrinking the operation as long as the shift amount will still be
1521 // in range.
1522 if ((ShAmt < DemandedBits.getActiveBits()) &&
1523 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1524 return true;
1525 }
1526
1527 // If we are only demanding sign bits then we can use the shift source
1528 // directly.
1529 if (const APInt *MaxSA =
1530 TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1531 unsigned ShAmt = MaxSA->getZExtValue();
1532 unsigned NumSignBits =
1533 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1534 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1535 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1536 return TLO.CombineTo(Op, Op0);
1537 }
1538 break;
1539 }
1540 case ISD::SRL: {
1541 SDValue Op0 = Op.getOperand(0);
1542 SDValue Op1 = Op.getOperand(1);
1543 EVT ShiftVT = Op1.getValueType();
1544
1545 if (const APInt *SA =
1546 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1547 unsigned ShAmt = SA->getZExtValue();
1548 if (ShAmt == 0)
1549 return TLO.CombineTo(Op, Op0);
1550
1551 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1552 // single shift. We can do this if the top bits (which are shifted out)
1553 // are never demanded.
1554 // TODO - support non-uniform vector amounts.
1555 if (Op0.getOpcode() == ISD::SHL) {
1556 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1557 if (const APInt *SA2 =
1558 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1559 unsigned C1 = SA2->getZExtValue();
1560 unsigned Opc = ISD::SRL;
1561 int Diff = ShAmt - C1;
1562 if (Diff < 0) {
1563 Diff = -Diff;
1564 Opc = ISD::SHL;
1565 }
1566 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1567 return TLO.CombineTo(
1568 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1569 }
1570 }
1571 }
1572
1573 APInt InDemandedMask = (DemandedBits << ShAmt);
1574
1575 // If the shift is exact, then it does demand the low bits (and knows that
1576 // they are zero).
1577 if (Op->getFlags().hasExact())
1578 InDemandedMask.setLowBits(ShAmt);
1579
1580 // Compute the new bits that are at the top now.
1581 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1582 Depth + 1))
1583 return true;
1584 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1585 Known.Zero.lshrInPlace(ShAmt);
1586 Known.One.lshrInPlace(ShAmt);
1587 // High bits known zero.
1588 Known.Zero.setHighBits(ShAmt);
1589 }
1590 break;
1591 }
1592 case ISD::SRA: {
1593 SDValue Op0 = Op.getOperand(0);
1594 SDValue Op1 = Op.getOperand(1);
1595 EVT ShiftVT = Op1.getValueType();
1596
1597 // If we only want bits that already match the signbit then we don't need
1598 // to shift.
1599 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1600 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1601 NumHiDemandedBits)
1602 return TLO.CombineTo(Op, Op0);
1603
1604 // If this is an arithmetic shift right and only the low-bit is set, we can
1605 // always convert this into a logical shr, even if the shift amount is
1606 // variable. The low bit of the shift cannot be an input sign bit unless
1607 // the shift amount is >= the size of the datatype, which is undefined.
1608 if (DemandedBits.isOneValue())
1609 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1610
1611 if (const APInt *SA =
1612 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1613 unsigned ShAmt = SA->getZExtValue();
1614 if (ShAmt == 0)
1615 return TLO.CombineTo(Op, Op0);
1616
1617 APInt InDemandedMask = (DemandedBits << ShAmt);
1618
1619 // If the shift is exact, then it does demand the low bits (and knows that
1620 // they are zero).
1621 if (Op->getFlags().hasExact())
1622 InDemandedMask.setLowBits(ShAmt);
1623
1624 // If any of the demanded bits are produced by the sign extension, we also
1625 // demand the input sign bit.
1626 if (DemandedBits.countLeadingZeros() < ShAmt)
1627 InDemandedMask.setSignBit();
1628
1629 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1630 Depth + 1))
1631 return true;
1632 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1633 Known.Zero.lshrInPlace(ShAmt);
1634 Known.One.lshrInPlace(ShAmt);
1635
1636 // If the input sign bit is known to be zero, or if none of the top bits
1637 // are demanded, turn this into an unsigned shift right.
1638 if (Known.Zero[BitWidth - ShAmt - 1] ||
1639 DemandedBits.countLeadingZeros() >= ShAmt) {
1640 SDNodeFlags Flags;
1641 Flags.setExact(Op->getFlags().hasExact());
1642 return TLO.CombineTo(
1643 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1644 }
1645
1646 int Log2 = DemandedBits.exactLogBase2();
1647 if (Log2 >= 0) {
1648 // The bit must come from the sign.
1649 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1650 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1651 }
1652
1653 if (Known.One[BitWidth - ShAmt - 1])
1654 // New bits are known one.
1655 Known.One.setHighBits(ShAmt);
1656
1657 // Attempt to avoid multi-use ops if we don't need anything from them.
1658 if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1659 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1660 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1661 if (DemandedOp0) {
1662 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1663 return TLO.CombineTo(Op, NewOp);
1664 }
1665 }
1666 }
1667 break;
1668 }
1669 case ISD::FSHL:
1670 case ISD::FSHR: {
1671 SDValue Op0 = Op.getOperand(0);
1672 SDValue Op1 = Op.getOperand(1);
1673 SDValue Op2 = Op.getOperand(2);
1674 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1675
1676 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1677 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1678
1679 // For fshl, 0-shift returns the 1st arg.
1680 // For fshr, 0-shift returns the 2nd arg.
1681 if (Amt == 0) {
1682 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1683 Known, TLO, Depth + 1))
1684 return true;
1685 break;
1686 }
1687
1688 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1689 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1690 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1691 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1692 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1693 Depth + 1))
1694 return true;
1695 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1696 Depth + 1))
1697 return true;
1698
1699 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1700 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1701 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1702 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1703 Known.One |= Known2.One;
1704 Known.Zero |= Known2.Zero;
1705 }
1706
1707 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1708 if (isPowerOf2_32(BitWidth)) {
1709 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1710 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1711 Known2, TLO, Depth + 1))
1712 return true;
1713 }
1714 break;
1715 }
1716 case ISD::ROTL:
1717 case ISD::ROTR: {
1718 SDValue Op0 = Op.getOperand(0);
1719 SDValue Op1 = Op.getOperand(1);
1720
1721 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1722 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1723 return TLO.CombineTo(Op, Op0);
1724
1725 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1726 if (isPowerOf2_32(BitWidth)) {
1727 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1728 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1729 Depth + 1))
1730 return true;
1731 }
1732 break;
1733 }
1734 case ISD::UMIN: {
1735 // Check if one arg is always less than (or equal) to the other arg.
1736 SDValue Op0 = Op.getOperand(0);
1737 SDValue Op1 = Op.getOperand(1);
1738 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1739 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1740 Known = KnownBits::umin(Known0, Known1);
1741 if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1742 return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1743 if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1744 return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1745 break;
1746 }
1747 case ISD::UMAX: {
1748 // Check if one arg is always greater than (or equal) to the other arg.
1749 SDValue Op0 = Op.getOperand(0);
1750 SDValue Op1 = Op.getOperand(1);
1751 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1752 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1753 Known = KnownBits::umax(Known0, Known1);
1754 if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1755 return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1756 if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1757 return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1758 break;
1759 }
1760 case ISD::BITREVERSE: {
1761 SDValue Src = Op.getOperand(0);
1762 APInt DemandedSrcBits = DemandedBits.reverseBits();
1763 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1764 Depth + 1))
1765 return true;
1766 Known.One = Known2.One.reverseBits();
1767 Known.Zero = Known2.Zero.reverseBits();
1768 break;
1769 }
1770 case ISD::BSWAP: {
1771 SDValue Src = Op.getOperand(0);
1772 APInt DemandedSrcBits = DemandedBits.byteSwap();
1773 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1774 Depth + 1))
1775 return true;
1776 Known.One = Known2.One.byteSwap();
1777 Known.Zero = Known2.Zero.byteSwap();
1778 break;
1779 }
1780 case ISD::CTPOP: {
1781 // If only 1 bit is demanded, replace with PARITY as long as we're before
1782 // op legalization.
1783 // FIXME: Limit to scalars for now.
1784 if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())
1785 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1786 Op.getOperand(0)));
1787
1788 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1789 break;
1790 }
1791 case ISD::SIGN_EXTEND_INREG: {
1792 SDValue Op0 = Op.getOperand(0);
1793 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1794 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1795
1796 // If we only care about the highest bit, don't bother shifting right.
1797 if (DemandedBits.isSignMask()) {
1798 unsigned NumSignBits =
1799 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1800 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1801 // However if the input is already sign extended we expect the sign
1802 // extension to be dropped altogether later and do not simplify.
1803 if (!AlreadySignExtended) {
1804 // Compute the correct shift amount type, which must be getShiftAmountTy
1805 // for scalar types after legalization.
1806 EVT ShiftAmtTy = VT;
1807 if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1808 ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1809
1810 SDValue ShiftAmt =
1811 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1812 return TLO.CombineTo(Op,
1813 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1814 }
1815 }
1816
1817 // If none of the extended bits are demanded, eliminate the sextinreg.
1818 if (DemandedBits.getActiveBits() <= ExVTBits)
1819 return TLO.CombineTo(Op, Op0);
1820
1821 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1822
1823 // Since the sign extended bits are demanded, we know that the sign
1824 // bit is demanded.
1825 InputDemandedBits.setBit(ExVTBits - 1);
1826
1827 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1828 return true;
1829 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1830
1831 // If the sign bit of the input is known set or clear, then we know the
1832 // top bits of the result.
1833
1834 // If the input sign bit is known zero, convert this into a zero extension.
1835 if (Known.Zero[ExVTBits - 1])
1836 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1837
1838 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1839 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1840 Known.One.setBitsFrom(ExVTBits);
1841 Known.Zero &= Mask;
1842 } else { // Input sign bit unknown
1843 Known.Zero &= Mask;
1844 Known.One &= Mask;
1845 }
1846 break;
1847 }
1848 case ISD::BUILD_PAIR: {
1849 EVT HalfVT = Op.getOperand(0).getValueType();
1850 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1851
1852 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1853 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1854
1855 KnownBits KnownLo, KnownHi;
1856
1857 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1858 return true;
1859
1860 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1861 return true;
1862
1863 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1864 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1865
1866 Known.One = KnownLo.One.zext(BitWidth) |
1867 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1868 break;
1869 }
1870 case ISD::ZERO_EXTEND:
1871 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1872 SDValue Src = Op.getOperand(0);
1873 EVT SrcVT = Src.getValueType();
1874 unsigned InBits = SrcVT.getScalarSizeInBits();
1875 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1876 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1877
1878 // If none of the top bits are demanded, convert this into an any_extend.
1879 if (DemandedBits.getActiveBits() <= InBits) {
1880 // If we only need the non-extended bits of the bottom element
1881 // then we can just bitcast to the result.
1882 if (IsVecInReg && DemandedElts == 1 &&
1883 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1884 TLO.DAG.getDataLayout().isLittleEndian())
1885 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1886
1887 unsigned Opc =
1888 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1889 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1890 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1891 }
1892
1893 APInt InDemandedBits = DemandedBits.trunc(InBits);
1894 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1895 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1896 Depth + 1))
1897 return true;
1898 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1899 assert(Known.getBitWidth() == InBits && "Src width has changed?")((void)0);
1900 Known = Known.zext(BitWidth);
1901
1902 // Attempt to avoid multi-use ops if we don't need anything from them.
1903 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1904 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1905 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1906 break;
1907 }
1908 case ISD::SIGN_EXTEND:
1909 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1910 SDValue Src = Op.getOperand(0);
1911 EVT SrcVT = Src.getValueType();
1912 unsigned InBits = SrcVT.getScalarSizeInBits();
1913 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1914 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1915
1916 // If none of the top bits are demanded, convert this into an any_extend.
1917 if (DemandedBits.getActiveBits() <= InBits) {
1918 // If we only need the non-extended bits of the bottom element
1919 // then we can just bitcast to the result.
1920 if (IsVecInReg && DemandedElts == 1 &&
1921 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1922 TLO.DAG.getDataLayout().isLittleEndian())
1923 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1924
1925 unsigned Opc =
1926 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1927 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1928 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1929 }
1930
1931 APInt InDemandedBits = DemandedBits.trunc(InBits);
1932 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1933
1934 // Since some of the sign extended bits are demanded, we know that the sign
1935 // bit is demanded.
1936 InDemandedBits.setBit(InBits - 1);
1937
1938 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1939 Depth + 1))
1940 return true;
1941 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1942 assert(Known.getBitWidth() == InBits && "Src width has changed?")((void)0);
1943
1944 // If the sign bit is known one, the top bits match.
1945 Known = Known.sext(BitWidth);
1946
1947 // If the sign bit is known zero, convert this to a zero extend.
1948 if (Known.isNonNegative()) {
1949 unsigned Opc =
1950 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1951 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1952 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1953 }
1954
1955 // Attempt to avoid multi-use ops if we don't need anything from them.
1956 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1957 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1958 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1959 break;
1960 }
1961 case ISD::ANY_EXTEND:
1962 case ISD::ANY_EXTEND_VECTOR_INREG: {
1963 SDValue Src = Op.getOperand(0);
1964 EVT SrcVT = Src.getValueType();
1965 unsigned InBits = SrcVT.getScalarSizeInBits();
1966 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1967 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1968
1969 // If we only need the bottom element then we can just bitcast.
1970 // TODO: Handle ANY_EXTEND?
1971 if (IsVecInReg && DemandedElts == 1 &&
1972 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1973 TLO.DAG.getDataLayout().isLittleEndian())
1974 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1975
1976 APInt InDemandedBits = DemandedBits.trunc(InBits);
1977 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1978 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1979 Depth + 1))
1980 return true;
1981 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
1982 assert(Known.getBitWidth() == InBits && "Src width has changed?")((void)0);
1983 Known = Known.anyext(BitWidth);
1984
1985 // Attempt to avoid multi-use ops if we don't need anything from them.
1986 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1987 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1988 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1989 break;
1990 }
1991 case ISD::TRUNCATE: {
1992 SDValue Src = Op.getOperand(0);
1993
1994 // Simplify the input, using demanded bit information, and compute the known
1995 // zero/one bits live out.
1996 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1997 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1998 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
1999 Depth + 1))
2000 return true;
2001 Known = Known.trunc(BitWidth);
2002
2003 // Attempt to avoid multi-use ops if we don't need anything from them.
2004 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2005 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2006 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2007
2008 // If the input is only used by this truncate, see if we can shrink it based
2009 // on the known demanded bits.
2010 if (Src.getNode()->hasOneUse()) {
2011 switch (Src.getOpcode()) {
2012 default:
2013 break;
2014 case ISD::SRL:
2015 // Shrink SRL by a constant if none of the high bits shifted in are
2016 // demanded.
2017 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2018 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2019 // undesirable.
2020 break;
2021
2022 const APInt *ShAmtC =
2023 TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2024 if (!ShAmtC || ShAmtC->uge(BitWidth))
2025 break;
2026 uint64_t ShVal = ShAmtC->getZExtValue();
2027
2028 APInt HighBits =
2029 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2030 HighBits.lshrInPlace(ShVal);
2031 HighBits = HighBits.trunc(BitWidth);
2032
2033 if (!(HighBits & DemandedBits)) {
2034 // None of the shifted in bits are needed. Add a truncate of the
2035 // shift input, then shift it.
2036 SDValue NewShAmt = TLO.DAG.getConstant(
2037 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2038 SDValue NewTrunc =
2039 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2040 return TLO.CombineTo(
2041 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2042 }
2043 break;
2044 }
2045 }
2046
2047 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
2048 break;
2049 }
2050 case ISD::AssertZext: {
2051 // AssertZext demands all of the high bits, plus any of the low bits
2052 // demanded by its users.
2053 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2054 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
2055 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2056 TLO, Depth + 1))
2057 return true;
2058 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((void)0);
2059
2060 Known.Zero |= ~InMask;
2061 break;
2062 }
2063 case ISD::EXTRACT_VECTOR_ELT: {
2064 SDValue Src = Op.getOperand(0);
2065 SDValue Idx = Op.getOperand(1);
2066 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2067 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2068
2069 if (SrcEltCnt.isScalable())
2070 return false;
2071
2072 // Demand the bits from every vector element without a constant index.
2073 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2074 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
2075 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2076 if (CIdx->getAPIntValue().ult(NumSrcElts))
2077 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2078
2079 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2080 // anything about the extended bits.
2081 APInt DemandedSrcBits = DemandedBits;
2082 if (BitWidth > EltBitWidth)
2083 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2084
2085 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2086 Depth + 1))
2087 return true;
2088
2089 // Attempt to avoid multi-use ops if we don't need anything from them.
2090 if (!DemandedSrcBits.isAllOnesValue() ||
2091 !DemandedSrcElts.isAllOnesValue()) {
2092 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2093 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2094 SDValue NewOp =
2095 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2096 return TLO.CombineTo(Op, NewOp);
2097 }
2098 }
2099
2100 Known = Known2;
2101 if (BitWidth > EltBitWidth)
2102 Known = Known.anyext(BitWidth);
2103 break;
2104 }
2105 case ISD::BITCAST: {
2106 SDValue Src = Op.getOperand(0);
2107 EVT SrcVT = Src.getValueType();
2108 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2109
2110 // If this is an FP->Int bitcast and if the sign bit is the only
2111 // thing demanded, turn this into a FGETSIGN.
2112 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2113 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2114 SrcVT.isFloatingPoint()) {
2115 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2116 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2117 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2118 SrcVT != MVT::f128) {
2119 // Cannot eliminate/lower SHL for f128 yet.
2120 EVT Ty = OpVTLegal ? VT : MVT::i32;
2121 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2122 // place. We expect the SHL to be eliminated by other optimizations.
2123 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2124 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2125 if (!OpVTLegal && OpVTSizeInBits > 32)
2126 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2127 unsigned ShVal = Op.getValueSizeInBits() - 1;
2128 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2129 return TLO.CombineTo(Op,
2130 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2131 }
2132 }
2133
2134 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2135 // Demand the elt/bit if any of the original elts/bits are demanded.
2136 // TODO - bigendian once we have test coverage.
2137 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2138 TLO.DAG.getDataLayout().isLittleEndian()) {
2139 unsigned Scale = BitWidth / NumSrcEltBits;
2140 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2141 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2142 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2143 for (unsigned i = 0; i != Scale; ++i) {
2144 unsigned Offset = i * NumSrcEltBits;
2145 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2146 if (!Sub.isNullValue()) {
2147 DemandedSrcBits |= Sub;
2148 for (unsigned j = 0; j != NumElts; ++j)
2149 if (DemandedElts[j])
2150 DemandedSrcElts.setBit((j * Scale) + i);
2151 }
2152 }
2153
2154 APInt KnownSrcUndef, KnownSrcZero;
2155 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2156 KnownSrcZero, TLO, Depth + 1))
2157 return true;
2158
2159 KnownBits KnownSrcBits;
2160 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2161 KnownSrcBits, TLO, Depth + 1))
2162 return true;
2163 } else if ((NumSrcEltBits % BitWidth) == 0 &&
2164 TLO.DAG.getDataLayout().isLittleEndian()) {
2165 unsigned Scale = NumSrcEltBits / BitWidth;
2166 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2167 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2168 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2169 for (unsigned i = 0; i != NumElts; ++i)
2170 if (DemandedElts[i]) {
2171 unsigned Offset = (i % Scale) * BitWidth;
2172 DemandedSrcBits.insertBits(DemandedBits, Offset);
2173 DemandedSrcElts.setBit(i / Scale);
2174 }
2175
2176 if (SrcVT.isVector()) {
2177 APInt KnownSrcUndef, KnownSrcZero;
2178 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2179 KnownSrcZero, TLO, Depth + 1))
2180 return true;
2181 }
2182
2183 KnownBits KnownSrcBits;
2184 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2185 KnownSrcBits, TLO, Depth + 1))
2186 return true;
2187 }
2188
2189 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2190 // recursive call where Known may be useful to the caller.
2191 if (Depth > 0) {
2192 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2193 return false;
2194 }
2195 break;
2196 }
2197 case ISD::ADD:
2198 case ISD::MUL:
2199 case ISD::SUB: {
2200 // Add, Sub, and Mul don't demand any bits in positions beyond that
2201 // of the highest bit demanded of them.
2202 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2203 SDNodeFlags Flags = Op.getNode()->getFlags();
2204 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2205 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2206 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2207 Depth + 1) ||
2208 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2209 Depth + 1) ||
2210 // See if the operation should be performed at a smaller bit width.
2211 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2212 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2213 // Disable the nsw and nuw flags. We can no longer guarantee that we
2214 // won't wrap after simplification.
2215 Flags.setNoSignedWrap(false);
2216 Flags.setNoUnsignedWrap(false);
2217 SDValue NewOp =
2218 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2219 return TLO.CombineTo(Op, NewOp);
2220 }
2221 return true;
2222 }
2223
2224 // Attempt to avoid multi-use ops if we don't need anything from them.
2225 if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
2226 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2227 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2228 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2229 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2230 if (DemandedOp0 || DemandedOp1) {
2231 Flags.setNoSignedWrap(false);
2232 Flags.setNoUnsignedWrap(false);
2233 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2234 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2235 SDValue NewOp =
2236 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2237 return TLO.CombineTo(Op, NewOp);
2238 }
2239 }
2240
2241 // If we have a constant operand, we may be able to turn it into -1 if we
2242 // do not demand the high bits. This can make the constant smaller to
2243 // encode, allow more general folding, or match specialized instruction
2244 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2245 // is probably not useful (and could be detrimental).
2246 ConstantSDNode *C = isConstOrConstSplat(Op1);
2247 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2248 if (C && !C->isAllOnesValue() && !C->isOne() &&
2249 (C->getAPIntValue() | HighMask).isAllOnesValue()) {
2250 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2251 // Disable the nsw and nuw flags. We can no longer guarantee that we
2252 // won't wrap after simplification.
2253 Flags.setNoSignedWrap(false);
2254 Flags.setNoUnsignedWrap(false);
2255 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2256 return TLO.CombineTo(Op, NewOp);
2257 }
2258
2259 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2260 }
2261 default:
2262 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2263 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2264 Known, TLO, Depth))
2265 return true;
2266 break;
2267 }
2268
2269 // Just use computeKnownBits to compute output bits.
2270 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2271 break;
2272 }
2273
2274 // If we know the value of all of the demanded bits, return this as a
2275 // constant.
2276 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2277 // Avoid folding to a constant if any OpaqueConstant is involved.
2278 const SDNode *N = Op.getNode();
2279 for (SDNode *Op :
2280 llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
2281 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2282 if (C->isOpaque())
2283 return false;
2284 }
2285 if (VT.isInteger())
2286 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2287 if (VT.isFloatingPoint())
2288 return TLO.CombineTo(
2289 Op,
2290 TLO.DAG.getConstantFP(
2291 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2292 }
2293
2294 return false;
2295}
2296
2297bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2298 const APInt &DemandedElts,
2299 APInt &KnownUndef,
2300 APInt &KnownZero,
2301 DAGCombinerInfo &DCI) const {
2302 SelectionDAG &DAG = DCI.DAG;
2303 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2304 !DCI.isBeforeLegalizeOps());
2305
2306 bool Simplified =
2307 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2308 if (Simplified) {
2309 DCI.AddToWorklist(Op.getNode());
2310 DCI.CommitTargetLoweringOpt(TLO);
2311 }
2312
2313 return Simplified;
2314}
2315
2316/// Given a vector binary operation and known undefined elements for each input
2317/// operand, compute whether each element of the output is undefined.
2318static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2319 const APInt &UndefOp0,
2320 const APInt &UndefOp1) {
2321 EVT VT = BO.getValueType();
2322 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&((void)0)
2323 "Vector binop only")((void)0);
2324
2325 EVT EltVT = VT.getVectorElementType();
2326 unsigned NumElts = VT.getVectorNumElements();
2327 assert(UndefOp0.getBitWidth() == NumElts &&((void)0)
2328 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")((void)0);
2329
2330 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2331 const APInt &UndefVals) {
2332 if (UndefVals[Index])
2333 return DAG.getUNDEF(EltVT);
2334
2335 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2336 // Try hard to make sure that the getNode() call is not creating temporary
2337 // nodes. Ignore opaque integers because they do not constant fold.
2338 SDValue Elt = BV->getOperand(Index);
2339 auto *C = dyn_cast<ConstantSDNode>(Elt);
2340 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2341 return Elt;
2342 }
2343
2344 return SDValue();
2345 };
2346
2347 APInt KnownUndef = APInt::getNullValue(NumElts);
2348 for (unsigned i = 0; i != NumElts; ++i) {
2349 // If both inputs for this element are either constant or undef and match
2350 // the element type, compute the constant/undef result for this element of
2351 // the vector.
2352 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2353 // not handle FP constants. The code within getNode() should be refactored
2354 // to avoid the danger of creating a bogus temporary node here.
2355 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2356 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2357 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2358 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2359 KnownUndef.setBit(i);
2360 }
2361 return KnownUndef;
2362}
2363
2364bool TargetLowering::SimplifyDemandedVectorElts(
2365 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2366 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2367 bool AssumeSingleUse) const {
2368 EVT VT = Op.getValueType();
2369 unsigned Opcode = Op.getOpcode();
2370 APInt DemandedElts = OriginalDemandedElts;
2371 unsigned NumElts = DemandedElts.getBitWidth();
2372 assert(VT.isVector() && "Expected vector op")((void)0);
2373
2374 KnownUndef = KnownZero = APInt::getNullValue(NumElts);
2375
2376 // TODO: For now we assume we know nothing about scalable vectors.
2377 if (VT.isScalableVector())
2378 return false;
2379
2380 assert(VT.getVectorNumElements() == NumElts &&((void)0)
2381 "Mask size mismatches value type element count!")((void)0);
2382
2383 // Undef operand.
2384 if (Op.isUndef()) {
2385 KnownUndef.setAllBits();
2386 return false;
2387 }
2388
2389 // If Op has other users, assume that all elements are needed.
2390 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2391 DemandedElts.setAllBits();
2392
2393 // Not demanding any elements from Op.
2394 if (DemandedElts == 0) {
2395 KnownUndef.setAllBits();
2396 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2397 }
2398
2399 // Limit search depth.
2400 if (Depth >= SelectionDAG::MaxRecursionDepth)
2401 return false;
2402
2403 SDLoc DL(Op);
2404 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2405
2406 // Helper for demanding the specified elements and all the bits of both binary
2407 // operands.
2408 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2409 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2410 TLO.DAG, Depth + 1);
2411 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2412 TLO.DAG, Depth + 1);
2413 if (NewOp0 || NewOp1) {
2414 SDValue NewOp = TLO.DAG.getNode(
2415 Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2416 return TLO.CombineTo(Op, NewOp);
2417 }
2418 return false;
2419 };
2420
2421 switch (Opcode) {
2422 case ISD::SCALAR_TO_VECTOR: {
2423 if (!DemandedElts[0]) {
2424 KnownUndef.setAllBits();
2425 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2426 }
2427 SDValue ScalarSrc = Op.getOperand(0);
2428 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2429 SDValue Src = ScalarSrc.getOperand(0);
2430 SDValue Idx = ScalarSrc.getOperand(1);
2431 EVT SrcVT = Src.getValueType();
2432
2433 ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2434
2435 if (SrcEltCnt.isScalable())
2436 return false;
2437
2438 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2439 if (isNullConstant(Idx)) {
2440 APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2441 APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2442 APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2443 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2444 TLO, Depth + 1))
2445 return true;
2446 }
2447 }
2448 KnownUndef.setHighBits(NumElts - 1);
2449 break;
2450 }
2451 case ISD::BITCAST: {
2452 SDValue Src = Op.getOperand(0);
2453 EVT SrcVT = Src.getValueType();
2454
2455 // We only handle vectors here.
2456 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2457 if (!SrcVT.isVector())
2458 break;
2459
2460 // Fast handling of 'identity' bitcasts.
2461 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2462 if (NumSrcElts == NumElts)
2463 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2464 KnownZero, TLO, Depth + 1);
2465
2466 APInt SrcZero, SrcUndef;
2467 APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
2468
2469 // Bitcast from 'large element' src vector to 'small element' vector, we
2470 // must demand a source element if any DemandedElt maps to it.
2471 if ((NumElts % NumSrcElts) == 0) {
2472 unsigned Scale = NumElts / NumSrcElts;
2473 for (unsigned i = 0; i != NumElts; ++i)
2474 if (DemandedElts[i])
2475 SrcDemandedElts.setBit(i / Scale);
2476
2477 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2478 TLO, Depth + 1))
2479 return true;
2480
2481 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2482 // of the large element.
2483 // TODO - bigendian once we have test coverage.
2484 if (TLO.DAG.getDataLayout().isLittleEndian()) {
2485 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2486 APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
2487 for (unsigned i = 0; i != NumElts; ++i)
2488 if (DemandedElts[i]) {
2489 unsigned Ofs = (i % Scale) * EltSizeInBits;
2490 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2491 }
2492
2493 KnownBits Known;
2494 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2495 TLO, Depth + 1))
2496 return true;
2497 }
2498
2499 // If the src element is zero/undef then all the output elements will be -
2500 // only demanded elements are guaranteed to be correct.
2501 for (unsigned i = 0; i != NumSrcElts; ++i) {
2502 if (SrcDemandedElts[i]) {
2503 if (SrcZero[i])
2504 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2505 if (SrcUndef[i])
2506 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2507 }
2508 }
2509 }
2510
2511 // Bitcast from 'small element' src vector to 'large element' vector, we
2512 // demand all smaller source elements covered by the larger demanded element
2513 // of this vector.
2514 if ((NumSrcElts % NumElts) == 0) {
2515 unsigned Scale = NumSrcElts / NumElts;
2516 for (unsigned i = 0; i != NumElts; ++i)
2517 if (DemandedElts[i])
2518 SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
2519
2520 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2521 TLO, Depth + 1))
2522 return true;
2523
2524 // If all the src elements covering an output element are zero/undef, then
2525 // the output element will be as well, assuming it was demanded.
2526 for (unsigned i = 0; i != NumElts; ++i) {
2527 if (DemandedElts[i]) {
2528 if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
2529 KnownZero.setBit(i);
2530 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
2531 KnownUndef.setBit(i);
2532 }
2533 }
2534 }
2535 break;
2536 }
2537 case ISD::BUILD_VECTOR: {
2538 // Check all elements and simplify any unused elements with UNDEF.
2539 if (!DemandedElts.isAllOnesValue()) {
2540 // Don't simplify BROADCASTS.
2541 if (llvm::any_of(Op->op_values(),
2542 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2543 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2544 bool Updated = false;
2545 for (unsigned i = 0; i != NumElts; ++i) {
2546 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2547 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2548 KnownUndef.setBit(i);
2549 Updated = true;
2550 }
2551 }
2552 if (Updated)
2553 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2554 }
2555 }
2556 for (unsigned i = 0; i != NumElts; ++i) {
2557 SDValue SrcOp = Op.getOperand(i);
2558 if (SrcOp.isUndef()) {
2559 KnownUndef.setBit(i);
2560 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2561 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2562 KnownZero.setBit(i);
2563 }
2564 }
2565 break;
2566 }
2567 case ISD::CONCAT_VECTORS: {
2568 EVT SubVT = Op.getOperand(0).getValueType();
2569 unsigned NumSubVecs = Op.getNumOperands();
2570 unsigned NumSubElts = SubVT.getVectorNumElements();
2571 for (unsigned i = 0; i != NumSubVecs; ++i) {
2572 SDValue SubOp = Op.getOperand(i);
2573 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2574 APInt SubUndef, SubZero;
2575 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2576 Depth + 1))
2577 return true;
2578 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2579 KnownZero.insertBits(SubZero, i * NumSubElts);
2580 }
2581 break;
2582 }
2583 case ISD::INSERT_SUBVECTOR: {
2584 // Demand any elements from the subvector and the remainder from the src its
2585 // inserted into.
2586 SDValue Src = Op.getOperand(0);
2587 SDValue Sub = Op.getOperand(1);
2588 uint64_t Idx = Op.getConstantOperandVal(2);
2589 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2590 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2591 APInt DemandedSrcElts = DemandedElts;
2592 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2593
2594 APInt SubUndef, SubZero;
2595 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2596 Depth + 1))
2597 return true;
2598
2599 // If none of the src operand elements are demanded, replace it with undef.
2600 if (!DemandedSrcElts && !Src.isUndef())
2601 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2602 TLO.DAG.getUNDEF(VT), Sub,
2603 Op.getOperand(2)));
2604
2605 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2606 TLO, Depth + 1))
2607 return true;
2608 KnownUndef.insertBits(SubUndef, Idx);
2609 KnownZero.insertBits(SubZero, Idx);
2610
2611 // Attempt to avoid multi-use ops if we don't need anything from them.
2612 if (!DemandedSrcElts.isAllOnesValue() ||
2613 !DemandedSubElts.isAllOnesValue()) {
2614 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2615 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2616 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2617 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2618 if (NewSrc || NewSub) {
2619 NewSrc = NewSrc ? NewSrc : Src;
2620 NewSub = NewSub ? NewSub : Sub;
2621 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2622 NewSub, Op.getOperand(2));
2623 return TLO.CombineTo(Op, NewOp);
2624 }
2625 }
2626 break;
2627 }
2628 case ISD::EXTRACT_SUBVECTOR: {
2629 // Offset the demanded elts by the subvector index.
2630 SDValue Src = Op.getOperand(0);
2631 if (Src.getValueType().isScalableVector())
2632 break;
2633 uint64_t Idx = Op.getConstantOperandVal(1);
2634 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2635 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2636
2637 APInt SrcUndef, SrcZero;
2638 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2639 Depth + 1))
2640 return true;
2641 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2642 KnownZero = SrcZero.extractBits(NumElts, Idx);
2643
2644 // Attempt to avoid multi-use ops if we don't need anything from them.
2645 if (!DemandedElts.isAllOnesValue()) {
2646 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2647 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2648 if (NewSrc) {
2649 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2650 Op.getOperand(1));
2651 return TLO.CombineTo(Op, NewOp);
2652 }
2653 }
2654 break;
2655 }
2656 case ISD::INSERT_VECTOR_ELT: {
2657 SDValue Vec = Op.getOperand(0);
2658 SDValue Scl = Op.getOperand(1);
2659 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2660
2661 // For a legal, constant insertion index, if we don't need this insertion
2662 // then strip it, else remove it from the demanded elts.
2663 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2664 unsigned Idx = CIdx->getZExtValue();
2665 if (!DemandedElts[Idx])
2666 return TLO.CombineTo(Op, Vec);
2667
2668 APInt DemandedVecElts(DemandedElts);
2669 DemandedVecElts.clearBit(Idx);
2670 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2671 KnownZero, TLO, Depth + 1))
2672 return true;
2673
2674 KnownUndef.setBitVal(Idx, Scl.isUndef());
2675
2676 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2677 break;
2678 }
2679
2680 APInt VecUndef, VecZero;
2681 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2682 Depth + 1))
2683 return true;
2684 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2685 break;
2686 }
2687 case ISD::VSELECT: {
2688 // Try to transform the select condition based on the current demanded
2689 // elements.
2690 // TODO: If a condition element is undef, we can choose from one arm of the
2691 // select (and if one arm is undef, then we can propagate that to the
2692 // result).
2693 // TODO - add support for constant vselect masks (see IR version of this).
2694 APInt UnusedUndef, UnusedZero;
2695 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2696 UnusedZero, TLO, Depth + 1))
2697 return true;
2698
2699 // See if we can simplify either vselect operand.
2700 APInt DemandedLHS(DemandedElts);
2701 APInt DemandedRHS(DemandedElts);
2702 APInt UndefLHS, ZeroLHS;
2703 APInt UndefRHS, ZeroRHS;
2704 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2705 ZeroLHS, TLO, Depth + 1))
2706 return true;
2707 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2708 ZeroRHS, TLO, Depth + 1))
2709 return true;
2710
2711 KnownUndef = UndefLHS & UndefRHS;
2712 KnownZero = ZeroLHS & ZeroRHS;
2713 break;
2714 }
2715 case ISD::VECTOR_SHUFFLE: {
2716 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2717
2718 // Collect demanded elements from shuffle operands..
2719 APInt DemandedLHS(NumElts, 0);
2720 APInt DemandedRHS(NumElts, 0);
2721 for (unsigned i = 0; i != NumElts; ++i) {
2722 int M = ShuffleMask[i];
2723 if (M < 0 || !DemandedElts[i])
2724 continue;
2725 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((void)0);
2726 if (M < (int)NumElts)
2727 DemandedLHS.setBit(M);
2728 else
2729 DemandedRHS.setBit(M - NumElts);
2730 }
2731
2732 // See if we can simplify either shuffle operand.
2733 APInt UndefLHS, ZeroLHS;
2734 APInt UndefRHS, ZeroRHS;
2735 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2736 ZeroLHS, TLO, Depth + 1))
2737 return true;
2738 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2739 ZeroRHS, TLO, Depth + 1))
2740 return true;
2741
2742 // Simplify mask using undef elements from LHS/RHS.
2743 bool Updated = false;
2744 bool IdentityLHS = true, IdentityRHS = true;
2745 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2746 for (unsigned i = 0; i != NumElts; ++i) {
2747 int &M = NewMask[i];
2748 if (M < 0)
2749 continue;
2750 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2751 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2752 Updated = true;
2753 M = -1;
2754 }
2755 IdentityLHS &= (M < 0) || (M == (int)i);
2756 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2757 }
2758
2759 // Update legal shuffle masks based on demanded elements if it won't reduce
2760 // to Identity which can cause premature removal of the shuffle mask.
2761 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2762 SDValue LegalShuffle =
2763 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2764 NewMask, TLO.DAG);
2765 if (LegalShuffle)
2766 return TLO.CombineTo(Op, LegalShuffle);
2767 }
2768
2769 // Propagate undef/zero elements from LHS/RHS.
2770 for (unsigned i = 0; i != NumElts; ++i) {
2771 int M = ShuffleMask[i];
2772 if (M < 0) {
2773 KnownUndef.setBit(i);
2774 } else if (M < (int)NumElts) {
2775 if (UndefLHS[M])
2776 KnownUndef.setBit(i);
2777 if (ZeroLHS[M])
2778 KnownZero.setBit(i);
2779 } else {
2780 if (UndefRHS[M - NumElts])
2781 KnownUndef.setBit(i);
2782 if (ZeroRHS[M - NumElts])
2783 KnownZero.setBit(i);
2784 }
2785 }
2786 break;
2787 }
2788 case ISD::ANY_EXTEND_VECTOR_INREG:
2789 case ISD::SIGN_EXTEND_VECTOR_INREG:
2790 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2791 APInt SrcUndef, SrcZero;
2792 SDValue Src = Op.getOperand(0);
2793 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2794 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2795 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2796 Depth + 1))
2797 return true;
2798 KnownZero = SrcZero.zextOrTrunc(NumElts);
2799 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2800
2801 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2802 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2803 DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2804 // aext - if we just need the bottom element then we can bitcast.
2805 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2806 }
2807
2808 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2809 // zext(undef) upper bits are guaranteed to be zero.
2810 if (DemandedElts.isSubsetOf(KnownUndef))
2811 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2812 KnownUndef.clearAllBits();
2813 }
2814 break;
2815 }
2816
2817 // TODO: There are more binop opcodes that could be handled here - MIN,
2818 // MAX, saturated math, etc.
2819 case ISD::OR:
2820 case ISD::XOR:
2821 case ISD::ADD:
2822 case ISD::SUB:
2823 case ISD::FADD:
2824 case ISD::FSUB:
2825 case ISD::FMUL:
2826 case ISD::FDIV:
2827 case ISD::FREM: {
2828 SDValue Op0 = Op.getOperand(0);
2829 SDValue Op1 = Op.getOperand(1);
2830
2831 APInt UndefRHS, ZeroRHS;
2832 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2833 Depth + 1))
2834 return true;
2835 APInt UndefLHS, ZeroLHS;
2836 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2837 Depth + 1))
2838 return true;
2839
2840 KnownZero = ZeroLHS & ZeroRHS;
2841 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2842
2843 // Attempt to avoid multi-use ops if we don't need anything from them.
2844 // TODO - use KnownUndef to relax the demandedelts?
2845 if (!DemandedElts.isAllOnesValue())
2846 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2847 return true;
2848 break;
2849 }
2850 case ISD::SHL:
2851 case ISD::SRL:
2852 case ISD::SRA:
2853 case ISD::ROTL:
2854 case ISD::ROTR: {
2855 SDValue Op0 = Op.getOperand(0);
2856 SDValue Op1 = Op.getOperand(1);
2857
2858 APInt UndefRHS, ZeroRHS;
2859 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2860 Depth + 1))
2861 return true;
2862 APInt UndefLHS, ZeroLHS;
2863 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2864 Depth + 1))
2865 return true;
2866
2867 KnownZero = ZeroLHS;
2868 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2869
2870 // Attempt to avoid multi-use ops if we don't need anything from them.
2871 // TODO - use KnownUndef to relax the demandedelts?
2872 if (!DemandedElts.isAllOnesValue())
2873 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2874 return true;
2875 break;
2876 }
2877 case ISD::MUL:
2878 case ISD::AND: {
2879 SDValue Op0 = Op.getOperand(0);
2880 SDValue Op1 = Op.getOperand(1);
2881
2882 APInt SrcUndef, SrcZero;
2883 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2884 Depth + 1))
2885 return true;
2886 if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2887 TLO, Depth + 1))
2888 return true;
2889
2890 // If either side has a zero element, then the result element is zero, even
2891 // if the other is an UNDEF.
2892 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2893 // and then handle 'and' nodes with the rest of the binop opcodes.
2894 KnownZero |= SrcZero;
2895 KnownUndef &= SrcUndef;
2896 KnownUndef &= ~KnownZero;
2897
2898 // Attempt to avoid multi-use ops if we don't need anything from them.
2899 // TODO - use KnownUndef to relax the demandedelts?
2900 if (!DemandedElts.isAllOnesValue())
2901 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2902 return true;
2903 break;
2904 }
2905 case ISD::TRUNCATE:
2906 case ISD::SIGN_EXTEND:
2907 case ISD::ZERO_EXTEND:
2908 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2909 KnownZero, TLO, Depth + 1))
2910 return true;
2911
2912 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2913 // zext(undef) upper bits are guaranteed to be zero.
2914 if (DemandedElts.isSubsetOf(KnownUndef))
2915 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2916 KnownUndef.clearAllBits();
2917 }
2918 break;
2919 default: {
2920 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2921 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2922 KnownZero, TLO, Depth))
2923 return true;
2924 } else {
2925 KnownBits Known;
2926 APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2927 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2928 TLO, Depth, AssumeSingleUse))
2929 return true;
2930 }
2931 break;
2932 }
2933 }
2934 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")((void)0);
2935
2936 // Constant fold all undef cases.
2937 // TODO: Handle zero cases as well.
2938 if (DemandedElts.isSubsetOf(KnownUndef))
2939 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2940
2941 return false;
2942}
2943
2944/// Determine which of the bits specified in Mask are known to be either zero or
2945/// one and return them in the Known.
2946void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2947 KnownBits &Known,
2948 const APInt &DemandedElts,
2949 const SelectionDAG &DAG,
2950 unsigned Depth) const {
2951 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
2952 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
2953 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
2954 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
2955 "Should use MaskedValueIsZero if you don't know whether Op"((void)0)
2956 " is a target node!")((void)0);
2957 Known.resetAll();
2958}
2959
2960void TargetLowering::computeKnownBitsForTargetInstr(
2961 GISelKnownBits &Analysis, Register R, KnownBits &Known,
2962 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2963 unsigned Depth) const {
2964 Known.resetAll();
2965}
2966
2967void TargetLowering::computeKnownBitsForFrameIndex(
2968 const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
2969 // The low bits are known zero if the pointer is aligned.
2970 Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2971}
2972
2973Align TargetLowering::computeKnownAlignForTargetInstr(
2974 GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
2975 unsigned Depth) const {
2976 return Align(1);
2977}
2978
2979/// This method can be implemented by targets that want to expose additional
2980/// information about sign bits to the DAG Combiner.
2981unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2982 const APInt &,
2983 const SelectionDAG &,
2984 unsigned Depth) const {
2985 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
2986 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
2987 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
2988 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
2989 "Should use ComputeNumSignBits if you don't know whether Op"((void)0)
2990 " is a target node!")((void)0);
2991 return 1;
2992}
2993
2994unsigned TargetLowering::computeNumSignBitsForTargetInstr(
2995 GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
2996 const MachineRegisterInfo &MRI, unsigned Depth) const {
2997 return 1;
2998}
2999
3000bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
3001 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3002 TargetLoweringOpt &TLO, unsigned Depth) const {
3003 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3004 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3005 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3006 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3007 "Should use SimplifyDemandedVectorElts if you don't know whether Op"((void)0)
3008 " is a target node!")((void)0);
3009 return false;
3010}
3011
3012bool TargetLowering::SimplifyDemandedBitsForTargetNode(
3013 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3014 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3015 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3016 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3017 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3018 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3019 "Should use SimplifyDemandedBits if you don't know whether Op"((void)0)
3020 " is a target node!")((void)0);
3021 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3022 return false;
3023}
3024
3025SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
3026 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3027 SelectionDAG &DAG, unsigned Depth) const {
3028 assert(((void)0)
3029 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3030 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3031 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3032 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3033 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"((void)0)
3034 " is a target node!")((void)0);
3035 return SDValue();
3036}
3037
3038SDValue
3039TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3040 SDValue N1, MutableArrayRef<int> Mask,
3041 SelectionDAG &DAG) const {
3042 bool LegalMask = isShuffleMaskLegal(Mask, VT);
3043 if (!LegalMask) {
3044 std::swap(N0, N1);
3045 ShuffleVectorSDNode::commuteMask(Mask);
3046 LegalMask = isShuffleMaskLegal(Mask, VT);
3047 }
3048
3049 if (!LegalMask)
3050 return SDValue();
3051
3052 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3053}
3054
3055const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
3056 return nullptr;
3057}
3058
3059bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
3060 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3061 bool PoisonOnly, unsigned Depth) const {
3062 assert(((void)0)
3063 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3064 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3065 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3066 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3067 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"((void)0)
3068 " is a target node!")((void)0);
3069 return false;
3070}
3071
3072bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3073 const SelectionDAG &DAG,
3074 bool SNaN,
3075 unsigned Depth) const {
3076 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||((void)0)
3077 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||((void)0)
3078 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||((void)0)
3079 Op.getOpcode() == ISD::INTRINSIC_VOID) &&((void)0)
3080 "Should use isKnownNeverNaN if you don't know whether Op"((void)0)
3081 " is a target node!")((void)0);
3082 return false;
3083}
3084
3085// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3086// work with truncating build vectors and vectors with elements of less than
3087// 8 bits.
3088bool TargetLowering::isConstTrueVal(const SDNode *N) const {
3089 if (!N)
3090 return false;
3091
3092 APInt CVal;
3093 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3094 CVal = CN->getAPIntValue();
3095 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3096 auto *CN = BV->getConstantSplatNode();
3097 if (!CN)
3098 return false;
3099
3100 // If this is a truncating build vector, truncate the splat value.
3101 // Otherwise, we may fail to match the expected values below.
3102 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3103 CVal = CN->getAPIntValue();
3104 if (BVEltWidth < CVal.getBitWidth())
3105 CVal = CVal.trunc(BVEltWidth);
3106 } else {
3107 return false;
3108 }
3109
3110 switch (getBooleanContents(N->getValueType(0))) {
3111 case UndefinedBooleanContent:
3112 return CVal[0];
3113 case ZeroOrOneBooleanContent:
3114 return CVal.isOneValue();
3115 case ZeroOrNegativeOneBooleanContent:
3116 return CVal.isAllOnesValue();
3117 }
3118
3119 llvm_unreachable("Invalid boolean contents")__builtin_unreachable();
3120}
3121
3122bool TargetLowering::isConstFalseVal(const SDNode *N) const {
3123 if (!N)
3124 return false;
3125
3126 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3127 if (!CN) {
3128 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3129 if (!BV)
3130 return false;
3131
3132 // Only interested in constant splats, we don't care about undef
3133 // elements in identifying boolean constants and getConstantSplatNode
3134 // returns NULL if all ops are undef;
3135 CN = BV->getConstantSplatNode();
3136 if (!CN)
3137 return false;
3138 }
3139
3140 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3141 return !CN->getAPIntValue()[0];
3142
3143 return CN->isNullValue();
3144}
3145
3146bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
3147 bool SExt) const {
3148 if (VT == MVT::i1)
3149 return N->isOne();
3150
3151 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
3152 switch (Cnt) {
3153 case TargetLowering::ZeroOrOneBooleanContent:
3154 // An extended value of 1 is always true, unless its original type is i1,
3155 // in which case it will be sign extended to -1.
3156 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3157 case TargetLowering::UndefinedBooleanContent:
3158 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3159 return N->isAllOnesValue() && SExt;
3160 }
3161 llvm_unreachable("Unexpected enumeration.")__builtin_unreachable();
3162}
3163
3164/// This helper function of SimplifySetCC tries to optimize the comparison when
3165/// either operand of the SetCC node is a bitwise-and instruction.
3166SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3167 ISD::CondCode Cond, const SDLoc &DL,
3168 DAGCombinerInfo &DCI) const {
3169 // Match these patterns in any of their permutations:
3170 // (X & Y) == Y
3171 // (X & Y) != Y
3172 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3173 std::swap(N0, N1);
3174
3175 EVT OpVT = N0.getValueType();
3176 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3177 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3178 return SDValue();
3179
3180 SDValue X, Y;
3181 if (N0.getOperand(0) == N1) {
3182 X = N0.getOperand(1);
3183 Y = N0.getOperand(0);
3184 } else if (N0.getOperand(1) == N1) {
3185 X = N0.getOperand(0);
3186 Y = N0.getOperand(1);
3187 } else {
3188 return SDValue();
3189 }
3190
3191 SelectionDAG &DAG = DCI.DAG;
3192 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3193 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3194 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3195 // Note that where Y is variable and is known to have at most one bit set
3196 // (for example, if it is Z & 1) we cannot do this; the expressions are not
3197 // equivalent when Y == 0.
3198 assert(OpVT.isInteger())((void)0);
3199 Cond = ISD::getSetCCInverse(Cond, OpVT);
3200 if (DCI.isBeforeLegalizeOps() ||
3201 isCondCodeLegal(Cond, N0.getSimpleValueType()))
3202 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3203 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3204 // If the target supports an 'and-not' or 'and-complement' logic operation,
3205 // try to use that to make a comparison operation more efficient.
3206 // But don't do this transform if the mask is a single bit because there are
3207 // more efficient ways to deal with that case (for example, 'bt' on x86 or
3208 // 'rlwinm' on PPC).
3209
3210 // Bail out if the compare operand that we want to turn into a zero is
3211 // already a zero (otherwise, infinite loop).
3212 auto *YConst = dyn_cast<ConstantSDNode>(Y);
3213 if (YConst && YConst->isNullValue())
3214 return SDValue();
3215
3216 // Transform this into: ~X & Y == 0.
3217 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3218 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3219 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3220 }
3221
3222 return SDValue();
3223}
3224
3225/// There are multiple IR patterns that could be checking whether certain
3226/// truncation of a signed number would be lossy or not. The pattern which is
3227/// best at IR level, may not lower optimally. Thus, we want to unfold it.
3228/// We are looking for the following pattern: (KeptBits is a constant)
3229/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3230/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3231/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3232/// We will unfold it into the natural trunc+sext pattern:
3233/// ((%x << C) a>> C) dstcond %x
3234/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3235SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3236 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3237 const SDLoc &DL) const {
3238 // We must be comparing with a constant.
3239 ConstantSDNode *C1;
3240 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3241 return SDValue();
3242
3243 // N0 should be: add %x, (1 << (KeptBits-1))
3244 if (N0->getOpcode() != ISD::ADD)
3245 return SDValue();
3246
3247 // And we must be 'add'ing a constant.
3248 ConstantSDNode *C01;
3249 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3250 return SDValue();
3251
3252 SDValue X = N0->getOperand(0);
3253 EVT XVT = X.getValueType();
3254
3255 // Validate constants ...
3256
3257 APInt I1 = C1->getAPIntValue();
3258
3259 ISD::CondCode NewCond;
3260 if (Cond == ISD::CondCode::SETULT) {
3261 NewCond = ISD::CondCode::SETEQ;
3262 } else if (Cond == ISD::CondCode::SETULE) {
3263 NewCond = ISD::CondCode::SETEQ;
3264 // But need to 'canonicalize' the constant.
3265 I1 += 1;
3266 } else if (Cond == ISD::CondCode::SETUGT) {
3267 NewCond = ISD::CondCode::SETNE;
3268 // But need to 'canonicalize' the constant.
3269 I1 += 1;
3270 } else if (Cond == ISD::CondCode::SETUGE) {
3271 NewCond = ISD::CondCode::SETNE;
3272 } else
3273 return SDValue();
3274
3275 APInt I01 = C01->getAPIntValue();
3276
3277 auto checkConstants = [&I1, &I01]() -> bool {
3278 // Both of them must be power-of-two, and the constant from setcc is bigger.
3279 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3280 };
3281
3282 if (checkConstants()) {
3283 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3284 } else {
3285 // What if we invert constants? (and the target predicate)
3286 I1.negate();
3287 I01.negate();
3288 assert(XVT.isInteger())((void)0);
3289 NewCond = getSetCCInverse(NewCond, XVT);
3290 if (!checkConstants())
3291 return SDValue();
3292 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3293 }
3294
3295 // They are power-of-two, so which bit is set?
3296 const unsigned KeptBits = I1.logBase2();
3297 const unsigned KeptBitsMinusOne = I01.logBase2();
3298
3299 // Magic!
3300 if (KeptBits != (KeptBitsMinusOne + 1))
3301 return SDValue();
3302 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")((void)0);
3303
3304 // We don't want to do this in every single case.
3305 SelectionDAG &DAG = DCI.DAG;
3306 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3307 XVT, KeptBits))
3308 return SDValue();
3309
3310 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3311 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")((void)0);
3312
3313 // Unfold into: ((%x << C) a>> C) cond %x
3314 // Where 'cond' will be either 'eq' or 'ne'.
3315 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3316 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3317 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3318 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3319
3320 return T2;
3321}
3322
3323// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3324SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3325 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3326 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3327 assert(isConstOrConstSplat(N1C) &&((void)0)
3328 isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&((void)0)
3329 "Should be a comparison with 0.")((void)0);
3330 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&((void)0)
3331 "Valid only for [in]equality comparisons.")((void)0);
3332
3333 unsigned NewShiftOpcode;
3334 SDValue X, C, Y;
3335
3336 SelectionDAG &DAG = DCI.DAG;
3337 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3338
3339 // Look for '(C l>>/<< Y)'.
3340 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3341 // The shift should be one-use.
3342 if (!V.hasOneUse())
3343 return false;
3344 unsigned OldShiftOpcode = V.getOpcode();
3345 switch (OldShiftOpcode) {
3346 case ISD::SHL:
3347 NewShiftOpcode = ISD::SRL;
3348 break;
3349 case ISD::SRL:
3350 NewShiftOpcode = ISD::SHL;
3351 break;
3352 default:
3353 return false; // must be a logical shift.
3354 }
3355 // We should be shifting a constant.
3356 // FIXME: best to use isConstantOrConstantVector().
3357 C = V.getOperand(0);
3358 ConstantSDNode *CC =
3359 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3360 if (!CC)
3361 return false;
3362 Y = V.getOperand(1);
3363
3364 ConstantSDNode *XC =
3365 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3366 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3367 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3368 };
3369
3370 // LHS of comparison should be an one-use 'and'.
3371 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3372 return SDValue();
3373
3374 X = N0.getOperand(0);
3375 SDValue Mask = N0.getOperand(1);
3376
3377 // 'and' is commutative!
3378 if (!Match(Mask)) {
3379 std::swap(X, Mask);
3380 if (!Match(Mask))
3381 return SDValue();
3382 }
3383
3384 EVT VT = X.getValueType();
3385
3386 // Produce:
3387 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3388 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3389 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3390 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3391 return T2;
3392}
3393
3394/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3395/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3396/// handle the commuted versions of these patterns.
3397SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3398 ISD::CondCode Cond, const SDLoc &DL,
3399 DAGCombinerInfo &DCI) const {
3400 unsigned BOpcode = N0.getOpcode();
3401 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&((void)0)
3402 "Unexpected binop")((void)0);
3403 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")((void)0);
3404
3405 // (X + Y) == X --> Y == 0
3406 // (X - Y) == X --> Y == 0
3407 // (X ^ Y) == X --> Y == 0
3408 SelectionDAG &DAG = DCI.DAG;
3409 EVT OpVT = N0.getValueType();
3410 SDValue X = N0.getOperand(0);
3411 SDValue Y = N0.getOperand(1);
3412 if (X == N1)
3413 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3414
3415 if (Y != N1)
3416 return SDValue();
3417
3418 // (X + Y) == Y --> X == 0
3419 // (X ^ Y) == Y --> X == 0
3420 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3421 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3422
3423 // The shift would not be valid if the operands are boolean (i1).
3424 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3425 return SDValue();
3426
3427 // (X - Y) == Y --> X == Y << 1
3428 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3429 !DCI.isBeforeLegalize());
3430 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3431 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3432 if (!DCI.isCalledByLegalizer())
3433 DCI.AddToWorklist(YShl1.getNode());
3434 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3435}
3436
3437static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
3438 SDValue N0, const APInt &C1,
3439 ISD::CondCode Cond, const SDLoc &dl,
3440 SelectionDAG &DAG) {
3441 // Look through truncs that don't change the value of a ctpop.
3442 // FIXME: Add vector support? Need to be careful with setcc result type below.
3443 SDValue CTPOP = N0;
3444 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3445 N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
3446 CTPOP = N0.getOperand(0);
3447
3448 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3449 return SDValue();
3450
3451 EVT CTVT = CTPOP.getValueType();
3452 SDValue CTOp = CTPOP.getOperand(0);
3453
3454 // If this is a vector CTPOP, keep the CTPOP if it is legal.
3455 // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3456 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3457 return SDValue();
3458
3459 // (ctpop x) u< 2 -> (x & x-1) == 0
3460 // (ctpop x) u> 1 -> (x & x-1) != 0
3461 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3462 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3463 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3464 return SDValue();
3465 if (C1 == 0 && (Cond == ISD::SETULT))
3466 return SDValue(); // This is handled elsewhere.
3467
3468 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3469
3470 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3471 SDValue Result = CTOp;
3472 for (unsigned i = 0; i < Passes; i++) {
3473 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3474 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3475 }
3476 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3477 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3478 }
3479
3480 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3481 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3482 // For scalars, keep CTPOP if it is legal or custom.
3483 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3484 return SDValue();
3485 // This is based on X86's custom lowering for CTPOP which produces more
3486 // instructions than the expansion here.
3487
3488 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3489 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3490 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3491 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3492 assert(CTVT.isInteger())((void)0);
3493 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3494 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3495 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3496 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3497 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3498 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3499 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3500 }
3501
3502 return SDValue();
3503}
3504
3505/// Try to simplify a setcc built with the specified operands and cc. If it is
3506/// unable to simplify it, return a null SDValue.
3507SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3508 ISD::CondCode Cond, bool foldBooleans,
3509 DAGCombinerInfo &DCI,
3510 const SDLoc &dl) const {
3511 SelectionDAG &DAG = DCI.DAG;
3512 const DataLayout &Layout = DAG.getDataLayout();
3513 EVT OpVT = N0.getValueType();
3514
3515 // Constant fold or commute setcc.
3516 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3517 return Fold;
3518
3519 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3520 // TODO: Handle non-splat vector constants. All undef causes trouble.
3521 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3522 // infinite loop here when we encounter one.
3523 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3524 if (isConstOrConstSplat(N0) &&
3525 (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3526 (DCI.isBeforeLegalizeOps() ||
3527 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3528 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3529
3530 // If we have a subtract with the same 2 non-constant operands as this setcc
3531 // -- but in reverse order -- then try to commute the operands of this setcc
3532 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3533 // instruction on some targets.
3534 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3535 (DCI.isBeforeLegalizeOps() ||
3536 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3537 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3538 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3539 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3540
3541 if (auto *N1C = isConstOrConstSplat(N1)) {
3542 const APInt &C1 = N1C->getAPIntValue();
3543
3544 // Optimize some CTPOP cases.
3545 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3546 return V;
3547
3548 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3549 // equality comparison, then we're just comparing whether X itself is
3550 // zero.
3551 if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3552 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3553 isPowerOf2_32(N0.getScalarValueSizeInBits())) {
3554 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3555 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3556 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3557 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3558 // (srl (ctlz x), 5) == 0 -> X != 0
3559 // (srl (ctlz x), 5) != 1 -> X != 0
3560 Cond = ISD::SETNE;
3561 } else {
3562 // (srl (ctlz x), 5) != 0 -> X == 0
3563 // (srl (ctlz x), 5) == 1 -> X == 0
3564 Cond = ISD::SETEQ;
3565 }
3566 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3567 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3568 Cond);
3569 }
3570 }
3571 }
3572 }
3573
3574 // FIXME: Support vectors.
3575 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3576 const APInt &C1 = N1C->getAPIntValue();
3577
3578 // (zext x) == C --> x == (trunc C)
3579 // (sext x) == C --> x == (trunc C)
3580 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3581 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3582 unsigned MinBits = N0.getValueSizeInBits();
3583 SDValue PreExt;
3584 bool Signed = false;
3585 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3586 // ZExt
3587 MinBits = N0->getOperand(0).getValueSizeInBits();
3588 PreExt = N0->getOperand(0);
3589 } else if (N0->getOpcode() == ISD::AND) {
3590 // DAGCombine turns costly ZExts into ANDs
3591 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3592 if ((C->getAPIntValue()+1).isPowerOf2()) {
3593 MinBits = C->getAPIntValue().countTrailingOnes();
3594 PreExt = N0->getOperand(0);
3595 }
3596 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3597 // SExt
3598 MinBits = N0->getOperand(0).getValueSizeInBits();
3599 PreExt = N0->getOperand(0);
3600 Signed = true;
3601 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3602 // ZEXTLOAD / SEXTLOAD
3603 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3604 MinBits = LN0->getMemoryVT().getSizeInBits();
3605 PreExt = N0;
3606 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3607 Signed = true;
3608 MinBits = LN0->getMemoryVT().getSizeInBits();
3609 PreExt = N0;
3610 }
3611 }
3612
3613 // Figure out how many bits we need to preserve this constant.
3614 unsigned ReqdBits = Signed ?
3615 C1.getBitWidth() - C1.getNumSignBits() + 1 :
3616 C1.getActiveBits();
3617
3618 // Make sure we're not losing bits from the constant.
3619 if (MinBits > 0 &&
3620 MinBits < C1.getBitWidth() &&
3621 MinBits >= ReqdBits) {
3622 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3623 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3624 // Will get folded away.
3625 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3626 if (MinBits == 1 && C1 == 1)
3627 // Invert the condition.
3628 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3629 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3630 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3631 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3632 }
3633
3634 // If truncating the setcc operands is not desirable, we can still
3635 // simplify the expression in some cases:
3636 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3637 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3638 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3639 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3640 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3641 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3642 SDValue TopSetCC = N0->getOperand(0);
3643 unsigned N0Opc = N0->getOpcode();
3644 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3645 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3646 TopSetCC.getOpcode() == ISD::SETCC &&
3647 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3648 (isConstFalseVal(N1C) ||
3649 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3650
3651 bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3652 (!N1C->isNullValue() && Cond == ISD::SETNE);
3653
3654 if (!Inverse)
3655 return TopSetCC;
3656
3657 ISD::CondCode InvCond = ISD::getSetCCInverse(
3658 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3659 TopSetCC.getOperand(0).getValueType());
3660 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3661 TopSetCC.getOperand(1),
3662 InvCond);
3663 }
3664 }
3665 }
3666
3667 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3668 // equality or unsigned, and all 1 bits of the const are in the same
3669 // partial word, see if we can shorten the load.
3670 if (DCI.isBeforeLegalize() &&
3671 !ISD::isSignedIntSetCC(Cond) &&
3672 N0.getOpcode() == ISD::AND && C1 == 0 &&
3673 N0.getNode()->hasOneUse() &&
3674 isa<LoadSDNode>(N0.getOperand(0)) &&
3675 N0.getOperand(0).getNode()->hasOneUse() &&
3676 isa<ConstantSDNode>(N0.getOperand(1))) {
3677 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3678 APInt bestMask;
3679 unsigned bestWidth = 0, bestOffset = 0;
3680 if (Lod->isSimple() && Lod->isUnindexed()) {
3681 unsigned origWidth = N0.getValueSizeInBits();
3682 unsigned maskWidth = origWidth;
3683 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3684 // 8 bits, but have to be careful...
3685 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3686 origWidth = Lod->getMemoryVT().getSizeInBits();
3687 const APInt &Mask = N0.getConstantOperandAPInt(1);
3688 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3689 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3690 for (unsigned offset=0; offset<origWidth/width; offset++) {
3691 if (Mask.isSubsetOf(newMask)) {
3692 if (Layout.isLittleEndian())
3693 bestOffset = (uint64_t)offset * (width/8);
3694 else
3695 bestOffset = (origWidth/width - offset - 1) * (width/8);
3696 bestMask = Mask.lshr(offset * (width/8) * 8);
3697 bestWidth = width;
3698 break;
3699 }
3700 newMask <<= width;
3701 }
3702 }
3703 }
3704 if (bestWidth) {
3705 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3706 if (newVT.isRound() &&
3707 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3708 SDValue Ptr = Lod->getBasePtr();
3709 if (bestOffset != 0)
3710 Ptr =
3711 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3712 SDValue NewLoad =
3713 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3714 Lod->getPointerInfo().getWithOffset(bestOffset),
3715 Lod->getOriginalAlign());
3716 return DAG.getSetCC(dl, VT,
3717 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3718 DAG.getConstant(bestMask.trunc(bestWidth),
3719 dl, newVT)),
3720 DAG.getConstant(0LL, dl, newVT), Cond);
3721 }
3722 }
3723 }
3724
3725 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3726 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3727 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3728
3729 // If the comparison constant has bits in the upper part, the
3730 // zero-extended value could never match.
3731 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3732 C1.getBitWidth() - InSize))) {
3733 switch (Cond) {
3734 case ISD::SETUGT:
3735 case ISD::SETUGE:
3736 case ISD::SETEQ:
3737 return DAG.getConstant(0, dl, VT);
3738 case ISD::SETULT:
3739 case ISD::SETULE:
3740 case ISD::SETNE:
3741 return DAG.getConstant(1, dl, VT);
3742 case ISD::SETGT:
3743 case ISD::SETGE:
3744 // True if the sign bit of C1 is set.
3745 return DAG.getConstant(C1.isNegative(), dl, VT);
3746 case ISD::SETLT:
3747 case ISD::SETLE:
3748 // True if the sign bit of C1 isn't set.
3749 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3750 default:
3751 break;
3752 }
3753 }
3754
3755 // Otherwise, we can perform the comparison with the low bits.
3756 switch (Cond) {
3757 case ISD::SETEQ:
3758 case ISD::SETNE:
3759 case ISD::SETUGT:
3760 case ISD::SETUGE:
3761 case ISD::SETULT:
3762 case ISD::SETULE: {
3763 EVT newVT = N0.getOperand(0).getValueType();
3764 if (DCI.isBeforeLegalizeOps() ||
3765 (isOperationLegal(ISD::SETCC, newVT) &&
3766 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3767 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3768 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3769
3770 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3771 NewConst, Cond);
3772 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3773 }
3774 break;
3775 }
3776 default:
3777 break; // todo, be more careful with signed comparisons
3778 }
3779 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3780 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3781 !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3782 OpVT)) {
3783 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3784 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3785 EVT ExtDstTy = N0.getValueType();
3786 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3787
3788 // If the constant doesn't fit into the number of bits for the source of
3789 // the sign extension, it is impossible for both sides to be equal.
3790 if (C1.getMinSignedBits() > ExtSrcTyBits)
3791 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3792
3793 assert(ExtDstTy == N0.getOperand(0).getValueType() &&((void)0)
3794 ExtDstTy != ExtSrcTy && "Unexpected types!")((void)0);
3795 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3796 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3797 DAG.getConstant(Imm, dl, ExtDstTy));
3798 if (!DCI.isCalledByLegalizer())
3799 DCI.AddToWorklist(ZextOp.getNode());
3800 // Otherwise, make this a use of a zext.
3801 return DAG.getSetCC(dl, VT, ZextOp,
3802 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3803 } else if ((N1C->isNullValue() || N1C->isOne()) &&
3804 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3805 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3806 if (N0.getOpcode() == ISD::SETCC &&
3807 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3808 (N0.getValueType() == MVT::i1 ||
3809 getBooleanContents(N0.getOperand(0).getValueType()) ==
3810 ZeroOrOneBooleanContent)) {
3811 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3812 if (TrueWhenTrue)
3813 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3814 // Invert the condition.
3815 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3816 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3817 if (DCI.isBeforeLegalizeOps() ||
3818 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3819 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3820 }
3821
3822 if ((N0.getOpcode() == ISD::XOR ||
3823 (N0.getOpcode() == ISD::AND &&
3824 N0.getOperand(0).getOpcode() == ISD::XOR &&
3825 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3826 isOneConstant(N0.getOperand(1))) {
3827 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3828 // can only do this if the top bits are known zero.
3829 unsigned BitWidth = N0.getValueSizeInBits();
3830 if (DAG.MaskedValueIsZero(N0,
3831 APInt::getHighBitsSet(BitWidth,
3832 BitWidth-1))) {
3833 // Okay, get the un-inverted input value.
3834 SDValue Val;
3835 if (N0.getOpcode() == ISD::XOR) {
3836 Val = N0.getOperand(0);
3837 } else {
3838 assert(N0.getOpcode() == ISD::AND &&((void)0)
3839 N0.getOperand(0).getOpcode() == ISD::XOR)((void)0);
3840 // ((X^1)&1)^1 -> X & 1
3841 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3842 N0.getOperand(0).getOperand(0),
3843 N0.getOperand(1));
3844 }
3845
3846 return DAG.getSetCC(dl, VT, Val, N1,
3847 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3848 }
3849 } else if (N1C->isOne()) {
3850 SDValue Op0 = N0;
3851 if (Op0.getOpcode() == ISD::TRUNCATE)
3852 Op0 = Op0.getOperand(0);
3853
3854 if ((Op0.getOpcode() == ISD::XOR) &&
3855 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3856 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3857 SDValue XorLHS = Op0.getOperand(0);
3858 SDValue XorRHS = Op0.getOperand(1);
3859 // Ensure that the input setccs return an i1 type or 0/1 value.
3860 if (Op0.getValueType() == MVT::i1 ||
3861 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3862 ZeroOrOneBooleanContent &&
3863 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3864 ZeroOrOneBooleanContent)) {
3865 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3866 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3867 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3868 }
3869 }
3870 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3871 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3872 if (Op0.getValueType().bitsGT(VT))
3873 Op0 = DAG.getNode(ISD::AND, dl, VT,
3874 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3875 DAG.getConstant(1, dl, VT));
3876 else if (Op0.getValueType().bitsLT(VT))
3877 Op0 = DAG.getNode(ISD::AND, dl, VT,
3878 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3879 DAG.getConstant(1, dl, VT));
3880
3881 return DAG.getSetCC(dl, VT, Op0,
3882 DAG.getConstant(0, dl, Op0.getValueType()),
3883 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3884 }
3885 if (Op0.getOpcode() == ISD::AssertZext &&
3886 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3887 return DAG.getSetCC(dl, VT, Op0,
3888 DAG.getConstant(0, dl, Op0.getValueType()),
3889 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3890 }
3891 }
3892
3893 // Given:
3894 // icmp eq/ne (urem %x, %y), 0
3895 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3896 // icmp eq/ne %x, 0
3897 if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3898 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3899 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3900 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3901 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3902 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3903 }
3904
3905 if (SDValue V =
3906 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3907 return V;
3908 }
3909
3910 // These simplifications apply to splat vectors as well.
3911 // TODO: Handle more splat vector cases.
3912 if (auto *N1C = isConstOrConstSplat(N1)) {
3913 const APInt &C1 = N1C->getAPIntValue();
3914
3915 APInt MinVal, MaxVal;
3916 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3917 if (ISD::isSignedIntSetCC(Cond)) {
3918 MinVal = APInt::getSignedMinValue(OperandBitSize);
3919 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3920 } else {
3921 MinVal = APInt::getMinValue(OperandBitSize);
3922 MaxVal = APInt::getMaxValue(OperandBitSize);
3923 }
3924
3925 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3926 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3927 // X >= MIN --> true
3928 if (C1 == MinVal)
3929 return DAG.getBoolConstant(true, dl, VT, OpVT);
3930
3931 if (!VT.isVector()) { // TODO: Support this for vectors.
3932 // X >= C0 --> X > (C0 - 1)
3933 APInt C = C1 - 1;
3934 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3935 if ((DCI.isBeforeLegalizeOps() ||
3936 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3937 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3938 isLegalICmpImmediate(C.getSExtValue())))) {
3939 return DAG.getSetCC(dl, VT, N0,
3940 DAG.getConstant(C, dl, N1.getValueType()),
3941 NewCC);
3942 }
3943 }
3944 }
3945
3946 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3947 // X <= MAX --> true
3948 if (C1 == MaxVal)
3949 return DAG.getBoolConstant(true, dl, VT, OpVT);
3950
3951 // X <= C0 --> X < (C0 + 1)
3952 if (!VT.isVector()) { // TODO: Support this for vectors.
3953 APInt C = C1 + 1;
3954 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3955 if ((DCI.isBeforeLegalizeOps() ||
3956 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3957 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3958 isLegalICmpImmediate(C.getSExtValue())))) {
3959 return DAG.getSetCC(dl, VT, N0,
3960 DAG.getConstant(C, dl, N1.getValueType()),
3961 NewCC);
3962 }
3963 }
3964 }
3965
3966 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3967 if (C1 == MinVal)
3968 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3969
3970 // TODO: Support this for vectors after legalize ops.
3971 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3972 // Canonicalize setlt X, Max --> setne X, Max
3973 if (C1 == MaxVal)
3974 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3975
3976 // If we have setult X, 1, turn it into seteq X, 0
3977 if (C1 == MinVal+1)
3978 return DAG.getSetCC(dl, VT, N0,
3979 DAG.getConstant(MinVal, dl, N0.getValueType()),
3980 ISD::SETEQ);
3981 }
3982 }
3983
3984 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3985 if (C1 == MaxVal)
3986 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3987
3988 // TODO: Support this for vectors after legalize ops.
3989 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3990 // Canonicalize setgt X, Min --> setne X, Min
3991 if (C1 == MinVal)
3992 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3993
3994 // If we have setugt X, Max-1, turn it into seteq X, Max
3995 if (C1 == MaxVal-1)
3996 return DAG.getSetCC(dl, VT, N0,
3997 DAG.getConstant(MaxVal, dl, N0.getValueType()),
3998 ISD::SETEQ);
3999 }
4000 }
4001
4002 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4003 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4004 if (C1.isNullValue())
4005 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4006 VT, N0, N1, Cond, DCI, dl))
4007 return CC;
4008
4009 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4010 // For example, when high 32-bits of i64 X are known clear:
4011 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4012 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4013 bool CmpZero = N1C->getAPIntValue().isNullValue();
4014 bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();
4015 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4016 // Match or(lo,shl(hi,bw/2)) pattern.
4017 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4018 unsigned EltBits = V.getScalarValueSizeInBits();
4019 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4020 return false;
4021 SDValue LHS = V.getOperand(0);
4022 SDValue RHS = V.getOperand(1);
4023 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4024 // Unshifted element must have zero upperbits.
4025 if (RHS.getOpcode() == ISD::SHL &&
4026 isa<ConstantSDNode>(RHS.getOperand(1)) &&
4027 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4028 DAG.MaskedValueIsZero(LHS, HiBits)) {
4029 Lo = LHS;
4030 Hi = RHS.getOperand(0);
4031 return true;
4032 }
4033 if (LHS.getOpcode() == ISD::SHL &&
4034 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4035 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4036 DAG.MaskedValueIsZero(RHS, HiBits)) {
4037 Lo = RHS;
4038 Hi = LHS.getOperand(0);
4039 return true;
4040 }
4041 return false;
4042 };
4043
4044 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4045 unsigned EltBits = N0.getScalarValueSizeInBits();
4046 unsigned HalfBits = EltBits / 2;
4047 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4048 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4049 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4050 SDValue NewN0 =
4051 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4052 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4053 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4054 };
4055
4056 SDValue Lo, Hi;
4057 if (IsConcat(N0, Lo, Hi))
4058 return MergeConcat(Lo, Hi);
4059
4060 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4061 SDValue Lo0, Lo1, Hi0, Hi1;
4062 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4063 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4064 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4065 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4066 }
4067 }
4068 }
4069 }
4070
4071 // If we have "setcc X, C0", check to see if we can shrink the immediate
4072 // by changing cc.
4073 // TODO: Support this for vectors after legalize ops.
4074 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4075 // SETUGT X, SINTMAX -> SETLT X, 0
4076 // SETUGE X, SINTMIN -> SETLT X, 0
4077 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4078 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4079 return DAG.getSetCC(dl, VT, N0,
4080 DAG.getConstant(0, dl, N1.getValueType()),
4081 ISD::SETLT);
4082
4083 // SETULT X, SINTMIN -> SETGT X, -1
4084 // SETULE X, SINTMAX -> SETGT X, -1
4085 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4086 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4087 return DAG.getSetCC(dl, VT, N0,
4088 DAG.getAllOnesConstant(dl, N1.getValueType()),
4089 ISD::SETGT);
4090 }
4091 }
4092
4093 // Back to non-vector simplifications.
4094 // TODO: Can we do these for vector splats?
4095 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4096 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4097 const APInt &C1 = N1C->getAPIntValue();
4098 EVT ShValTy = N0.getValueType();
4099
4100 // Fold bit comparisons when we can. This will result in an
4101 // incorrect value when boolean false is negative one, unless
4102 // the bitsize is 1 in which case the false value is the same
4103 // in practice regardless of the representation.
4104 if ((VT.getSizeInBits() == 1 ||
4105 getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
4106 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4107 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4108 N0.getOpcode() == ISD::AND) {
4109 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4110 EVT ShiftTy =
4111 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4112 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4113 // Perform the xform if the AND RHS is a single bit.
4114 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4115 if (AndRHS->getAPIntValue().isPowerOf2() &&
4116 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4117 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4118 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4119 DAG.getConstant(ShCt, dl, ShiftTy)));
4120 }
4121 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4122 // (X & 8) == 8 --> (X & 8) >> 3
4123 // Perform the xform if C1 is a single bit.
4124 unsigned ShCt = C1.logBase2();
4125 if (C1.isPowerOf2() &&
4126 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4127 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4128 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4129 DAG.getConstant(ShCt, dl, ShiftTy)));
4130 }
4131 }
4132 }
4133 }
4134
4135 if (C1.getMinSignedBits() <= 64 &&
4136 !isLegalICmpImmediate(C1.getSExtValue())) {
4137 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4138 // (X & -256) == 256 -> (X >> 8) == 1
4139 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4140 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4141 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4142 const APInt &AndRHSC = AndRHS->getAPIntValue();
4143 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
4144 unsigned ShiftBits = AndRHSC.countTrailingZeros();
4145 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4146 SDValue Shift =
4147 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4148 DAG.getConstant(ShiftBits, dl, ShiftTy));
4149 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4150 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4151 }
4152 }
4153 }
4154 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4155 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4156 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4157 // X < 0x100000000 -> (X >> 32) < 1
4158 // X >= 0x100000000 -> (X >> 32) >= 1
4159 // X <= 0x0ffffffff -> (X >> 32) < 1
4160 // X > 0x0ffffffff -> (X >> 32) >= 1
4161 unsigned ShiftBits;
4162 APInt NewC = C1;
4163 ISD::CondCode NewCond = Cond;
4164 if (AdjOne) {
4165 ShiftBits = C1.countTrailingOnes();
4166 NewC = NewC + 1;
4167 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4168 } else {
4169 ShiftBits = C1.countTrailingZeros();
4170 }
4171 NewC.lshrInPlace(ShiftBits);
4172 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4173 isLegalICmpImmediate(NewC.getSExtValue()) &&
4174 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4175 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4176 DAG.getConstant(ShiftBits, dl, ShiftTy));
4177 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4178 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4179 }
4180 }
4181 }
4182 }
4183
4184 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4185 auto *CFP = cast<ConstantFPSDNode>(N1);
4186 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")((void)0);
4187
4188 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4189 // constant if knowing that the operand is non-nan is enough. We prefer to
4190 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4191 // materialize 0.0.
4192 if (Cond == ISD::SETO || Cond == ISD::SETUO)
4193 return DAG.getSetCC(dl, VT, N0, N0, Cond);
4194
4195 // setcc (fneg x), C -> setcc swap(pred) x, -C
4196 if (N0.getOpcode() == ISD::FNEG) {
4197 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4198 if (DCI.isBeforeLegalizeOps() ||
4199 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4200 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4201 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4202 }
4203 }
4204
4205 // If the condition is not legal, see if we can find an equivalent one
4206 // which is legal.
4207 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4208 // If the comparison was an awkward floating-point == or != and one of
4209 // the comparison operands is infinity or negative infinity, convert the
4210 // condition to a less-awkward <= or >=.
4211 if (CFP->getValueAPF().isInfinity()) {
4212 bool IsNegInf = CFP->getValueAPF().isNegative();
4213 ISD::CondCode NewCond = ISD::SETCC_INVALID;
4214 switch (Cond) {
4215 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4216 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4217 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4218 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4219 default: break;
4220 }
4221 if (NewCond != ISD::SETCC_INVALID &&
4222 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4223 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4224 }
4225 }
4226 }
4227
4228 if (N0 == N1) {
4229 // The sext(setcc()) => setcc() optimization relies on the appropriate
4230 // constant being emitted.
4231 assert(!N0.getValueType().isInteger() &&((void)0)
4232 "Integer types should be handled by FoldSetCC")((void)0);
4233
4234 bool EqTrue = ISD::isTrueWhenEqual(Cond);
4235 unsigned UOF = ISD::getUnorderedFlavor(Cond);
4236 if (UOF == 2) // FP operators that are undefined on NaNs.
4237 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4238 if (UOF == unsigned(EqTrue))
4239 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4240 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4241 // if it is not already.
4242 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4243 if (NewCond != Cond &&
4244 (DCI.isBeforeLegalizeOps() ||
4245 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4246 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4247 }
4248
4249 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4250 N0.getValueType().isInteger()) {
4251 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4252 N0.getOpcode() == ISD::XOR) {
4253 // Simplify (X+Y) == (X+Z) --> Y == Z
4254 if (N0.getOpcode() == N1.getOpcode()) {
4255 if (N0.getOperand(0) == N1.getOperand(0))
4256 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4257 if (N0.getOperand(1) == N1.getOperand(1))
4258 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4259 if (isCommutativeBinOp(N0.getOpcode())) {
4260 // If X op Y == Y op X, try other combinations.
4261 if (N0.getOperand(0) == N1.getOperand(1))
4262 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4263 Cond);
4264 if (N0.getOperand(1) == N1.getOperand(0))
4265 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4266 Cond);
4267 }
4268 }
4269
4270 // If RHS is a legal immediate value for a compare instruction, we need
4271 // to be careful about increasing register pressure needlessly.
4272 bool LegalRHSImm = false;
4273
4274 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4275 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4276 // Turn (X+C1) == C2 --> X == C2-C1
4277 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4278 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4279 DAG.getConstant(RHSC->getAPIntValue()-
4280 LHSR->getAPIntValue(),
4281 dl, N0.getValueType()), Cond);
4282 }
4283
4284 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4285 if (N0.getOpcode() == ISD::XOR)
4286 // If we know that all of the inverted bits are zero, don't bother
4287 // performing the inversion.
4288 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4289 return
4290 DAG.getSetCC(dl, VT, N0.getOperand(0),
4291 DAG.getConstant(LHSR->getAPIntValue() ^
4292 RHSC->getAPIntValue(),
4293 dl, N0.getValueType()),
4294 Cond);
4295 }
4296
4297 // Turn (C1-X) == C2 --> X == C1-C2
4298 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4299 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4300 return
4301 DAG.getSetCC(dl, VT, N0.getOperand(1),
4302 DAG.getConstant(SUBC->getAPIntValue() -
4303 RHSC->getAPIntValue(),
4304 dl, N0.getValueType()),
4305 Cond);
4306 }
4307 }
4308
4309 // Could RHSC fold directly into a compare?
4310 if (RHSC->getValueType(0).getSizeInBits() <= 64)
4311 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4312 }
4313
4314 // (X+Y) == X --> Y == 0 and similar folds.
4315 // Don't do this if X is an immediate that can fold into a cmp
4316 // instruction and X+Y has other uses. It could be an induction variable
4317 // chain, and the transform would increase register pressure.
4318 if (!LegalRHSImm || N0.hasOneUse())
4319 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4320 return V;
4321 }
4322
4323 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4324 N1.getOpcode() == ISD::XOR)
4325 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4326 return V;
4327
4328 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4329 return V;
4330 }
4331
4332 // Fold remainder of division by a constant.
4333 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4334 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4335 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4336
4337 // When division is cheap or optimizing for minimum size,
4338 // fall through to DIVREM creation by skipping this fold.
4339 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
4340 if (N0.getOpcode() == ISD::UREM) {
4341 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4342 return Folded;
4343 } else if (N0.getOpcode() == ISD::SREM) {
4344 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4345 return Folded;
4346 }
4347 }
4348 }
4349
4350 // Fold away ALL boolean setcc's.
4351 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4352 SDValue Temp;
4353 switch (Cond) {
4354 default: llvm_unreachable("Unknown integer setcc!")__builtin_unreachable();
4355 case ISD::SETEQ: // X == Y -> ~(X^Y)
4356 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4357 N0 = DAG.getNOT(dl, Temp, OpVT);
4358 if (!DCI.isCalledByLegalizer())
4359 DCI.AddToWorklist(Temp.getNode());
4360 break;
4361 case ISD::SETNE: // X != Y --> (X^Y)
4362 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4363 break;
4364 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4365 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4366 Temp = DAG.getNOT(dl, N0, OpVT);
4367 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4368 if (!DCI.isCalledByLegalizer())
4369 DCI.AddToWorklist(Temp.getNode());
4370 break;
4371 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4372 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4373 Temp = DAG.getNOT(dl, N1, OpVT);
4374 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4375 if (!DCI.isCalledByLegalizer())
4376 DCI.AddToWorklist(Temp.getNode());
4377 break;
4378 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4379 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4380 Temp = DAG.getNOT(dl, N0, OpVT);
4381 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4382 if (!DCI.isCalledByLegalizer())
4383 DCI.AddToWorklist(Temp.getNode());
4384 break;
4385 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4386 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4387 Temp = DAG.getNOT(dl, N1, OpVT);
4388 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4389 break;
4390 }
4391 if (VT.getScalarType() != MVT::i1) {
4392 if (!DCI.isCalledByLegalizer())
4393 DCI.AddToWorklist(N0.getNode());
4394 // FIXME: If running after legalize, we probably can't do this.
4395 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4396 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4397 }
4398 return N0;
4399 }
4400
4401 // Could not fold it.
4402 return SDValue();
4403}
4404
4405/// Returns true (and the GlobalValue and the offset) if the node is a
4406/// GlobalAddress + offset.
4407bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4408 int64_t &Offset) const {
4409
4410 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4411
4412 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4413 GA = GASD->getGlobal();
4414 Offset += GASD->getOffset();
4415 return true;
4416 }
4417
4418 if (N->getOpcode() == ISD::ADD) {
4419 SDValue N1 = N->getOperand(0);
4420 SDValue N2 = N->getOperand(1);
4421 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4422 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4423 Offset += V->getSExtValue();
4424 return true;
4425 }
4426 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4427 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4428 Offset += V->getSExtValue();
4429 return true;
4430 }
4431 }
4432 }
4433
4434 return false;
4435}
4436
4437SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4438 DAGCombinerInfo &DCI) const {
4439 // Default implementation: no optimization.
4440 return SDValue();
4441}
4442
4443//===----------------------------------------------------------------------===//
4444// Inline Assembler Implementation Methods
4445//===----------------------------------------------------------------------===//
4446
4447TargetLowering::ConstraintType
4448TargetLowering::getConstraintType(StringRef Constraint) const {
4449 unsigned S = Constraint.size();
4450
4451 if (S == 1) {
4452 switch (Constraint[0]) {
4453 default: break;
4454 case 'r':
4455 return C_RegisterClass;
4456 case 'm': // memory
4457 case 'o': // offsetable
4458 case 'V': // not offsetable
4459 return C_Memory;
4460 case 'n': // Simple Integer
4461 case 'E': // Floating Point Constant
4462 case 'F': // Floating Point Constant
4463 return C_Immediate;
4464 case 'i': // Simple Integer or Relocatable Constant
4465 case 's': // Relocatable Constant
4466 case 'p': // Address.
4467 case 'X': // Allow ANY value.
4468 case 'I': // Target registers.
4469 case 'J':
4470 case 'K':
4471 case 'L':
4472 case 'M':
4473 case 'N':
4474 case 'O':
4475 case 'P':
4476 case '<':
4477 case '>':
4478 return C_Other;
4479 }
4480 }
4481
4482 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4483 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4484 return C_Memory;
4485 return C_Register;
4486 }
4487 return C_Unknown;
4488}
4489
4490/// Try to replace an X constraint, which matches anything, with another that
4491/// has more specific requirements based on the type of the corresponding
4492/// operand.
4493const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4494 if (ConstraintVT.isInteger())
4495 return "r";
4496 if (ConstraintVT.isFloatingPoint())
4497 return "f"; // works for many targets
4498 return nullptr;
4499}
4500
4501SDValue TargetLowering::LowerAsmOutputForConstraint(
4502 SDValue &Chain, SDValue &Flag, const SDLoc &DL,
4503 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
4504 return SDValue();
4505}
4506
4507/// Lower the specified operand into the Ops vector.
4508/// If it is invalid, don't add anything to Ops.
4509void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4510 std::string &Constraint,
4511 std::vector<SDValue> &Ops,
4512 SelectionDAG &DAG) const {
4513
4514 if (Constraint.length() > 1) return;
10
←
Assuming the condition is false
→
11
←
Taking false branch
→
4515
4516 char ConstraintLetter = Constraint[0];
4517 switch (ConstraintLetter) {
12
←
Control jumps to 'case 105:' at line 4526
→
4518 default: break;
4519 case 'X': // Allows any operand; labels (basic block) use this.
4520 if (Op.getOpcode() == ISD::BasicBlock ||
4521 Op.getOpcode() == ISD::TargetBlockAddress) {
4522 Ops.push_back(Op);
4523 return;
4524 }
4525 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4526 case 'i': // Simple Integer or Relocatable Constant
4527 case 'n': // Simple Integer
4528 case 's': { // Relocatable Constant
4529
4530 GlobalAddressSDNode *GA;
4531 ConstantSDNode *C;
4532 BlockAddressSDNode *BA;
4533 uint64_t Offset = 0;
4534
4535 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4536 // etc., since getelementpointer is variadic. We can't use
4537 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4538 // while in this case the GA may be furthest from the root node which is
4539 // likely an ISD::ADD.
4540 while (1) {
13
←
Loop condition is true. Entering loop body
→
38
←
Loop condition is true. Entering loop body
→
4541 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
14
←
Calling 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
18
←
Returning from 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
19
←
Assuming 'GA' is null
→
39
←
Calling 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
43
←
Returning from 'dyn_cast<llvm::GlobalAddressSDNode, llvm::SDValue>'
→
44
←
Assuming 'GA' is null
→
4542 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4543 GA->getValueType(0),
4544 Offset + GA->getOffset()));
4545 return;
4546 }
4547 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
20
←
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
24
←
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
25
←
Assuming 'C' is null
→
45
←
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
49
←
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
→
50
←
Assuming 'C' is null
→
4548 // gcc prints these as sign extended. Sign extend value to 64 bits
4549 // now; without this it would get ZExt'd later in
4550 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4551 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4552 BooleanContent BCont = getBooleanContents(MVT::i64);
4553 ISD::NodeType ExtOpc =
4554 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
4555 int64_t ExtVal =
4556 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
4557 Ops.push_back(
4558 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
4559 return;
4560 }
4561 if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {
26
←
Calling 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
30
←
Returning from 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
31
←
Assuming 'BA' is null
→
51
←
Calling 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
64
←
Returning from 'dyn_cast<llvm::BlockAddressSDNode, llvm::SDValue>'
→
65
←
Assuming 'BA' is null
→
66
←
Assuming pointer value is null
→
4562 Ops.push_back(DAG.getTargetBlockAddress(
4563 BA->getBlockAddress(), BA->getValueType(0),
4564 Offset + BA->getOffset(), BA->getTargetFlags()));
4565 return;
4566 }
4567 const unsigned OpCode = Op.getOpcode();
67
←
Calling 'SDValue::getOpcode'
→
4568 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
32
←
Assuming 'OpCode' is equal to ADD
→
4569 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
33
←
Assuming 'C' is non-null
→
34
←
Taking true branch
→
4570 Op = Op.getOperand(1);
35
←
Value assigned to 'Op.Node'
→
4571 // Subtraction is not commutative.
4572 else if (OpCode == ISD::ADD &&
4573 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4574 Op = Op.getOperand(0);
4575 else
4576 return;
4577 Offset += (OpCode
35.1
'OpCode' is equal to ADD
35.1
'OpCode' is equal to ADD
35.1
'OpCode' is equal to ADD
 == ISD::ADD ? 1 : -1) * C->getSExtValue();
36
←
'?' condition is true
→
4578 continue;
37
←
 Execution continues on line 4540
→
4579 }
4580 return;
4581 }
4582 break;
4583 }
4584 }
4585}
4586
4587std::pair<unsigned, const TargetRegisterClass *>
4588TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4589 StringRef Constraint,
4590 MVT VT) const {
4591 if (Constraint.empty() || Constraint[0] != '{')
4592 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4593 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")((void)0);
4594
4595 // Remove the braces from around the name.
4596 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4597
4598 std::pair<unsigned, const TargetRegisterClass *> R =
4599 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4600
4601 // Figure out which register class contains this reg.
4602 for (const TargetRegisterClass *RC : RI->regclasses()) {
4603 // If none of the value types for this register class are valid, we
4604 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4605 if (!isLegalRC(*RI, *RC))
4606 continue;
4607
4608 for (const MCPhysReg &PR : *RC) {
4609 if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
4610 std::pair<unsigned, const TargetRegisterClass *> S =
4611 std::make_pair(PR, RC);
4612
4613 // If this register class has the requested value type, return it,
4614 // otherwise keep searching and return the first class found
4615 // if no other is found which explicitly has the requested type.
4616 if (RI->isTypeLegalForClass(*RC, VT))
4617 return S;
4618 if (!R.second)
4619 R = S;
4620 }
4621 }
4622 }
4623
4624 return R;
4625}
4626
4627//===----------------------------------------------------------------------===//
4628// Constraint Selection.
4629
4630/// Return true of this is an input operand that is a matching constraint like
4631/// "4".
4632bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4633 assert(!ConstraintCode.empty() && "No known constraint!")((void)0);
4634 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4635}
4636
4637/// If this is an input matching constraint, this method returns the output
4638/// operand it matches.
4639unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4640 assert(!ConstraintCode.empty() && "No known constraint!")((void)0);
4641 return atoi(ConstraintCode.c_str());
4642}
4643
4644/// Split up the constraint string from the inline assembly value into the
4645/// specific constraints and their prefixes, and also tie in the associated
4646/// operand values.
4647/// If this returns an empty vector, and if the constraint string itself
4648/// isn't empty, there was an error parsing.
4649TargetLowering::AsmOperandInfoVector
4650TargetLowering::ParseConstraints(const DataLayout &DL,
4651 const TargetRegisterInfo *TRI,
4652 const CallBase &Call) const {
4653 /// Information about all of the constraints.
4654 AsmOperandInfoVector ConstraintOperands;
4655 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
4656 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4657
4658 // Do a prepass over the constraints, canonicalizing them, and building up the
4659 // ConstraintOperands list.
4660 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4661 unsigned ResNo = 0; // ResNo - The result number of the next output.
4662
4663 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4664 ConstraintOperands.emplace_back(std::move(CI));
4665 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4666
4667 // Update multiple alternative constraint count.
4668 if (OpInfo.multipleAlternatives.size() > maCount)
4669 maCount = OpInfo.multipleAlternatives.size();
4670
4671 OpInfo.ConstraintVT = MVT::Other;
4672
4673 // Compute the value type for each operand.
4674 switch (OpInfo.Type) {
4675 case InlineAsm::isOutput:
4676 // Indirect outputs just consume an argument.
4677 if (OpInfo.isIndirect) {
4678 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4679 break;
4680 }
4681
4682 // The return value of the call is this value. As such, there is no
4683 // corresponding argument.
4684 assert(!Call.getType()->isVoidTy() && "Bad inline asm!")((void)0);
4685 if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
4686 OpInfo.ConstraintVT =
4687 getSimpleValueType(DL, STy->getElementType(ResNo));
4688 } else {
4689 assert(ResNo == 0 && "Asm only has one result!")((void)0);
4690 OpInfo.ConstraintVT =
4691 getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
4692 }
4693 ++ResNo;
4694 break;
4695 case InlineAsm::isInput:
4696 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4697 break;
4698 case InlineAsm::isClobber:
4699 // Nothing to do.
4700 break;
4701 }
4702
4703 if (OpInfo.CallOperandVal) {
4704 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4705 if (OpInfo.isIndirect) {
4706 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
4707 if (!PtrTy)
4708 report_fatal_error("Indirect operand for inline asm not a pointer!");
4709 OpTy = PtrTy->getElementType();
4710 }
4711
4712 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4713 if (StructType *STy = dyn_cast<StructType>(OpTy))
4714 if (STy->getNumElements() == 1)
4715 OpTy = STy->getElementType(0);
4716
4717 // If OpTy is not a single value, it may be a struct/union that we
4718 // can tile with integers.
4719 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4720 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4721 switch (BitSize) {
4722 default: break;
4723 case 1:
4724 case 8:
4725 case 16:
4726 case 32:
4727 case 64:
4728 case 128:
4729 OpInfo.ConstraintVT =
4730 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4731 break;
4732 }
4733 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4734 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4735 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4736 } else {
4737 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4738 }
4739 }
4740 }
4741
4742 // If we have multiple alternative constraints, select the best alternative.
4743 if (!ConstraintOperands.empty()) {
4744 if (maCount) {
4745 unsigned bestMAIndex = 0;
4746 int bestWeight = -1;
4747 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4748 int weight = -1;
4749 unsigned maIndex;
4750 // Compute the sums of the weights for each alternative, keeping track
4751 // of the best (highest weight) one so far.
4752 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4753 int weightSum = 0;
4754 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4755 cIndex != eIndex; ++cIndex) {
4756 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4757 if (OpInfo.Type == InlineAsm::isClobber)
4758 continue;
4759
4760 // If this is an output operand with a matching input operand,
4761 // look up the matching input. If their types mismatch, e.g. one
4762 // is an integer, the other is floating point, or their sizes are
4763 // different, flag it as an maCantMatch.
4764 if (OpInfo.hasMatchingInput()) {
4765 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4766 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4767 if ((OpInfo.ConstraintVT.isInteger() !=
4768 Input.ConstraintVT.isInteger()) ||
4769 (OpInfo.ConstraintVT.getSizeInBits() !=
4770 Input.ConstraintVT.getSizeInBits())) {
4771 weightSum = -1; // Can't match.
4772 break;
4773 }
4774 }
4775 }
4776 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4777 if (weight == -1) {
4778 weightSum = -1;
4779 break;
4780 }
4781 weightSum += weight;
4782 }
4783 // Update best.
4784 if (weightSum > bestWeight) {
4785 bestWeight = weightSum;
4786 bestMAIndex = maIndex;
4787 }
4788 }
4789
4790 // Now select chosen alternative in each constraint.
4791 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4792 cIndex != eIndex; ++cIndex) {
4793 AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
4794 if (cInfo.Type == InlineAsm::isClobber)
4795 continue;
4796 cInfo.selectAlternative(bestMAIndex);
4797 }
4798 }
4799 }
4800
4801 // Check and hook up tied operands, choose constraint code to use.
4802 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4803 cIndex != eIndex; ++cIndex) {
4804 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4805
4806 // If this is an output operand with a matching input operand, look up the
4807 // matching input. If their types mismatch, e.g. one is an integer, the
4808 // other is floating point, or their sizes are different, flag it as an
4809 // error.
4810 if (OpInfo.hasMatchingInput()) {
4811 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4812
4813 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4814 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4815 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4816 OpInfo.ConstraintVT);
4817 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4818 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4819 Input.ConstraintVT);
4820 if ((OpInfo.ConstraintVT.isInteger() !=
4821 Input.ConstraintVT.isInteger()) ||
4822 (MatchRC.second != InputRC.second)) {
4823 report_fatal_error("Unsupported asm: input constraint"
4824 " with a matching output constraint of"
4825 " incompatible type!");
4826 }
4827 }
4828 }
4829 }
4830
4831 return ConstraintOperands;
4832}
4833
4834/// Return an integer indicating how general CT is.
4835static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4836 switch (CT) {
4837 case TargetLowering::C_Immediate:
4838 case TargetLowering::C_Other:
4839 case TargetLowering::C_Unknown:
4840 return 0;
4841 case TargetLowering::C_Register:
4842 return 1;
4843 case TargetLowering::C_RegisterClass:
4844 return 2;
4845 case TargetLowering::C_Memory:
4846 return 3;
4847 }
4848 llvm_unreachable("Invalid constraint type")__builtin_unreachable();
4849}
4850
4851/// Examine constraint type and operand type and determine a weight value.
4852/// This object must already have been set up with the operand type
4853/// and the current alternative constraint selected.
4854TargetLowering::ConstraintWeight
4855 TargetLowering::getMultipleConstraintMatchWeight(
4856 AsmOperandInfo &info, int maIndex) const {
4857 InlineAsm::ConstraintCodeVector *rCodes;
4858 if (maIndex >= (int)info.multipleAlternatives.size())
4859 rCodes = &info.Codes;
4860 else
4861 rCodes = &info.multipleAlternatives[maIndex].Codes;
4862 ConstraintWeight BestWeight = CW_Invalid;
4863
4864 // Loop over the options, keeping track of the most general one.
4865 for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
4866 ConstraintWeight weight =
4867 getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
4868 if (weight > BestWeight)
4869 BestWeight = weight;
4870 }
4871
4872 return BestWeight;
4873}
4874
4875/// Examine constraint type and operand type and determine a weight value.
4876/// This object must already have been set up with the operand type
4877/// and the current alternative constraint selected.
4878TargetLowering::ConstraintWeight
4879 TargetLowering::getSingleConstraintMatchWeight(
4880 AsmOperandInfo &info, const char *constraint) const {
4881 ConstraintWeight weight = CW_Invalid;
4882 Value *CallOperandVal = info.CallOperandVal;
4883 // If we don't have a value, we can't do a match,
4884 // but allow it at the lowest weight.
4885 if (!CallOperandVal)
4886 return CW_Default;
4887 // Look at the constraint type.
4888 switch (*constraint) {
4889 case 'i': // immediate integer.
4890 case 'n': // immediate integer with a known value.
4891 if (isa<ConstantInt>(CallOperandVal))
4892 weight = CW_Constant;
4893 break;
4894 case 's': // non-explicit intregal immediate.
4895 if (isa<GlobalValue>(CallOperandVal))
4896 weight = CW_Constant;
4897 break;
4898 case 'E': // immediate float if host format.
4899 case 'F': // immediate float.
4900 if (isa<ConstantFP>(CallOperandVal))
4901 weight = CW_Constant;
4902 break;
4903 case '<': // memory operand with autodecrement.
4904 case '>': // memory operand with autoincrement.
4905 case 'm': // memory operand.
4906 case 'o': // offsettable memory operand
4907 case 'V': // non-offsettable memory operand
4908 weight = CW_Memory;
4909 break;
4910 case 'r': // general register.
4911 case 'g': // general register, memory operand or immediate integer.
4912 // note: Clang converts "g" to "imr".
4913 if (CallOperandVal->getType()->isIntegerTy())
4914 weight = CW_Register;
4915 break;
4916 case 'X': // any operand.
4917 default:
4918 weight = CW_Default;
4919 break;
4920 }
4921 return weight;
4922}
4923
4924/// If there are multiple different constraints that we could pick for this
4925/// operand (e.g. "imr") try to pick the 'best' one.
4926/// This is somewhat tricky: constraints fall into four classes:
4927/// Other -> immediates and magic values
4928/// Register -> one specific register
4929/// RegisterClass -> a group of regs
4930/// Memory -> memory
4931/// Ideally, we would pick the most specific constraint possible: if we have
4932/// something that fits into a register, we would pick it. The problem here
4933/// is that if we have something that could either be in a register or in
4934/// memory that use of the register could cause selection of *other*
4935/// operands to fail: they might only succeed if we pick memory. Because of
4936/// this the heuristic we use is:
4937///
4938/// 1) If there is an 'other' constraint, and if the operand is valid for
4939/// that constraint, use it. This makes us take advantage of 'i'
4940/// constraints when available.
4941/// 2) Otherwise, pick the most general constraint present. This prefers
4942/// 'm' over 'r', for example.
4943///
4944static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
4945 const TargetLowering &TLI,
4946 SDValue Op, SelectionDAG *DAG) {
4947 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")((void)0);
4948 unsigned BestIdx = 0;
4949 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
4950 int BestGenerality = -1;
4951
4952 // Loop over the options, keeping track of the most general one.
4953 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
4
←
Assuming 'i' is not equal to 'e'
→
5
←
Loop condition is true. Entering loop body
→
4954 TargetLowering::ConstraintType CType =
4955 TLI.getConstraintType(OpInfo.Codes[i]);
4956
4957 // Indirect 'other' or 'immediate' constraints are not allowed.
4958 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6
←
Assuming field 'isIndirect' is false
→
4959 CType == TargetLowering::C_Register ||
4960 CType == TargetLowering::C_RegisterClass))
4961 continue;
4962
4963 // If this is an 'other' or 'immediate' constraint, see if the operand is
4964 // valid for it. For example, on X86 we might have an 'rI' constraint. If
4965 // the operand is an integer in the range [0..31] we want to use I (saving a
4966 // load of a register), otherwise we must use 'r'.
4967 if ((CType
6.1
'CType' is equal to C_Other
6.1
'CType' is equal to C_Other
6.1
'CType' is equal to C_Other
 == TargetLowering::C_Other ||
8
←
Taking true branch
→
4968 CType == TargetLowering::C_Immediate) && Op.getNode()) {
7
←
Assuming the condition is true
→
4969 assert(OpInfo.Codes[i].size() == 1 &&((void)0)
4970 "Unhandled multi-letter 'other' constraint")((void)0);
4971 std::vector<SDValue> ResultOps;
4972 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
9
←
Calling 'TargetLowering::LowerAsmOperandForConstraint'
→
4973 ResultOps, *DAG);
4974 if (!ResultOps.empty()) {
4975 BestType = CType;
4976 BestIdx = i;
4977 break;
4978 }
4979 }
4980
4981 // Things with matching constraints can only be registers, per gcc
4982 // documentation. This mainly affects "g" constraints.
4983 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
4984 continue;
4985
4986 // This constraint letter is more general than the previous one, use it.
4987 int Generality = getConstraintGenerality(CType);
4988 if (Generality > BestGenerality) {
4989 BestType = CType;
4990 BestIdx = i;
4991 BestGenerality = Generality;
4992 }
4993 }
4994
4995 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
4996 OpInfo.ConstraintType = BestType;
4997}
4998
4999/// Determines the constraint code and constraint type to use for the specific
5000/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5001void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
5002 SDValue Op,
5003 SelectionDAG *DAG) const {
5004 assert(!OpInfo.Codes.empty() && "Must have at least one constraint")((void)0);
5005
5006 // Single-letter constraints ('r') are very common.
5007 if (OpInfo.Codes.size() == 1) {
1
Assuming the condition is false
→
2
←
Taking false branch
→
5008 OpInfo.ConstraintCode = OpInfo.Codes[0];
5009 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5010 } else {
5011 ChooseConstraint(OpInfo, *this, Op, DAG);
3
←
Calling 'ChooseConstraint'
→
5012 }
5013
5014 // 'X' matches anything.
5015 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
5016 // Labels and constants are handled elsewhere ('X' is the only thing
5017 // that matches labels). For Functions, the type here is the type of
5018 // the result, which is not what we want to look at; leave them alone.
5019 Value *v = OpInfo.CallOperandVal;
5020 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
5021 OpInfo.CallOperandVal = v;
5022 return;
5023 }
5024
5025 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
5026 return;
5027
5028 // Otherwise, try to resolve it to something we know about by looking at
5029 // the actual operand type.
5030 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
5031 OpInfo.ConstraintCode = Repl;
5032 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5033 }
5034 }
5035}
5036
5037/// Given an exact SDIV by a constant, create a multiplication
5038/// with the multiplicative inverse of the constant.
5039static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
5040 const SDLoc &dl, SelectionDAG &DAG,
5041 SmallVectorImpl<SDNode *> &Created) {
5042 SDValue Op0 = N->getOperand(0);
5043 SDValue Op1 = N->getOperand(1);
5044 EVT VT = N->getValueType(0);
5045 EVT SVT = VT.getScalarType();
5046 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5047 EVT ShSVT = ShVT.getScalarType();
5048
5049 bool UseSRA = false;
5050 SmallVector<SDValue, 16> Shifts, Factors;
5051
5052 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5053 if (C->isNullValue())
5054 return false;
5055 APInt Divisor = C->getAPIntValue();
5056 unsigned Shift = Divisor.countTrailingZeros();
5057 if (Shift) {
5058 Divisor.ashrInPlace(Shift);
5059 UseSRA = true;
5060 }
5061 // Calculate the multiplicative inverse, using Newton's method.
5062 APInt t;
5063 APInt Factor = Divisor;
5064 while ((t = Divisor * Factor) != 1)
5065 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
5066 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
5067 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
5068 return true;
5069 };
5070
5071 // Collect all magic values from the build vector.
5072 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
5073 return SDValue();
5074
5075 SDValue Shift, Factor;
5076 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
5077 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5078 Factor = DAG.getBuildVector(VT, dl, Factors);
5079 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
5080 assert(Shifts.size() == 1 && Factors.size() == 1 &&((void)0)
5081 "Expected matchUnaryPredicate to return one element for scalable "((void)0)
5082 "vectors")((void)0);
5083 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5084 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5085 } else {
5086 assert(isa<ConstantSDNode>(Op1) && "Expected a constant")((void)0);
5087 Shift = Shifts[0];
5088 Factor = Factors[0];
5089 }
5090
5091 SDValue Res = Op0;
5092
5093 // Shift the value upfront if it is even, so the LSB is one.
5094 if (UseSRA) {
5095 // TODO: For UDIV use SRL instead of SRA.
5096 SDNodeFlags Flags;
5097 Flags.setExact(true);
5098 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
5099 Created.push_back(Res.getNode());
5100 }
5101
5102 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5103}
5104
5105SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5106 SelectionDAG &DAG,
5107 SmallVectorImpl<SDNode *> &Created) const {
5108 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5109 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5110 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
5111 return SDValue(N, 0); // Lower SDIV as SDIV
5112 return SDValue();
5113}
5114
5115/// Given an ISD::SDIV node expressing a divide by constant,
5116/// return a DAG expression to select that will generate the same value by
5117/// multiplying by a magic number.
5118/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5119SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
5120 bool IsAfterLegalization,
5121 SmallVectorImpl<SDNode *> &Created) const {
5122 SDLoc dl(N);
5123 EVT VT = N->getValueType(0);
5124 EVT SVT = VT.getScalarType();
5125 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5126 EVT ShSVT = ShVT.getScalarType();
5127 unsigned EltBits = VT.getScalarSizeInBits();
5128 EVT MulVT;
5129
5130 // Check to see if we can do this.
5131 // FIXME: We should be more aggressive here.
5132 if (!isTypeLegal(VT)) {
5133 // Limit this to simple scalars for now.
5134 if (VT.isVector() || !VT.isSimple())
5135 return SDValue();
5136
5137 // If this type will be promoted to a large enough type with a legal
5138 // multiply operation, we can go ahead and do this transform.
5139 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5140 return SDValue();
5141
5142 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5143 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5144 !isOperationLegal(ISD::MUL, MulVT))
5145 return SDValue();
5146 }
5147
5148 // If the sdiv has an 'exact' bit we can use a simpler lowering.
5149 if (N->getFlags().hasExact())
5150 return BuildExactSDIV(*this, N, dl, DAG, Created);
5151
5152 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5153
5154 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5155 if (C->isNullValue())
5156 return false;
5157
5158 const APInt &Divisor = C->getAPIntValue();
5159 APInt::ms magics = Divisor.magic();
5160 int NumeratorFactor = 0;
5161 int ShiftMask = -1;
5162
5163 if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
5164 // If d is +1/-1, we just multiply the numerator by +1/-1.
5165 NumeratorFactor = Divisor.getSExtValue();
5166 magics.m = 0;
5167 magics.s = 0;
5168 ShiftMask = 0;
5169 } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
5170 // If d > 0 and m < 0, add the numerator.
5171 NumeratorFactor = 1;
5172 } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
5173 // If d < 0 and m > 0, subtract the numerator.
5174 NumeratorFactor = -1;
5175 }
5176
5177 MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
5178 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
5179 Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
5180 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
5181 return true;
5182 };
5183
5184 SDValue N0 = N->getOperand(0);
5185 SDValue N1 = N->getOperand(1);
5186
5187 // Collect the shifts / magic values from each element.
5188 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5189 return SDValue();
5190
5191 SDValue MagicFactor, Factor, Shift, ShiftMask;
5192 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5193 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5194 Factor = DAG.getBuildVector(VT, dl, Factors);
5195 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5196 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
5197 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5198 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&((void)0)
5199 Shifts.size() == 1 && ShiftMasks.size() == 1 &&((void)0)
5200 "Expected matchUnaryPredicate to return one element for scalable "((void)0)
5201 "vectors")((void)0);
5202 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5203 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5204 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5205 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
5206 } else {
5207 assert(isa<ConstantSDNode>(N1) && "Expected a constant")((void)0);
5208 MagicFactor = MagicFactors[0];
5209 Factor = Factors[0];
5210 Shift = Shifts[0];
5211 ShiftMask = ShiftMasks[0];
5212 }
5213
5214 // Multiply the numerator (operand 0) by the magic value.
5215 // FIXME: We should support doing a MUL in a wider type.
5216 auto GetMULHS = [&](SDValue X, SDValue Y) {
5217 // If the type isn't legal, use a wider mul of the the type calculated
5218 // earlier.
5219 if (!isTypeLegal(VT)) {
5220 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
5221 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
5222 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5223 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5224 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5225 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5226 }
5227
5228 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
5229 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
5230 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
5231 SDValue LoHi =
5232 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5233 return SDValue(LoHi.getNode(), 1);
5234 }
5235 return SDValue();
5236 };
5237
5238 SDValue Q = GetMULHS(N0, MagicFactor);
5239 if (!Q)
5240 return SDValue();
5241
5242 Created.push_back(Q.getNode());
5243
5244 // (Optionally) Add/subtract the numerator using Factor.
5245 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5246 Created.push_back(Factor.getNode());
5247 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5248 Created.push_back(Q.getNode());
5249
5250 // Shift right algebraic by shift value.
5251 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5252 Created.push_back(Q.getNode());
5253
5254 // Extract the sign bit, mask it and add it to the quotient.
5255 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5256 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5257 Created.push_back(T.getNode());
5258 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5259 Created.push_back(T.getNode());
5260 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5261}
5262
5263/// Given an ISD::UDIV node expressing a divide by constant,
5264/// return a DAG expression to select that will generate the same value by
5265/// multiplying by a magic number.
5266/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5267SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5268 bool IsAfterLegalization,
5269 SmallVectorImpl<SDNode *> &Created) const {
5270 SDLoc dl(N);
5271 EVT VT = N->getValueType(0);
5272 EVT SVT = VT.getScalarType();
5273 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5274 EVT ShSVT = ShVT.getScalarType();
5275 unsigned EltBits = VT.getScalarSizeInBits();
5276 EVT MulVT;
5277
5278 // Check to see if we can do this.
5279 // FIXME: We should be more aggressive here.
5280 if (!isTypeLegal(VT)) {
5281 // Limit this to simple scalars for now.
5282 if (VT.isVector() || !VT.isSimple())
5283 return SDValue();
5284
5285 // If this type will be promoted to a large enough type with a legal
5286 // multiply operation, we can go ahead and do this transform.
5287 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5288 return SDValue();
5289
5290 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5291 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5292 !isOperationLegal(ISD::MUL, MulVT))
5293 return SDValue();
5294 }
5295
5296 bool UseNPQ = false;
5297 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5298
5299 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
5300 if (C->isNullValue())
5301 return false;
5302 // FIXME: We should use a narrower constant when the upper
5303 // bits are known to be zero.
5304 const APInt& Divisor = C->getAPIntValue();
5305 APInt::mu magics = Divisor.magicu();
5306 unsigned PreShift = 0, PostShift = 0;
5307
5308 // If the divisor is even, we can avoid using the expensive fixup by
5309 // shifting the divided value upfront.
5310 if (magics.a != 0 && !Divisor[0]) {
5311 PreShift = Divisor.countTrailingZeros();
5312 // Get magic number for the shifted divisor.
5313 magics = Divisor.lshr(PreShift).magicu(PreShift);
5314 assert(magics.a == 0 && "Should use cheap fixup now")((void)0);
5315 }
5316
5317 APInt Magic = magics.m;
5318
5319 unsigned SelNPQ;
5320 if (magics.a == 0 || Divisor.isOneValue()) {
5321 assert(magics.s < Divisor.getBitWidth() &&((void)0)
5322 "We shouldn't generate an undefined shift!")((void)0);
5323 PostShift = magics.s;
5324 SelNPQ = false;
5325 } else {
5326 PostShift = magics.s - 1;
5327 SelNPQ = true;
5328 }
5329
5330 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
5331 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
5332 NPQFactors.push_back(
5333 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5334 : APInt::getNullValue(EltBits),
5335 dl, SVT));
5336 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
5337 UseNPQ |= SelNPQ;
5338 return true;
5339 };
5340
5341 SDValue N0 = N->getOperand(0);
5342 SDValue N1 = N->getOperand(1);
5343
5344 // Collect the shifts/magic values from each element.
5345 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5346 return SDValue();
5347
5348 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5349 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5350 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
5351 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5352 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
5353 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
5354 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5355 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&((void)0)
5356 NPQFactors.size() == 1 && PostShifts.size() == 1 &&((void)0)
5357 "Expected matchUnaryPredicate to return one for scalable vectors")((void)0);
5358 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
5359 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5360 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
5361 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
5362 } else {
5363 assert(isa<ConstantSDNode>(N1) && "Expected a constant")((void)0);
5364 PreShift = PreShifts[0];
5365 MagicFactor = MagicFactors[0];
5366 PostShift = PostShifts[0];
5367 }
5368
5369 SDValue Q = N0;
5370 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5371 Created.push_back(Q.getNode());
5372
5373 // FIXME: We should support doing a MUL in a wider type.
5374 auto GetMULHU = [&](SDValue X, SDValue Y) {
5375 // If the type isn't legal, use a wider mul of the the type calculated
5376 // earlier.
5377 if (!isTypeLegal(VT)) {
5378 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
5379 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
5380 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5381 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5382 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5383 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5384 }
5385
5386 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
5387 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5388 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
5389 SDValue LoHi =
5390 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5391 return SDValue(LoHi.getNode(), 1);
5392 }
5393 return SDValue(); // No mulhu or equivalent
5394 };
5395
5396 // Multiply the numerator (operand 0) by the magic value.
5397 Q = GetMULHU(Q, MagicFactor);
5398 if (!Q)
5399 return SDValue();
5400
5401 Created.push_back(Q.getNode());
5402
5403 if (UseNPQ) {
5404 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5405 Created.push_back(NPQ.getNode());
5406
5407 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5408 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
5409 if (VT.isVector())
5410 NPQ = GetMULHU(NPQ, NPQFactor);
5411 else
5412 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5413
5414 Created.push_back(NPQ.getNode());
5415
5416 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5417 Created.push_back(Q.getNode());
5418 }
5419
5420 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5421 Created.push_back(Q.getNode());
5422
5423 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5424
5425 SDValue One = DAG.getConstant(1, dl, VT);
5426 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
5427 return DAG.getSelect(dl, VT, IsOne, N0, Q);
5428}
5429
5430/// If all values in Values that *don't* match the predicate are same 'splat'
5431/// value, then replace all values with that splat value.
5432/// Else, if AlternativeReplacement was provided, then replace all values that
5433/// do match predicate with AlternativeReplacement value.
5434static void
5435turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5436 std::function<bool(SDValue)> Predicate,
5437 SDValue AlternativeReplacement = SDValue()) {
5438 SDValue Replacement;
5439 // Is there a value for which the Predicate does *NOT* match? What is it?
5440 auto SplatValue = llvm::find_if_not(Values, Predicate);
5441 if (SplatValue != Values.end()) {
5442 // Does Values consist only of SplatValue's and values matching Predicate?
5443 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5444 return Value == *SplatValue || Predicate(Value);
5445 })) // Then we shall replace values matching predicate with SplatValue.
5446 Replacement = *SplatValue;
5447 }
5448 if (!Replacement) {
5449 // Oops, we did not find the "baseline" splat value.
5450 if (!AlternativeReplacement)
5451 return; // Nothing to do.
5452 // Let's replace with provided value then.
5453 Replacement = AlternativeReplacement;
5454 }
5455 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5456}
5457
5458/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5459/// where the divisor is constant and the comparison target is zero,
5460/// return a DAG expression that will generate the same comparison result
5461/// using only multiplications, additions and shifts/rotations.
5462/// Ref: "Hacker's Delight" 10-17.
5463SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5464 SDValue CompTargetNode,
5465 ISD::CondCode Cond,
5466 DAGCombinerInfo &DCI,
5467 const SDLoc &DL) const {
5468 SmallVector<SDNode *, 5> Built;
5469 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5470 DCI, DL, Built)) {
5471 for (SDNode *N : Built)
5472 DCI.AddToWorklist(N);
5473 return Folded;
5474 }
5475
5476 return SDValue();
5477}
5478
5479SDValue
5480TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5481 SDValue CompTargetNode, ISD::CondCode Cond,
5482 DAGCombinerInfo &DCI, const SDLoc &DL,
5483 SmallVectorImpl<SDNode *> &Created) const {
5484 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5485 // - D must be constant, with D = D0 * 2^K where D0 is odd
5486 // - P is the multiplicative inverse of D0 modulo 2^W
5487 // - Q = floor(((2^W) - 1) / D)
5488 // where W is the width of the common type of N and D.
5489 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&((void)0)
5490 "Only applicable for (in)equality comparisons.")((void)0);
5491
5492 SelectionDAG &DAG = DCI.DAG;
5493
5494 EVT VT = REMNode.getValueType();
5495 EVT SVT = VT.getScalarType();
5496 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5497 EVT ShSVT = ShVT.getScalarType();
5498
5499 // If MUL is unavailable, we cannot proceed in any case.
5500 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5501 return SDValue();
5502
5503 bool ComparingWithAllZeros = true;
5504 bool AllComparisonsWithNonZerosAreTautological = true;
5505 bool HadTautologicalLanes = false;
5506 bool AllLanesAreTautological = true;
5507 bool HadEvenDivisor = false;
5508 bool AllDivisorsArePowerOfTwo = true;
5509 bool HadTautologicalInvertedLanes = false;
5510 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5511
5512 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5513 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5514 if (CDiv->isNullValue())
5515 return false;
5516
5517 const APInt &D = CDiv->getAPIntValue();
5518 const APInt &Cmp = CCmp->getAPIntValue();
5519
5520 ComparingWithAllZeros &= Cmp.isNullValue();
5521
5522 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5523 // if C2 is not less than C1, the comparison is always false.
5524 // But we will only be able to produce the comparison that will give the
5525 // opposive tautological answer. So this lane would need to be fixed up.
5526 bool TautologicalInvertedLane = D.ule(Cmp);
5527 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
5528
5529 // If all lanes are tautological (either all divisors are ones, or divisor
5530 // is not greater than the constant we are comparing with),
5531 // we will prefer to avoid the fold.
5532 bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;
5533 HadTautologicalLanes |= TautologicalLane;
5534 AllLanesAreTautological &= TautologicalLane;
5535
5536 // If we are comparing with non-zero, we need'll need to subtract said
5537 // comparison value from the LHS. But there is no point in doing that if
5538 // every lane where we are comparing with non-zero is tautological..
5539 if (!Cmp.isNullValue())
5540 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
5541
5542 // Decompose D into D0 * 2^K
5543 unsigned K = D.countTrailingZeros();
5544 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")((void)0);
5545 APInt D0 = D.lshr(K);
5546
5547 // D is even if it has trailing zeros.
5548 HadEvenDivisor |= (K != 0);
5549 // D is a power-of-two if D0 is one.
5550 // If all divisors are power-of-two, we will prefer to avoid the fold.
5551 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5552
5553 // P = inv(D0, 2^W)
5554 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5555 unsigned W = D.getBitWidth();
5556 APInt P = D0.zext(W + 1)
5557 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5558 .trunc(W);
5559 assert(!P.isNullValue() && "No multiplicative inverse!")((void)0); // unreachable
5560 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")((void)0);
5561
5562 // Q = floor((2^W - 1) u/ D)
5563 // R = ((2^W - 1) u% D)
5564 APInt Q, R;
5565 APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);
5566
5567 // If we are comparing with zero, then that comparison constant is okay,
5568 // else it may need to be one less than that.
5569 if (Cmp.ugt(R))
5570 Q -= 1;
5571
5572 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((void)0)
5573 "We are expecting that K is always less than all-ones for ShSVT")((void)0);
5574
5575 // If the lane is tautological the result can be constant-folded.
5576 if (TautologicalLane) {
5577 // Set P and K amount to a bogus values so we can try to splat them.
5578 P = 0;
5579 K = -1;
5580 // And ensure that comparison constant is tautological,
5581 // it will always compare true/false.
5582 Q = -1;
5583 }
5584
5585 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5586 KAmts.push_back(
5587 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5588 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5589 return true;
5590 };
5591
5592 SDValue N = REMNode.getOperand(0);
5593 SDValue D = REMNode.getOperand(1);
5594
5595 // Collect the values from each element.
5596 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5597 return SDValue();
5598
5599 // If all lanes are tautological, the result can be constant-folded.
5600 if (AllLanesAreTautological)
5601 return SDValue();
5602
5603 // If this is a urem by a powers-of-two, avoid the fold since it can be
5604 // best implemented as a bit test.
5605 if (AllDivisorsArePowerOfTwo)
5606 return SDValue();
5607
5608 SDValue PVal, KVal, QVal;
5609 if (D.getOpcode() == ISD::BUILD_VECTOR) {
5610 if (HadTautologicalLanes) {
5611 // Try to turn PAmts into a splat, since we don't care about the values
5612 // that are currently '0'. If we can't, just keep '0'`s.
5613 turnVectorIntoSplatVector(PAmts, isNullConstant);
5614 // Try to turn KAmts into a splat, since we don't care about the values
5615 // that are currently '-1'. If we can't, change them to '0'`s.
5616 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5617 DAG.getConstant(0, DL, ShSVT));
5618 }
5619
5620 PVal = DAG.getBuildVector(VT, DL, PAmts);
5621 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5622 QVal = DAG.getBuildVector(VT, DL, QAmts);
5623 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
5624 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&((void)0)
5625 "Expected matchBinaryPredicate to return one element for "((void)0)
5626 "SPLAT_VECTORs")((void)0);
5627 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
5628 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
5629 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
5630 } else {
5631 PVal = PAmts[0];
5632 KVal = KAmts[0];
5633 QVal = QAmts[0];
5634 }
5635
5636 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
5637 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
5638 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5639 assert(CompTargetNode.getValueType() == N.getValueType() &&((void)0)
5640 "Expecting that the types on LHS and RHS of comparisons match.")((void)0);
5641 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5642 }
5643
5644 // (mul N, P)
5645 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5646 Created.push_back(Op0.getNode());
5647
5648 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5649 // divisors as a performance improvement, since rotating by 0 is a no-op.
5650 if (HadEvenDivisor) {
5651 // We need ROTR to do this.
5652 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5653 return SDValue();
5654 // UREM: (rotr (mul N, P), K)
5655 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
5656 Created.push_back(Op0.getNode());
5657 }
5658
5659 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
5660 SDValue NewCC =
5661 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5662 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5663 if (!HadTautologicalInvertedLanes)
5664 return NewCC;
5665
5666 // If any lanes previously compared always-false, the NewCC will give
5667 // always-true result for them, so we need to fixup those lanes.
5668 // Or the other way around for inequality predicate.
5669 assert(VT.isVector() && "Can/should only get here for vectors.")((void)0);
5670 Created.push_back(NewCC.getNode());
5671
5672 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5673 // if C2 is not less than C1, the comparison is always false.
5674 // But we have produced the comparison that will give the
5675 // opposive tautological answer. So these lanes would need to be fixed up.
5676 SDValue TautologicalInvertedChannels =
5677 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5678 Created.push_back(TautologicalInvertedChannels.getNode());
5679
5680 // NOTE: we avoid letting illegal types through even if we're before legalize
5681 // ops – legalization has a hard time producing good code for this.
5682 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5683 // If we have a vector select, let's replace the comparison results in the
5684 // affected lanes with the correct tautological result.
5685 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5686 DL, SETCCVT, SETCCVT);
5687 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5688 Replacement, NewCC);
5689 }
5690
5691 // Else, we can just invert the comparison result in the appropriate lanes.
5692 //
5693 // NOTE: see the note above VSELECT above.
5694 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5695 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5696 TautologicalInvertedChannels);
5697
5698 return SDValue(); // Don't know how to lower.
5699}
5700
5701/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5702/// where the divisor is constant and the comparison target is zero,
5703/// return a DAG expression that will generate the same comparison result
5704/// using only multiplications, additions and shifts/rotations.
5705/// Ref: "Hacker's Delight" 10-17.
5706SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5707 SDValue CompTargetNode,
5708 ISD::CondCode Cond,
5709 DAGCombinerInfo &DCI,
5710 const SDLoc &DL) const {
5711 SmallVector<SDNode *, 7> Built;
5712 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5713 DCI, DL, Built)) {
5714 assert(Built.size() <= 7 && "Max size prediction failed.")((void)0);
5715 for (SDNode *N : Built)
5716 DCI.AddToWorklist(N);
5717 return Folded;
5718 }
5719
5720 return SDValue();
5721}
5722
5723SDValue
5724TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5725 SDValue CompTargetNode, ISD::CondCode Cond,
5726 DAGCombinerInfo &DCI, const SDLoc &DL,
5727 SmallVectorImpl<SDNode *> &Created) const {
5728 // Fold:
5729 // (seteq/ne (srem N, D), 0)
5730 // To:
5731 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
5732 //
5733 // - D must be constant, with D = D0 * 2^K where D0 is odd
5734 // - P is the multiplicative inverse of D0 modulo 2^W
5735 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
5736 // - Q = floor((2 * A) / (2^K))
5737 // where W is the width of the common type of N and D.
5738 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&((void)0)
5739 "Only applicable for (in)equality comparisons.")((void)0);
5740
5741 SelectionDAG &DAG = DCI.DAG;
5742
5743 EVT VT = REMNode.getValueType();
5744 EVT SVT = VT.getScalarType();
5745 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5746 EVT ShSVT = ShVT.getScalarType();
5747
5748 // If we are after ops legalization, and MUL is unavailable, we can not
5749 // proceed.
5750 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5751 return SDValue();
5752
5753 // TODO: Could support comparing with non-zero too.
5754 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
5755 if (!CompTarget || !CompTarget->isNullValue())
5756 return SDValue();
5757
5758 bool HadIntMinDivisor = false;
5759 bool HadOneDivisor = false;
5760 bool AllDivisorsAreOnes = true;
5761 bool HadEvenDivisor = false;
5762 bool NeedToApplyOffset = false;
5763 bool AllDivisorsArePowerOfTwo = true;
5764 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5765
5766 auto BuildSREMPattern = [&](ConstantSDNode *C) {
5767 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5768 if (C->isNullValue())
5769 return false;
5770
5771 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
5772
5773 // WARNING: this fold is only valid for positive divisors!
5774 APInt D = C->getAPIntValue();
5775 if (D.isNegative())
5776 D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`
5777
5778 HadIntMinDivisor |= D.isMinSignedValue();
5779
5780 // If all divisors are ones, we will prefer to avoid the fold.
5781 HadOneDivisor |= D.isOneValue();
5782 AllDivisorsAreOnes &= D.isOneValue();
5783
5784 // Decompose D into D0 * 2^K
5785 unsigned K = D.countTrailingZeros();
5786 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")((void)0);
5787 APInt D0 = D.lshr(K);
5788
5789 if (!D.isMinSignedValue()) {
5790 // D is even if it has trailing zeros; unless it's INT_MIN, in which case
5791 // we don't care about this lane in this fold, we'll special-handle it.
5792 HadEvenDivisor |= (K != 0);
5793 }
5794
5795 // D is a power-of-two if D0 is one. This includes INT_MIN.
5796 // If all divisors are power-of-two, we will prefer to avoid the fold.
5797 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5798
5799 // P = inv(D0, 2^W)
5800 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5801 unsigned W = D.getBitWidth();
5802 APInt P = D0.zext(W + 1)
5803 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5804 .trunc(W);
5805 assert(!P.isNullValue() && "No multiplicative inverse!")((void)0); // unreachable
5806 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")((void)0);
5807
5808 // A = floor((2^(W - 1) - 1) / D0) & -2^K
5809 APInt A = APInt::getSignedMaxValue(W).udiv(D0);
5810 A.clearLowBits(K);
5811
5812 if (!D.isMinSignedValue()) {
5813 // If divisor INT_MIN, then we don't care about this lane in this fold,
5814 // we'll special-handle it.
5815 NeedToApplyOffset |= A != 0;
5816 }
5817
5818 // Q = floor((2 * A) / (2^K))
5819 APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
5820
5821 assert(APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&((void)0)
5822 "We are expecting that A is always less than all-ones for SVT")((void)0);
5823 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((void)0)
5824 "We are expecting that K is always less than all-ones for ShSVT")((void)0);
5825
5826 // If the divisor is 1 the result can be constant-folded. Likewise, we
5827 // don't care about INT_MIN lanes, those can be set to undef if appropriate.
5828 if (D.isOneValue()) {
5829 // Set P, A and K to a bogus values so we can try to splat them.
5830 P = 0;
5831 A = -1;
5832 K = -1;
5833
5834 // x ?% 1 == 0 <--> true <--> x u<= -1
5835 Q = -1;
5836 }
5837
5838 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5839 AAmts.push_back(DAG.getConstant(A, DL, SVT));
5840 KAmts.push_back(
5841 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5842 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5843 return true;
5844 };
5845
5846 SDValue N = REMNode.getOperand(0);
5847 SDValue D = REMNode.getOperand(1);
5848
5849 // Collect the values from each element.
5850 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
5851 return SDValue();
5852
5853 // If this is a srem by a one, avoid the fold since it can be constant-folded.
5854 if (AllDivisorsAreOnes)
5855 return SDValue();
5856
5857 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
5858 // since it can be best implemented as a bit test.
5859 if (AllDivisorsArePowerOfTwo)
5860 return SDValue();
5861
5862 SDValue PVal, AVal, KVal, QVal;
5863 if (D.getOpcode() == ISD::BUILD_VECTOR) {
5864 if (HadOneDivisor) {
5865 // Try to turn PAmts into a splat, since we don't care about the values
5866 // that are currently '0'. If we can't, just keep '0'`s.
5867 turnVectorIntoSplatVector(PAmts, isNullConstant);
5868 // Try to turn AAmts into a splat, since we don't care about the
5869 // values that are currently '-1'. If we can't, change them to '0'`s.
5870 turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
5871 DAG.getConstant(0, DL, SVT));
5872 // Try to turn KAmts into a splat, since we don't care about the values
5873 // that are currently '-1'. If we can't, change them to '0'`s.
5874 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5875 DAG.getConstant(0, DL, ShSVT));
5876 }
5877
5878 PVal = DAG.getBuildVector(VT, DL, PAmts);
5879 AVal = DAG.getBuildVector(VT, DL, AAmts);
5880 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5881 QVal = DAG.getBuildVector(VT, DL, QAmts);
5882 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
5883 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&((void)0)
5884 QAmts.size() == 1 &&((void)0)
5885 "Expected matchUnaryPredicate to return one element for scalable "((void)0)
5886 "vectors")((void)0);
5887 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
5888 AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
5889 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
5890 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
5891 } else {
5892 assert(isa<ConstantSDNode>(D) && "Expected a constant")((void)0);
5893 PVal = PAmts[0];
5894 AVal = AAmts[0];
5895 KVal = KAmts[0];
5896 QVal = QAmts[0];
5897 }
5898
5899 // (mul N, P)
5900 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5901 Created.push_back(Op0.getNode());
5902
5903 if (NeedToApplyOffset) {
5904 // We need ADD to do this.
5905 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
5906 return SDValue();
5907
5908 // (add (mul N, P), A)
5909 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
5910 Created.push_back(Op0.getNode());
5911 }
5912
5913 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5914 // divisors as a performance improvement, since rotating by 0 is a no-op.
5915 if (HadEvenDivisor) {
5916 // We need ROTR to do this.
5917 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5918 return SDValue();
5919 // SREM: (rotr (add (mul N, P), A), K)
5920 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
5921 Created.push_back(Op0.getNode());
5922 }
5923
5924 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
5925 SDValue Fold =
5926 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5927 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5928
5929 // If we didn't have lanes with INT_MIN divisor, then we're done.
5930 if (!HadIntMinDivisor)
5931 return Fold;
5932
5933 // That fold is only valid for positive divisors. Which effectively means,
5934 // it is invalid for INT_MIN divisors. So if we have such a lane,
5935 // we must fix-up results for said lanes.
5936 assert(VT.isVector() && "Can/should only get here for vectors.")((void)0);
5937
5938 // NOTE: we avoid letting illegal types through even if we're before legalize
5939 // ops – legalization has a hard time producing good code for the code that
5940 // follows.
5941 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
5942 !isOperationLegalOrCustom(ISD::AND, VT) ||
5943 !isOperationLegalOrCustom(Cond, VT) ||
5944 !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT))
5945 return SDValue();
5946
5947 Created.push_back(Fold.getNode());
5948
5949 SDValue IntMin = DAG.getConstant(
5950 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
5951 SDValue IntMax = DAG.getConstant(
5952 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
5953 SDValue Zero =
5954 DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT);
5955
5956 // Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
5957 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5958 Created.push_back(DivisorIsIntMin.getNode());
5959
5960 // (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0
5961 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5962 Created.push_back(Masked.getNode());
5963 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
5964 Created.push_back(MaskedIsZero.getNode());
5965
5966 // To produce final result we need to blend 2 vectors: 'SetCC' and
5967 // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
5968 // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
5969 // constant-folded, select can get lowered to a shuffle with constant mask.
5970 SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin,
5971 MaskedIsZero, Fold);
5972
5973 return Blended;
5974}
5975
5976bool TargetLowering::
5977verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
5978 if (!isa<ConstantSDNode>(Op.getOperand(0))) {
5979 DAG.getContext()->emitError("argument to '__builtin_return_address' must "
5980 "be a constant integer");
5981 return true;
5982 }
5983
5984 return false;
5985}
5986
5987SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
5988 const DenormalMode &Mode) const {
5989 SDLoc DL(Op);
5990 EVT VT = Op.getValueType();
5991 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5992 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
5993 // Testing it with denormal inputs to avoid wrong estimate.
5994 if (Mode.Input == DenormalMode::IEEE) {
5995 // This is specifically a check for the handling of denormal inputs,
5996 // not the result.
5997
5998 // Test = fabs(X) < SmallestNormal
5999 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
6000 APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);