clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeIntegerTypes.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 1 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/AMDGPU -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Analysis -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ASMParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/BinaryFormat -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitcode -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Bitstream -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /include/llvm/CodeGen -I /include/llvm/CodeGen/PBQP -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IR -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Coroutines -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData/Coverage -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/CodeView -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/DWARF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/MSF -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/PDB -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Demangle -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/JITLink -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ExecutionEngine/Orc -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenACC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Frontend/OpenMP -I /include/llvm/CodeGen/GlobalISel -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/IRReader -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/Transforms/InstCombine -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/LTO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Linker -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/MC/MCParser -I /include/llvm/CodeGen/MIRParser -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Object -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Option -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Passes -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ProfileData -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Scalar -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/ADT -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Support -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/DebugInfo/Symbolize -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Target -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Utils -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/Vectorize -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include/llvm/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Target/X86 -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/Transforms/IPO -I /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libLLVM/../include -I /usr/src/gnu/usr.bin/clang/libLLVM/obj -I /usr/src/gnu/usr.bin/clang/libLLVM/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -D PIC -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libLLVM/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -D_RET_PROTECTOR -ret-protector -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | |
14 | |
15 | |
16 | |
17 | |
18 | |
19 | |
20 | #include "LegalizeTypes.h" |
21 | #include "llvm/Analysis/TargetLibraryInfo.h" |
22 | #include "llvm/IR/DerivedTypes.h" |
23 | #include "llvm/Support/ErrorHandling.h" |
24 | #include "llvm/Support/KnownBits.h" |
25 | #include "llvm/Support/raw_ostream.h" |
26 | using namespace llvm; |
27 | |
28 | #define DEBUG_TYPE "legalize-types" |
29 | |
30 | |
31 | |
32 | |
33 | |
34 | |
35 | |
36 | |
37 | |
38 | void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { |
39 | LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); |
40 | dbgs() << "\n"); |
41 | SDValue Res = SDValue(); |
42 | |
43 | |
44 | if (CustomLowerNode(N, N->getValueType(ResNo), true)) { |
45 | LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); |
46 | return; |
47 | } |
48 | |
49 | switch (N->getOpcode()) { |
50 | default: |
51 | #ifndef NDEBUG |
52 | dbgs() << "PromoteIntegerResult #" << ResNo << ": "; |
53 | N->dump(&DAG); dbgs() << "\n"; |
54 | #endif |
55 | llvm_unreachable("Do not know how to promote this operator!"); |
56 | case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; |
57 | case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; |
58 | case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; |
59 | case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; |
60 | case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; |
61 | case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; |
62 | case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; |
63 | case ISD::Constant: Res = PromoteIntRes_Constant(N); break; |
64 | case ISD::CTLZ_ZERO_UNDEF: |
65 | case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; |
66 | case ISD::PARITY: |
67 | case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break; |
68 | case ISD::CTTZ_ZERO_UNDEF: |
69 | case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; |
70 | case ISD::EXTRACT_VECTOR_ELT: |
71 | Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; |
72 | case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; |
73 | case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); |
74 | break; |
75 | case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); |
76 | break; |
77 | case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; |
78 | case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; |
79 | case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; |
80 | case ISD::STRICT_FSETCC: |
81 | case ISD::STRICT_FSETCCS: |
82 | case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; |
83 | case ISD::SMIN: |
84 | case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; |
85 | case ISD::UMIN: |
86 | case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break; |
87 | |
88 | case ISD::SHL: Res = PromoteIntRes_SHL(N); break; |
89 | case ISD::SIGN_EXTEND_INREG: |
90 | Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; |
91 | case ISD::SRA: Res = PromoteIntRes_SRA(N); break; |
92 | case ISD::SRL: Res = PromoteIntRes_SRL(N); break; |
93 | case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; |
94 | case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; |
95 | case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; |
96 | case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break; |
97 | |
98 | case ISD::EXTRACT_SUBVECTOR: |
99 | Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; |
100 | case ISD::INSERT_SUBVECTOR: |
101 | Res = PromoteIntRes_INSERT_SUBVECTOR(N); break; |
102 | case ISD::VECTOR_REVERSE: |
103 | Res = PromoteIntRes_VECTOR_REVERSE(N); break; |
104 | case ISD::VECTOR_SHUFFLE: |
105 | Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; |
106 | case ISD::VECTOR_SPLICE: |
107 | Res = PromoteIntRes_VECTOR_SPLICE(N); break; |
108 | case ISD::INSERT_VECTOR_ELT: |
109 | Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; |
110 | case ISD::BUILD_VECTOR: |
111 | Res = PromoteIntRes_BUILD_VECTOR(N); break; |
112 | case ISD::SCALAR_TO_VECTOR: |
113 | Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; |
114 | case ISD::SPLAT_VECTOR: |
115 | Res = PromoteIntRes_SPLAT_VECTOR(N); break; |
116 | case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break; |
117 | case ISD::CONCAT_VECTORS: |
118 | Res = PromoteIntRes_CONCAT_VECTORS(N); break; |
119 | |
120 | case ISD::ANY_EXTEND_VECTOR_INREG: |
121 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
122 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
123 | Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; |
124 | |
125 | case ISD::SIGN_EXTEND: |
126 | case ISD::ZERO_EXTEND: |
127 | case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; |
128 | |
129 | case ISD::STRICT_FP_TO_SINT: |
130 | case ISD::STRICT_FP_TO_UINT: |
131 | case ISD::FP_TO_SINT: |
132 | case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; |
133 | |
134 | case ISD::FP_TO_SINT_SAT: |
135 | case ISD::FP_TO_UINT_SAT: |
136 | Res = PromoteIntRes_FP_TO_XINT_SAT(N); break; |
137 | |
138 | case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; |
139 | |
140 | case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break; |
141 | |
142 | case ISD::AND: |
143 | case ISD::OR: |
144 | case ISD::XOR: |
145 | case ISD::ADD: |
146 | case ISD::SUB: |
147 | case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; |
148 | |
149 | case ISD::SDIV: |
150 | case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; |
151 | |
152 | case ISD::UDIV: |
153 | case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; |
154 | |
155 | case ISD::SADDO: |
156 | case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; |
157 | case ISD::UADDO: |
158 | case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; |
159 | case ISD::SMULO: |
160 | case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; |
161 | |
162 | case ISD::ADDE: |
163 | case ISD::SUBE: |
164 | case ISD::ADDCARRY: |
165 | case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; |
166 | |
167 | case ISD::SADDO_CARRY: |
168 | case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break; |
169 | |
170 | case ISD::SADDSAT: |
171 | case ISD::UADDSAT: |
172 | case ISD::SSUBSAT: |
173 | case ISD::USUBSAT: |
174 | case ISD::SSHLSAT: |
175 | case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break; |
176 | |
177 | case ISD::SMULFIX: |
178 | case ISD::SMULFIXSAT: |
179 | case ISD::UMULFIX: |
180 | case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break; |
181 | |
182 | case ISD::SDIVFIX: |
183 | case ISD::SDIVFIXSAT: |
184 | case ISD::UDIVFIX: |
185 | case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break; |
186 | |
187 | case ISD::ABS: Res = PromoteIntRes_ABS(N); break; |
188 | |
189 | case ISD::ATOMIC_LOAD: |
190 | Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; |
191 | |
192 | case ISD::ATOMIC_LOAD_ADD: |
193 | case ISD::ATOMIC_LOAD_SUB: |
194 | case ISD::ATOMIC_LOAD_AND: |
195 | case ISD::ATOMIC_LOAD_CLR: |
196 | case ISD::ATOMIC_LOAD_OR: |
197 | case ISD::ATOMIC_LOAD_XOR: |
198 | case ISD::ATOMIC_LOAD_NAND: |
199 | case ISD::ATOMIC_LOAD_MIN: |
200 | case ISD::ATOMIC_LOAD_MAX: |
201 | case ISD::ATOMIC_LOAD_UMIN: |
202 | case ISD::ATOMIC_LOAD_UMAX: |
203 | case ISD::ATOMIC_SWAP: |
204 | Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; |
205 | |
206 | case ISD::ATOMIC_CMP_SWAP: |
207 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
208 | Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); |
209 | break; |
210 | |
211 | case ISD::VECREDUCE_ADD: |
212 | case ISD::VECREDUCE_MUL: |
213 | case ISD::VECREDUCE_AND: |
214 | case ISD::VECREDUCE_OR: |
215 | case ISD::VECREDUCE_XOR: |
216 | case ISD::VECREDUCE_SMAX: |
217 | case ISD::VECREDUCE_SMIN: |
218 | case ISD::VECREDUCE_UMAX: |
219 | case ISD::VECREDUCE_UMIN: |
220 | Res = PromoteIntRes_VECREDUCE(N); |
221 | break; |
222 | |
223 | case ISD::FREEZE: |
224 | Res = PromoteIntRes_FREEZE(N); |
225 | break; |
226 | |
227 | case ISD::ROTL: |
228 | case ISD::ROTR: |
229 | Res = PromoteIntRes_Rotate(N); |
230 | break; |
231 | |
232 | case ISD::FSHL: |
233 | case ISD::FSHR: |
234 | Res = PromoteIntRes_FunnelShift(N); |
235 | break; |
236 | } |
237 | |
238 | |
239 | if (Res.getNode()) |
240 | SetPromotedInteger(SDValue(N, ResNo), Res); |
241 | } |
242 | |
243 | SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, |
244 | unsigned ResNo) { |
245 | SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
246 | return GetPromotedInteger(Op); |
247 | } |
248 | |
249 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { |
250 | |
251 | SDValue Op = SExtPromotedInteger(N->getOperand(0)); |
252 | return DAG.getNode(ISD::AssertSext, SDLoc(N), |
253 | Op.getValueType(), Op, N->getOperand(1)); |
254 | } |
255 | |
256 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { |
257 | |
258 | SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
259 | return DAG.getNode(ISD::AssertZext, SDLoc(N), |
260 | Op.getValueType(), Op, N->getOperand(1)); |
261 | } |
262 | |
263 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { |
264 | EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
265 | SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
266 | N->getMemoryVT(), ResVT, |
267 | N->getChain(), N->getBasePtr(), |
268 | N->getMemOperand()); |
269 | |
270 | |
271 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
272 | return Res; |
273 | } |
274 | |
275 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { |
276 | SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
277 | SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
278 | N->getMemoryVT(), |
279 | N->getChain(), N->getBasePtr(), |
280 | Op2, N->getMemOperand()); |
281 | |
282 | |
283 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
284 | return Res; |
285 | } |
286 | |
287 | SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, |
288 | unsigned ResNo) { |
289 | if (ResNo == 1) { |
290 | assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
291 | EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); |
292 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
293 | |
294 | |
295 | |
296 | if (!TLI.isTypeLegal(SVT)) |
297 | SVT = NVT; |
298 | |
299 | SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); |
300 | SDValue Res = DAG.getAtomicCmpSwap( |
301 | ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, |
302 | N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), |
303 | N->getMemOperand()); |
304 | ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); |
305 | ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); |
306 | return Res.getValue(1); |
307 | } |
308 | |
309 | |
310 | |
311 | SDValue Op2 = N->getOperand(2); |
312 | SDValue Op3 = GetPromotedInteger(N->getOperand(3)); |
313 | switch (TLI.getExtendForAtomicCmpSwapArg()) { |
314 | case ISD::SIGN_EXTEND: |
315 | Op2 = SExtPromotedInteger(Op2); |
316 | break; |
317 | case ISD::ZERO_EXTEND: |
318 | Op2 = ZExtPromotedInteger(Op2); |
319 | break; |
320 | case ISD::ANY_EXTEND: |
321 | Op2 = GetPromotedInteger(Op2); |
322 | break; |
323 | default: |
324 | llvm_unreachable("Invalid atomic op extension"); |
325 | } |
326 | |
327 | SDVTList VTs = |
328 | DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); |
329 | SDValue Res = DAG.getAtomicCmpSwap( |
330 | N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), |
331 | N->getBasePtr(), Op2, Op3, N->getMemOperand()); |
332 | |
333 | for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i) |
334 | ReplaceValueWith(SDValue(N, i), Res.getValue(i)); |
335 | return Res; |
336 | } |
337 | |
338 | SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { |
339 | SDValue InOp = N->getOperand(0); |
340 | EVT InVT = InOp.getValueType(); |
341 | EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); |
342 | EVT OutVT = N->getValueType(0); |
343 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
344 | SDLoc dl(N); |
345 | |
346 | switch (getTypeAction(InVT)) { |
347 | case TargetLowering::TypeLegal: |
348 | break; |
349 | case TargetLowering::TypePromoteInteger: |
350 | if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) |
351 | |
352 | return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); |
353 | break; |
354 | case TargetLowering::TypeSoftenFloat: |
355 | |
356 | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); |
357 | case TargetLowering::TypeSoftPromoteHalf: |
358 | |
359 | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp)); |
360 | case TargetLowering::TypePromoteFloat: { |
361 | |
362 | if (!NOutVT.isVector()) |
363 | return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp)); |
364 | break; |
365 | } |
366 | case TargetLowering::TypeExpandInteger: |
367 | case TargetLowering::TypeExpandFloat: |
368 | break; |
369 | case TargetLowering::TypeScalarizeVector: |
370 | |
371 | if (!NOutVT.isVector()) |
372 | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
373 | BitConvertToInteger(GetScalarizedVector(InOp))); |
374 | break; |
375 | case TargetLowering::TypeScalarizeScalableVector: |
376 | report_fatal_error("Scalarization of scalable vectors is not supported."); |
377 | case TargetLowering::TypeSplitVector: { |
378 | if (!NOutVT.isVector()) { |
379 | |
380 | |
381 | SDValue Lo, Hi; |
382 | GetSplitVector(N->getOperand(0), Lo, Hi); |
383 | Lo = BitConvertToInteger(Lo); |
384 | Hi = BitConvertToInteger(Hi); |
385 | |
386 | if (DAG.getDataLayout().isBigEndian()) |
387 | std::swap(Lo, Hi); |
388 | |
389 | InOp = DAG.getNode(ISD::ANY_EXTEND, dl, |
390 | EVT::getIntegerVT(*DAG.getContext(), |
391 | NOutVT.getSizeInBits()), |
392 | JoinIntegers(Lo, Hi)); |
393 | return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); |
394 | } |
395 | break; |
396 | } |
397 | case TargetLowering::TypeWidenVector: |
398 | |
399 | |
400 | |
401 | if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) { |
402 | SDValue Res = |
403 | DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); |
404 | |
405 | |
406 | |
407 | if (DAG.getDataLayout().isBigEndian()) { |
408 | unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); |
409 | EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout()); |
410 | assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!"); |
411 | Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res, |
412 | DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); |
413 | } |
414 | return Res; |
415 | } |
416 | |
417 | |
418 | |
419 | if (NOutVT.isVector()) { |
420 | unsigned WidenInSize = NInVT.getSizeInBits(); |
421 | unsigned OutSize = OutVT.getSizeInBits(); |
422 | if (WidenInSize % OutSize == 0) { |
423 | unsigned Scale = WidenInSize / OutSize; |
424 | EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(), |
425 | OutVT.getVectorElementType(), |
426 | OutVT.getVectorNumElements() * Scale); |
427 | if (isTypeLegal(WideOutVT)) { |
428 | InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp)); |
429 | InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp, |
430 | DAG.getVectorIdxConstant(0, dl)); |
431 | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp); |
432 | } |
433 | } |
434 | } |
435 | } |
436 | |
437 | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
438 | CreateStackStoreLoad(InOp, OutVT)); |
439 | } |
440 | |
441 | |
442 | |
443 | static EVT getShiftAmountTyForConstant(EVT VT, const TargetLowering &TLI, |
444 | SelectionDAG &DAG) { |
445 | EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
446 | |
447 | |
448 | if (!ShiftVT.isVector() && |
449 | ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits())) |
450 | ShiftVT = MVT::i32; |
451 | return ShiftVT; |
452 | } |
453 | |
454 | SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) { |
455 | SDValue V = GetPromotedInteger(N->getOperand(0)); |
456 | return DAG.getNode(ISD::FREEZE, SDLoc(N), |
457 | V.getValueType(), V); |
458 | } |
459 | |
460 | SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { |
461 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
462 | EVT OVT = N->getValueType(0); |
463 | EVT NVT = Op.getValueType(); |
464 | SDLoc dl(N); |
465 | |
466 | |
467 | |
468 | |
469 | |
470 | if (!OVT.isVector() && |
471 | !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) { |
472 | if (SDValue Res = TLI.expandBSWAP(N, DAG)) |
473 | return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); |
474 | } |
475 | |
476 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
477 | EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); |
478 | return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), |
479 | DAG.getConstant(DiffBits, dl, ShiftVT)); |
480 | } |
481 | |
482 | SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { |
483 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
484 | EVT OVT = N->getValueType(0); |
485 | EVT NVT = Op.getValueType(); |
486 | SDLoc dl(N); |
487 | |
488 | |
489 | |
490 | |
491 | |
492 | if (!OVT.isVector() && OVT.isSimple() && |
493 | !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) { |
494 | if (SDValue Res = TLI.expandBITREVERSE(N, DAG)) |
495 | return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res); |
496 | } |
497 | |
498 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
499 | EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG); |
500 | return DAG.getNode(ISD::SRL, dl, NVT, |
501 | DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), |
502 | DAG.getConstant(DiffBits, dl, ShiftVT)); |
503 | } |
504 | |
505 | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { |
506 | |
507 | |
508 | return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), |
509 | TLI.getTypeToTransformTo(*DAG.getContext(), |
510 | N->getValueType(0)), JoinIntegers(N->getOperand(0), |
511 | N->getOperand(1))); |
512 | } |
513 | |
514 | SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { |
515 | EVT VT = N->getValueType(0); |
516 | |
517 | SDLoc dl(N); |
518 | |
519 | |
520 | unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
521 | SDValue Result = DAG.getNode(Opc, dl, |
522 | TLI.getTypeToTransformTo(*DAG.getContext(), VT), |
523 | SDValue(N, 0)); |
524 | assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); |
525 | return Result; |
526 | } |
527 | |
528 | SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { |
529 | |
530 | SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
531 | SDLoc dl(N); |
532 | EVT OVT = N->getValueType(0); |
533 | EVT NVT = Op.getValueType(); |
534 | Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); |
535 | |
536 | return DAG.getNode( |
537 | ISD::SUB, dl, NVT, Op, |
538 | DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, |
539 | NVT)); |
540 | } |
541 | |
542 | SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) { |
543 | |
544 | SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
545 | return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op); |
546 | } |
547 | |
548 | SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { |
549 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
550 | EVT OVT = N->getValueType(0); |
551 | EVT NVT = Op.getValueType(); |
552 | SDLoc dl(N); |
553 | if (N->getOpcode() == ISD::CTTZ) { |
554 | |
555 | |
556 | |
557 | auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), |
558 | OVT.getScalarSizeInBits()); |
559 | Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); |
560 | } |
561 | return DAG.getNode(N->getOpcode(), dl, NVT, Op); |
562 | } |
563 | |
564 | SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { |
565 | SDLoc dl(N); |
566 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
567 | |
568 | SDValue Op0 = N->getOperand(0); |
569 | SDValue Op1 = N->getOperand(1); |
570 | |
571 | |
572 | |
573 | if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType()) |
574 | == TargetLowering::TypePromoteInteger) { |
575 | SDValue In = GetPromotedInteger(Op0); |
576 | |
577 | |
578 | |
579 | EVT SVT = In.getValueType().getScalarType(); |
580 | if (SVT.bitsGE(NVT)) { |
581 | SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1); |
582 | return DAG.getAnyExtOrTrunc(Ext, dl, NVT); |
583 | } |
584 | } |
585 | |
586 | return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1); |
587 | } |
588 | |
589 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { |
590 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
591 | unsigned NewOpc = N->getOpcode(); |
592 | SDLoc dl(N); |
593 | |
594 | |
595 | |
596 | |
597 | |
598 | if (N->getOpcode() == ISD::FP_TO_UINT && |
599 | !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && |
600 | TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) |
601 | NewOpc = ISD::FP_TO_SINT; |
602 | |
603 | if (N->getOpcode() == ISD::STRICT_FP_TO_UINT && |
604 | !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) && |
605 | TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) |
606 | NewOpc = ISD::STRICT_FP_TO_SINT; |
607 | |
608 | SDValue Res; |
609 | if (N->isStrictFPOpcode()) { |
610 | Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, |
611 | {N->getOperand(0), N->getOperand(1)}); |
612 | |
613 | |
614 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
615 | } else |
616 | Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); |
617 | |
618 | |
619 | |
620 | |
621 | |
622 | |
623 | |
624 | |
625 | return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT || |
626 | N->getOpcode() == ISD::STRICT_FP_TO_UINT) ? |
627 | ISD::AssertZext : ISD::AssertSext, dl, NVT, Res, |
628 | DAG.getValueType(N->getValueType(0).getScalarType())); |
629 | } |
630 | |
631 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) { |
632 | |
633 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
634 | SDLoc dl(N); |
635 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), |
636 | N->getOperand(1)); |
637 | } |
638 | |
639 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { |
640 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
641 | SDLoc dl(N); |
642 | |
643 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
644 | } |
645 | |
646 | SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) { |
647 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
648 | SDLoc dl(N); |
649 | |
650 | SDValue Res = |
651 | DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0)); |
652 | |
653 | |
654 | |
655 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
656 | return Res; |
657 | } |
658 | |
659 | SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { |
660 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
661 | SDLoc dl(N); |
662 | |
663 | if (getTypeAction(N->getOperand(0).getValueType()) |
664 | == TargetLowering::TypePromoteInteger) { |
665 | SDValue Res = GetPromotedInteger(N->getOperand(0)); |
666 | assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); |
667 | |
668 | |
669 | |
670 | if (NVT == Res.getValueType()) { |
671 | |
672 | if (N->getOpcode() == ISD::SIGN_EXTEND) |
673 | return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
674 | DAG.getValueType(N->getOperand(0).getValueType())); |
675 | if (N->getOpcode() == ISD::ZERO_EXTEND) |
676 | return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType()); |
677 | assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); |
678 | return Res; |
679 | } |
680 | } |
681 | |
682 | |
683 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
684 | } |
685 | |
686 | SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { |
687 | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
688 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
689 | ISD::LoadExtType ExtType = |
690 | ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType(); |
691 | SDLoc dl(N); |
692 | SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), |
693 | N->getMemoryVT(), N->getMemOperand()); |
694 | |
695 | |
696 | |
697 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
698 | return Res; |
699 | } |
700 | |
701 | SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { |
702 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
703 | SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); |
704 | |
705 | SDLoc dl(N); |
706 | SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), |
707 | N->getOffset(), N->getMask(), ExtPassThru, |
708 | N->getMemoryVT(), N->getMemOperand(), |
709 | N->getAddressingMode(), ISD::EXTLOAD); |
710 | |
711 | |
712 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
713 | return Res; |
714 | } |
715 | |
716 | SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { |
717 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
718 | SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); |
719 | assert(NVT == ExtPassThru.getValueType() && |
720 | "Gather result type and the passThru argument type should be the same"); |
721 | |
722 | ISD::LoadExtType ExtType = N->getExtensionType(); |
723 | if (ExtType == ISD::NON_EXTLOAD) |
724 | ExtType = ISD::EXTLOAD; |
725 | |
726 | SDLoc dl(N); |
727 | SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(), |
728 | N->getIndex(), N->getScale() }; |
729 | SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), |
730 | N->getMemoryVT(), dl, Ops, |
731 | N->getMemOperand(), N->getIndexType(), |
732 | ExtType); |
733 | |
734 | |
735 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
736 | return Res; |
737 | } |
738 | |
739 | |
740 | SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { |
741 | |
742 | |
743 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
744 | EVT VT = N->getValueType(0); |
745 | EVT SVT = getSetCCResultType(VT); |
746 | SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; |
747 | unsigned NumOps = N->getNumOperands(); |
748 | assert(NumOps <= 3 && "Too many operands"); |
749 | if (NumOps == 3) |
750 | Ops[2] = N->getOperand(2); |
751 | |
752 | SDLoc dl(N); |
753 | SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), |
754 | makeArrayRef(Ops, NumOps)); |
755 | |
756 | |
757 | |
758 | ReplaceValueWith(SDValue(N, 0), Res); |
759 | |
760 | |
761 | return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); |
762 | } |
763 | |
764 | SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) { |
765 | |
766 | |
767 | |
768 | |
769 | |
770 | |
771 | |
772 | SDLoc dl(N); |
773 | SDValue Op1 = N->getOperand(0); |
774 | SDValue Op2 = N->getOperand(1); |
775 | unsigned OldBits = Op1.getScalarValueSizeInBits(); |
776 | |
777 | unsigned Opcode = N->getOpcode(); |
778 | bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT; |
779 | |
780 | SDValue Op1Promoted, Op2Promoted; |
781 | if (IsShift) { |
782 | Op1Promoted = GetPromotedInteger(Op1); |
783 | Op2Promoted = ZExtPromotedInteger(Op2); |
784 | } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) { |
785 | Op1Promoted = ZExtPromotedInteger(Op1); |
786 | Op2Promoted = ZExtPromotedInteger(Op2); |
787 | } else { |
788 | Op1Promoted = SExtPromotedInteger(Op1); |
789 | Op2Promoted = SExtPromotedInteger(Op2); |
790 | } |
791 | EVT PromotedType = Op1Promoted.getValueType(); |
792 | unsigned NewBits = PromotedType.getScalarSizeInBits(); |
793 | |
794 | if (Opcode == ISD::UADDSAT) { |
795 | APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits); |
796 | SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); |
797 | SDValue Add = |
798 | DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted); |
799 | return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax); |
800 | } |
801 | |
802 | |
803 | if (Opcode == ISD::USUBSAT) |
804 | return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted, |
805 | Op2Promoted); |
806 | |
807 | |
808 | |
809 | if (IsShift || TLI.isOperationLegalOrCustom(Opcode, PromotedType)) { |
810 | unsigned ShiftOp; |
811 | switch (Opcode) { |
812 | case ISD::SADDSAT: |
813 | case ISD::SSUBSAT: |
814 | case ISD::SSHLSAT: |
815 | ShiftOp = ISD::SRA; |
816 | break; |
817 | case ISD::USHLSAT: |
818 | ShiftOp = ISD::SRL; |
819 | break; |
820 | default: |
821 | llvm_unreachable("Expected opcode to be signed or unsigned saturation " |
822 | "addition, subtraction or left shift"); |
823 | } |
824 | |
825 | unsigned SHLAmount = NewBits - OldBits; |
826 | EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); |
827 | SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT); |
828 | Op1Promoted = |
829 | DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount); |
830 | if (!IsShift) |
831 | Op2Promoted = |
832 | DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount); |
833 | |
834 | SDValue Result = |
835 | DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted); |
836 | return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); |
837 | } |
838 | |
839 | unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB; |
840 | APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits); |
841 | APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits); |
842 | SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType); |
843 | SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType); |
844 | SDValue Result = |
845 | DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted); |
846 | Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax); |
847 | Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin); |
848 | return Result; |
849 | } |
850 | |
851 | SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { |
852 | |
853 | SDLoc dl(N); |
854 | SDValue Op1Promoted, Op2Promoted; |
855 | bool Signed = |
856 | N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; |
857 | bool Saturating = |
858 | N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT; |
859 | if (Signed) { |
860 | Op1Promoted = SExtPromotedInteger(N->getOperand(0)); |
861 | Op2Promoted = SExtPromotedInteger(N->getOperand(1)); |
862 | } else { |
863 | Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); |
864 | Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); |
865 | } |
866 | EVT OldType = N->getOperand(0).getValueType(); |
867 | EVT PromotedType = Op1Promoted.getValueType(); |
868 | unsigned DiffSize = |
869 | PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); |
870 | |
871 | if (Saturating) { |
872 | |
873 | |
874 | |
875 | |
876 | EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); |
877 | Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, |
878 | DAG.getConstant(DiffSize, dl, ShiftTy)); |
879 | SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, |
880 | Op2Promoted, N->getOperand(2)); |
881 | unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; |
882 | return DAG.getNode(ShiftOp, dl, PromotedType, Result, |
883 | DAG.getConstant(DiffSize, dl, ShiftTy)); |
884 | } |
885 | return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, |
886 | N->getOperand(2)); |
887 | } |
888 | |
889 | static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, |
890 | unsigned SatW, bool Signed, |
891 | const TargetLowering &TLI, |
892 | SelectionDAG &DAG) { |
893 | EVT VT = V.getValueType(); |
894 | unsigned VTW = VT.getScalarSizeInBits(); |
895 | |
896 | if (!Signed) { |
897 | |
898 | |
899 | return DAG.getNode(ISD::UMIN, dl, VT, V, |
900 | DAG.getConstant(APInt::getLowBitsSet(VTW, SatW), |
901 | dl, VT)); |
902 | } |
903 | |
904 | |
905 | |
906 | V = DAG.getNode(ISD::SMIN, dl, VT, V, |
907 | DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1), |
908 | dl, VT)); |
909 | |
910 | |
911 | V = DAG.getNode(ISD::SMAX, dl, VT, V, |
912 | DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1), |
913 | dl, VT)); |
914 | return V; |
915 | } |
916 | |
917 | static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, |
918 | unsigned Scale, const TargetLowering &TLI, |
919 | SelectionDAG &DAG, unsigned SatW = 0) { |
920 | EVT VT = LHS.getValueType(); |
921 | unsigned VTSize = VT.getScalarSizeInBits(); |
922 | bool Signed = N->getOpcode() == ISD::SDIVFIX || |
923 | N->getOpcode() == ISD::SDIVFIXSAT; |
924 | bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
925 | N->getOpcode() == ISD::UDIVFIXSAT; |
926 | |
927 | SDLoc dl(N); |
928 | |
929 | |
930 | EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2); |
931 | if (VT.isVector()) |
932 | WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, |
933 | VT.getVectorElementCount()); |
934 | if (Signed) { |
935 | LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT); |
936 | RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT); |
937 | } else { |
938 | LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT); |
939 | RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT); |
940 | } |
941 | |
942 | SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale, |
943 | DAG); |
944 | assert(Res && "Expanding DIVFIX with wide type failed?"); |
945 | if (Saturating) { |
946 | |
947 | |
948 | |
949 | assert(SatW <= VTSize && |
950 | "Tried to saturate to more than the original type?"); |
951 | Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed, |
952 | TLI, DAG); |
953 | } |
954 | return DAG.getZExtOrTrunc(Res, dl, VT); |
955 | } |
956 | |
957 | SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) { |
958 | SDLoc dl(N); |
959 | SDValue Op1Promoted, Op2Promoted; |
960 | bool Signed = N->getOpcode() == ISD::SDIVFIX || |
961 | N->getOpcode() == ISD::SDIVFIXSAT; |
962 | bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT || |
963 | N->getOpcode() == ISD::UDIVFIXSAT; |
964 | if (Signed) { |
965 | Op1Promoted = SExtPromotedInteger(N->getOperand(0)); |
966 | Op2Promoted = SExtPromotedInteger(N->getOperand(1)); |
967 | } else { |
968 | Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); |
969 | Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); |
970 | } |
971 | EVT PromotedType = Op1Promoted.getValueType(); |
972 | unsigned Scale = N->getConstantOperandVal(2); |
973 | |
974 | |
975 | |
976 | if (TLI.isTypeLegal(PromotedType)) { |
977 | TargetLowering::LegalizeAction Action = |
978 | TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale); |
979 | if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) { |
980 | EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); |
981 | unsigned Diff = PromotedType.getScalarSizeInBits() - |
982 | N->getValueType(0).getScalarSizeInBits(); |
983 | if (Saturating) |
984 | Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, |
985 | DAG.getConstant(Diff, dl, ShiftTy)); |
986 | SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, |
987 | Op2Promoted, N->getOperand(2)); |
988 | if (Saturating) |
989 | Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res, |
990 | DAG.getConstant(Diff, dl, ShiftTy)); |
991 | return Res; |
992 | } |
993 | } |
994 | |
995 | |
996 | if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted, |
997 | Op2Promoted, Scale, DAG)) { |
998 | if (Saturating) |
999 | Res = SaturateWidenedDIVFIX(Res, dl, |
1000 | N->getValueType(0).getScalarSizeInBits(), |
1001 | Signed, TLI, DAG); |
1002 | return Res; |
1003 | } |
1004 | |
1005 | |
1006 | |
1007 | return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG, |
1008 | N->getValueType(0).getScalarSizeInBits()); |
1009 | } |
1010 | |
1011 | SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { |
1012 | if (ResNo == 1) |
1013 | return PromoteIntRes_Overflow(N); |
1014 | |
1015 | |
1016 | |
1017 | SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
1018 | SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
1019 | EVT OVT = N->getOperand(0).getValueType(); |
1020 | EVT NVT = LHS.getValueType(); |
1021 | SDLoc dl(N); |
1022 | |
1023 | |
1024 | unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB; |
1025 | SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
1026 | |
1027 | |
1028 | |
1029 | SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
1030 | DAG.getValueType(OVT)); |
1031 | |
1032 | Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
1033 | |
1034 | |
1035 | ReplaceValueWith(SDValue(N, 1), Ofl); |
1036 | |
1037 | return Res; |
1038 | } |
1039 | |
1040 | SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { |
1041 | SDValue LHS = GetPromotedInteger(N->getOperand(1)); |
1042 | SDValue RHS = GetPromotedInteger(N->getOperand(2)); |
1043 | return DAG.getSelect(SDLoc(N), |
1044 | LHS.getValueType(), N->getOperand(0), LHS, RHS); |
1045 | } |
1046 | |
1047 | SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { |
1048 | SDValue Mask = N->getOperand(0); |
1049 | |
1050 | SDValue LHS = GetPromotedInteger(N->getOperand(1)); |
1051 | SDValue RHS = GetPromotedInteger(N->getOperand(2)); |
1052 | return DAG.getNode(ISD::VSELECT, SDLoc(N), |
1053 | LHS.getValueType(), Mask, LHS, RHS); |
1054 | } |
1055 | |
1056 | SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { |
1057 | SDValue LHS = GetPromotedInteger(N->getOperand(2)); |
1058 | SDValue RHS = GetPromotedInteger(N->getOperand(3)); |
1059 | return DAG.getNode(ISD::SELECT_CC, SDLoc(N), |
1060 | LHS.getValueType(), N->getOperand(0), |
1061 | N->getOperand(1), LHS, RHS, N->getOperand(4)); |
1062 | } |
1063 | |
1064 | SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { |
1065 | unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; |
1066 | EVT InVT = N->getOperand(OpNo).getValueType(); |
1067 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1068 | |
1069 | EVT SVT = getSetCCResultType(InVT); |
1070 | |
1071 | |
1072 | |
1073 | |
1074 | if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { |
1075 | if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { |
1076 | InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); |
1077 | SVT = getSetCCResultType(InVT); |
1078 | } else { |
1079 | |
1080 | SVT = NVT; |
1081 | } |
1082 | } |
1083 | |
1084 | SDLoc dl(N); |
1085 | assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && |
1086 | "Vector compare must return a vector result!"); |
1087 | |
1088 | |
1089 | SDValue SetCC; |
1090 | if (N->isStrictFPOpcode()) { |
1091 | EVT VTs[] = {SVT, MVT::Other}; |
1092 | SDValue Opers[] = {N->getOperand(0), N->getOperand(1), |
1093 | N->getOperand(2), N->getOperand(3)}; |
1094 | SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers); |
1095 | |
1096 | |
1097 | ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1)); |
1098 | } else |
1099 | SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0), |
1100 | N->getOperand(1), N->getOperand(2)); |
1101 | |
1102 | |
1103 | return DAG.getSExtOrTrunc(SetCC, dl, NVT); |
1104 | } |
1105 | |
1106 | SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { |
1107 | SDValue LHS = GetPromotedInteger(N->getOperand(0)); |
1108 | SDValue RHS = N->getOperand(1); |
1109 | if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1110 | RHS = ZExtPromotedInteger(RHS); |
1111 | return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); |
1112 | } |
1113 | |
1114 | SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { |
1115 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1116 | return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), |
1117 | Op.getValueType(), Op, N->getOperand(1)); |
1118 | } |
1119 | |
1120 | SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { |
1121 | |
1122 | |
1123 | |
1124 | SDValue LHS = GetPromotedInteger(N->getOperand(0)); |
1125 | SDValue RHS = GetPromotedInteger(N->getOperand(1)); |
1126 | return DAG.getNode(N->getOpcode(), SDLoc(N), |
1127 | LHS.getValueType(), LHS, RHS); |
1128 | } |
1129 | |
1130 | SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { |
1131 | |
1132 | SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
1133 | SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
1134 | return DAG.getNode(N->getOpcode(), SDLoc(N), |
1135 | LHS.getValueType(), LHS, RHS); |
1136 | } |
1137 | |
1138 | SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { |
1139 | |
1140 | SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
1141 | SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
1142 | return DAG.getNode(N->getOpcode(), SDLoc(N), |
1143 | LHS.getValueType(), LHS, RHS); |
1144 | } |
1145 | |
1146 | SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) { |
1147 | |
1148 | |
1149 | SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0)); |
1150 | SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1)); |
1151 | return DAG.getNode(N->getOpcode(), SDLoc(N), |
1152 | LHS.getValueType(), LHS, RHS); |
1153 | } |
1154 | |
1155 | SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { |
1156 | |
1157 | SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
1158 | SDValue RHS = N->getOperand(1); |
1159 | if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1160 | RHS = ZExtPromotedInteger(RHS); |
1161 | return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); |
1162 | } |
1163 | |
1164 | SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { |
1165 | |
1166 | SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
1167 | SDValue RHS = N->getOperand(1); |
1168 | if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
1169 | RHS = ZExtPromotedInteger(RHS); |
1170 | return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); |
1171 | } |
1172 | |
1173 | SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) { |
1174 | |
1175 | SDValue Res; |
1176 | TLI.expandROT(N, true , Res, DAG); |
1177 | ReplaceValueWith(SDValue(N, 0), Res); |
1178 | return SDValue(); |
1179 | } |
1180 | |
1181 | SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) { |
1182 | SDValue Hi = GetPromotedInteger(N->getOperand(0)); |
1183 | SDValue Lo = GetPromotedInteger(N->getOperand(1)); |
1184 | SDValue Amount = GetPromotedInteger(N->getOperand(2)); |
1185 | |
1186 | SDLoc DL(N); |
1187 | EVT OldVT = N->getOperand(0).getValueType(); |
1188 | EVT VT = Lo.getValueType(); |
1189 | unsigned Opcode = N->getOpcode(); |
1190 | bool IsFSHR = Opcode == ISD::FSHR; |
1191 | unsigned OldBits = OldVT.getScalarSizeInBits(); |
1192 | unsigned NewBits = VT.getScalarSizeInBits(); |
1193 | |
1194 | |
1195 | Amount = |
1196 | DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT)); |
1197 | |
1198 | |
1199 | |
1200 | |
1201 | |
1202 | |
1203 | if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) && |
1204 | !TLI.isOperationLegalOrCustom(Opcode, VT)) { |
1205 | SDValue HiShift = DAG.getConstant(OldBits, DL, VT); |
1206 | Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift); |
1207 | Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT); |
1208 | SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo); |
1209 | Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount); |
1210 | if (!IsFSHR) |
1211 | Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift); |
1212 | return Res; |
1213 | } |
1214 | |
1215 | |
1216 | SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT); |
1217 | Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset); |
1218 | |
1219 | |
1220 | |
1221 | if (IsFSHR) |
1222 | Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset); |
1223 | |
1224 | return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount); |
1225 | } |
1226 | |
1227 | SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { |
1228 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1229 | SDValue Res; |
1230 | SDValue InOp = N->getOperand(0); |
1231 | SDLoc dl(N); |
1232 | |
1233 | switch (getTypeAction(InOp.getValueType())) { |
1234 | default: llvm_unreachable("Unknown type action!"); |
1235 | case TargetLowering::TypeLegal: |
1236 | case TargetLowering::TypeExpandInteger: |
1237 | Res = InOp; |
1238 | break; |
1239 | case TargetLowering::TypePromoteInteger: |
1240 | Res = GetPromotedInteger(InOp); |
1241 | break; |
1242 | case TargetLowering::TypeSplitVector: { |
1243 | EVT InVT = InOp.getValueType(); |
1244 | assert(InVT.isVector() && "Cannot split scalar types"); |
1245 | ElementCount NumElts = InVT.getVectorElementCount(); |
1246 | assert(NumElts == NVT.getVectorElementCount() && |
1247 | "Dst and Src must have the same number of elements"); |
1248 | assert(isPowerOf2_32(NumElts.getKnownMinValue()) && |
1249 | "Promoted vector type must be a power of two"); |
1250 | |
1251 | SDValue EOp1, EOp2; |
1252 | GetSplitVector(InOp, EOp1, EOp2); |
1253 | |
1254 | EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), |
1255 | NumElts.divideCoefficientBy(2)); |
1256 | EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); |
1257 | EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); |
1258 | |
1259 | return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); |
1260 | } |
1261 | case TargetLowering::TypeWidenVector: { |
1262 | SDValue WideInOp = GetWidenedVector(InOp); |
1263 | |
1264 | |
1265 | unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); |
1266 | EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), |
1267 | N->getValueType(0).getScalarType(), NumElem); |
1268 | SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); |
1269 | |
1270 | |
1271 | EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), |
1272 | NumElem); |
1273 | SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); |
1274 | |
1275 | |
1276 | SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl); |
1277 | return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); |
1278 | } |
1279 | } |
1280 | |
1281 | |
1282 | return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); |
1283 | } |
1284 | |
1285 | SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { |
1286 | if (ResNo == 1) |
1287 | return PromoteIntRes_Overflow(N); |
1288 | |
1289 | |
1290 | |
1291 | SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
1292 | SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
1293 | EVT OVT = N->getOperand(0).getValueType(); |
1294 | EVT NVT = LHS.getValueType(); |
1295 | SDLoc dl(N); |
1296 | |
1297 | |
1298 | unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB; |
1299 | SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
1300 | |
1301 | |
1302 | |
1303 | SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); |
1304 | |
1305 | Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
1306 | |
1307 | |
1308 | ReplaceValueWith(SDValue(N, 1), Ofl); |
1309 | |
1310 | return Res; |
1311 | } |
1312 | |
1313 | |
1314 | |
1315 | |
1316 | SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { |
1317 | if (ResNo == 1) |
1318 | return PromoteIntRes_Overflow(N); |
1319 | |
1320 | |
1321 | |
1322 | |
1323 | |
1324 | |
1325 | |
1326 | |
1327 | |
1328 | |
1329 | |
1330 | SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
1331 | SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
1332 | |
1333 | EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)}; |
1334 | |
1335 | |
1336 | SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs), |
1337 | LHS, RHS, N->getOperand(2)); |
1338 | |
1339 | |
1340 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
1341 | |
1342 | return SDValue(Res.getNode(), 0); |
1343 | } |
1344 | |
1345 | SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N, |
1346 | unsigned ResNo) { |
1347 | assert(ResNo == 1 && "Don't know how to promote other results yet."); |
1348 | return PromoteIntRes_Overflow(N); |
1349 | } |
1350 | |
1351 | SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { |
1352 | SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); |
1353 | return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); |
1354 | } |
1355 | |
1356 | SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { |
1357 | |
1358 | if (ResNo == 1) |
1359 | return PromoteIntRes_Overflow(N); |
1360 | |
1361 | SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
1362 | SDLoc DL(N); |
1363 | EVT SmallVT = LHS.getValueType(); |
1364 | |
1365 | |
1366 | |
1367 | |
1368 | |
1369 | if (N->getOpcode() == ISD::SMULO) { |
1370 | LHS = SExtPromotedInteger(LHS); |
1371 | RHS = SExtPromotedInteger(RHS); |
1372 | } else { |
1373 | LHS = ZExtPromotedInteger(LHS); |
1374 | RHS = ZExtPromotedInteger(RHS); |
1375 | } |
1376 | SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); |
1377 | SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); |
1378 | |
1379 | |
1380 | |
1381 | |
1382 | SDValue Overflow; |
1383 | if (N->getOpcode() == ISD::UMULO) { |
1384 | |
1385 | unsigned Shift = SmallVT.getScalarSizeInBits(); |
1386 | EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG); |
1387 | SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, |
1388 | DAG.getConstant(Shift, DL, ShiftTy)); |
1389 | Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, |
1390 | DAG.getConstant(0, DL, Hi.getValueType()), |
1391 | ISD::SETNE); |
1392 | } else { |
1393 | |
1394 | SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), |
1395 | Mul, DAG.getValueType(SmallVT)); |
1396 | Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); |
1397 | } |
1398 | |
1399 | |
1400 | |
1401 | Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, |
1402 | SDValue(Mul.getNode(), 1)); |
1403 | |
1404 | |
1405 | ReplaceValueWith(SDValue(N, 1), Overflow); |
1406 | return Mul; |
1407 | } |
1408 | |
1409 | SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { |
1410 | return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), |
1411 | N->getValueType(0))); |
1412 | } |
1413 | |
1414 | SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) { |
1415 | EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1416 | |
1417 | APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue(); |
1418 | return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits())); |
1419 | } |
1420 | |
1421 | SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { |
1422 | SDValue Chain = N->getOperand(0); |
1423 | SDValue Ptr = N->getOperand(1); |
1424 | EVT VT = N->getValueType(0); |
1425 | SDLoc dl(N); |
1426 | |
1427 | MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); |
1428 | unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); |
1429 | |
1430 | |
1431 | SmallVector<SDValue, 8> Parts(NumRegs); |
1432 | for (unsigned i = 0; i < NumRegs; ++i) { |
1433 | Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), |
1434 | N->getConstantOperandVal(3)); |
1435 | Chain = Parts[i].getValue(1); |
1436 | } |
1437 | |
1438 | |
1439 | if (DAG.getDataLayout().isBigEndian()) |
1440 | std::reverse(Parts.begin(), Parts.end()); |
1441 | |
1442 | |
1443 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1444 | SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); |
1445 | for (unsigned i = 1; i < NumRegs; ++i) { |
1446 | SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); |
1447 | |
1448 | Part = DAG.getNode(ISD::SHL, dl, NVT, Part, |
1449 | DAG.getConstant(i * RegVT.getSizeInBits(), dl, |
1450 | TLI.getPointerTy(DAG.getDataLayout()))); |
1451 | Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); |
1452 | } |
1453 | |
1454 | |
1455 | |
1456 | ReplaceValueWith(SDValue(N, 1), Chain); |
1457 | |
1458 | return Res; |
1459 | } |
1460 | |
1461 | |
1462 | |
1463 | |
1464 | |
1465 | |
1466 | |
1467 | |
1468 | |
1469 | bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { |
1470 | LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); |
1471 | dbgs() << "\n"); |
1472 | SDValue Res = SDValue(); |
1473 | if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { |
1474 | LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); |
1475 | return false; |
1476 | } |
1477 | |
1478 | switch (N->getOpcode()) { |
1479 | default: |
1480 | #ifndef NDEBUG |
1481 | dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; |
1482 | N->dump(&DAG); dbgs() << "\n"; |
1483 | #endif |
1484 | llvm_unreachable("Do not know how to promote this operator's operand!"); |
1485 | |
1486 | case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; |
1487 | case ISD::ATOMIC_STORE: |
1488 | Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); |
1489 | break; |
1490 | case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; |
1491 | case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; |
1492 | case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; |
1493 | case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; |
1494 | case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; |
1495 | case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; |
1496 | case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; |
1497 | case ISD::INSERT_VECTOR_ELT: |
1498 | Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; |
1499 | case ISD::SCALAR_TO_VECTOR: |
1500 | Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; |
1501 | case ISD::SPLAT_VECTOR: |
1502 | Res = PromoteIntOp_SPLAT_VECTOR(N); break; |
1503 | case ISD::VSELECT: |
1504 | case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; |
1505 | case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; |
1506 | case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; |
1507 | case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; |
1508 | case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; |
1509 | case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; |
1510 | case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), |
1511 | OpNo); break; |
1512 | case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), |
1513 | OpNo); break; |
1514 | case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), |
1515 | OpNo); break; |
1516 | case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), |
1517 | OpNo); break; |
1518 | case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), |
1519 | OpNo); break; |
1520 | case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; |
1521 | case ISD::FP16_TO_FP: |
1522 | case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; |
1523 | case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; |
1524 | case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; |
1525 | case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; |
1526 | |
1527 | case ISD::SHL: |
1528 | case ISD::SRA: |
1529 | case ISD::SRL: |
1530 | case ISD::ROTL: |
1531 | case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; |
1532 | |
1533 | case ISD::SADDO_CARRY: |
1534 | case ISD::SSUBO_CARRY: |
1535 | case ISD::ADDCARRY: |
1536 | case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; |
1537 | |
1538 | case ISD::FRAMEADDR: |
1539 | case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break; |
1540 | |
1541 | case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; |
1542 | |
1543 | case ISD::SMULFIX: |
1544 | case ISD::SMULFIXSAT: |
1545 | case ISD::UMULFIX: |
1546 | case ISD::UMULFIXSAT: |
1547 | case ISD::SDIVFIX: |
1548 | case ISD::SDIVFIXSAT: |
1549 | case ISD::UDIVFIX: |
1550 | case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break; |
1551 | |
1552 | case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break; |
1553 | |
1554 | case ISD::VECREDUCE_ADD: |
1555 | case ISD::VECREDUCE_MUL: |
1556 | case ISD::VECREDUCE_AND: |
1557 | case ISD::VECREDUCE_OR: |
1558 | case ISD::VECREDUCE_XOR: |
1559 | case ISD::VECREDUCE_SMAX: |
1560 | case ISD::VECREDUCE_SMIN: |
1561 | case ISD::VECREDUCE_UMAX: |
1562 | case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; |
1563 | |
1564 | case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break; |
1565 | } |
1566 | |
1567 | |
1568 | if (!Res.getNode()) return false; |
1569 | |
1570 | |
1571 | |
1572 | if (Res.getNode() == N) |
1573 | return true; |
1574 | |
1575 | const bool IsStrictFp = N->isStrictFPOpcode(); |
1576 | assert(Res.getValueType() == N->getValueType(0) && |
1577 | N->getNumValues() == (IsStrictFp ? 2 : 1) && |
1578 | "Invalid operand expansion"); |
1579 | LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: "; |
1580 | Res.dump()); |
1581 | |
1582 | ReplaceValueWith(SDValue(N, 0), Res); |
1583 | if (IsStrictFp) |
1584 | ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); |
1585 | |
1586 | return false; |
1587 | } |
1588 | |
1589 | |
1590 | |
1591 | void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, |
1592 | ISD::CondCode CCCode) { |
1593 | |
1594 | |
1595 | |
1596 | |
1597 | switch (CCCode) { |
1598 | default: llvm_unreachable("Unknown integer comparison!"); |
1599 | case ISD::SETEQ: |
1600 | case ISD::SETNE: { |
1601 | SDValue OpL = GetPromotedInteger(NewLHS); |
1602 | SDValue OpR = GetPromotedInteger(NewRHS); |
1603 | |
1604 | |
1605 | |
1606 | |
1607 | |
1608 | unsigned OpLEffectiveBits = |
1609 | OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1; |
1610 | unsigned OpREffectiveBits = |
1611 | OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1; |
1612 | if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() && |
1613 | OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) { |
1614 | NewLHS = OpL; |
1615 | NewRHS = OpR; |
1616 | } else { |
1617 | NewLHS = SExtOrZExtPromotedInteger(NewLHS); |
1618 | NewRHS = SExtOrZExtPromotedInteger(NewRHS); |
1619 | } |
1620 | break; |
1621 | } |
1622 | case ISD::SETUGE: |
1623 | case ISD::SETUGT: |
1624 | case ISD::SETULE: |
1625 | case ISD::SETULT: |
1626 | NewLHS = SExtOrZExtPromotedInteger(NewLHS); |
1627 | NewRHS = SExtOrZExtPromotedInteger(NewRHS); |
1628 | break; |
1629 | case ISD::SETGE: |
1630 | case ISD::SETGT: |
1631 | case ISD::SETLT: |
1632 | case ISD::SETLE: |
1633 | NewLHS = SExtPromotedInteger(NewLHS); |
1634 | NewRHS = SExtPromotedInteger(NewRHS); |
1635 | break; |
1636 | } |
1637 | } |
1638 | |
1639 | SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { |
1640 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1641 | return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); |
1642 | } |
1643 | |
1644 | SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { |
1645 | SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
1646 | return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), |
1647 | N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); |
1648 | } |
1649 | |
1650 | SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { |
1651 | |
1652 | |
1653 | return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); |
1654 | } |
1655 | |
1656 | SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { |
1657 | assert(OpNo == 2 && "Don't know how to promote this operand!"); |
1658 | |
1659 | SDValue LHS = N->getOperand(2); |
1660 | SDValue RHS = N->getOperand(3); |
1661 | PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); |
1662 | |
1663 | |
1664 | |
1665 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1666 | N->getOperand(1), LHS, RHS, N->getOperand(4)), |
1667 | 0); |
1668 | } |
1669 | |
1670 | SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { |
1671 | assert(OpNo == 1 && "only know how to promote condition"); |
1672 | |
1673 | |
1674 | SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); |
1675 | |
1676 | |
1677 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, |
1678 | N->getOperand(2)), 0); |
1679 | } |
1680 | |
1681 | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { |
1682 | |
1683 | EVT OVT = N->getOperand(0).getValueType(); |
1684 | SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); |
1685 | SDValue Hi = GetPromotedInteger(N->getOperand(1)); |
1686 | assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); |
1687 | SDLoc dl(N); |
1688 | |
1689 | Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, |
1690 | DAG.getConstant(OVT.getSizeInBits(), dl, |
1691 | TLI.getPointerTy(DAG.getDataLayout()))); |
1692 | return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); |
1693 | } |
1694 | |
1695 | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { |
1696 | |
1697 | |
1698 | |
1699 | EVT VecVT = N->getValueType(0); |
1700 | unsigned NumElts = VecVT.getVectorNumElements(); |
1701 | assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && |
1702 | "Legal vector of one illegal element?"); |
1703 | |
1704 | |
1705 | |
1706 | |
1707 | assert(N->getOperand(0).getValueSizeInBits() >= |
1708 | N->getValueType(0).getScalarSizeInBits() && |
1709 | "Type of inserted value narrower than vector element type!"); |
1710 | |
1711 | SmallVector<SDValue, 16> NewOps; |
1712 | for (unsigned i = 0; i < NumElts; ++i) |
1713 | NewOps.push_back(GetPromotedInteger(N->getOperand(i))); |
1714 | |
1715 | return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
1716 | } |
1717 | |
1718 | SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, |
1719 | unsigned OpNo) { |
1720 | if (OpNo == 1) { |
1721 | |
1722 | |
1723 | |
1724 | |
1725 | assert(N->getOperand(1).getValueSizeInBits() >= |
1726 | N->getValueType(0).getScalarSizeInBits() && |
1727 | "Type of inserted value narrower than vector element type!"); |
1728 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1729 | GetPromotedInteger(N->getOperand(1)), |
1730 | N->getOperand(2)), |
1731 | 0); |
1732 | } |
1733 | |
1734 | assert(OpNo == 2 && "Different operand and result vector types?"); |
1735 | |
1736 | |
1737 | SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), |
1738 | TLI.getVectorIdxTy(DAG.getDataLayout())); |
1739 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1740 | N->getOperand(1), Idx), 0); |
1741 | } |
1742 | |
1743 | SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { |
1744 | |
1745 | |
1746 | return SDValue(DAG.UpdateNodeOperands(N, |
1747 | GetPromotedInteger(N->getOperand(0))), 0); |
1748 | } |
1749 | |
1750 | SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) { |
1751 | |
1752 | |
1753 | return SDValue( |
1754 | DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0); |
1755 | } |
1756 | |
1757 | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { |
1758 | assert(OpNo == 0 && "Only know how to promote the condition!"); |
1759 | SDValue Cond = N->getOperand(0); |
1760 | EVT OpTy = N->getOperand(1).getValueType(); |
1761 | |
1762 | if (N->getOpcode() == ISD::VSELECT) |
1763 | if (SDValue Res = WidenVSELECTMask(N)) |
1764 | return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), |
1765 | Res, N->getOperand(1), N->getOperand(2)); |
1766 | |
1767 | |
1768 | EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy; |
1769 | Cond = PromoteTargetBoolean(Cond, OpVT); |
1770 | |
1771 | return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), |
1772 | N->getOperand(2)), 0); |
1773 | } |
1774 | |
1775 | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { |
1776 | assert(OpNo == 0 && "Don't know how to promote this operand!"); |
1777 | |
1778 | SDValue LHS = N->getOperand(0); |
1779 | SDValue RHS = N->getOperand(1); |
1780 | PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); |
1781 | |
1782 | |
1783 | return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), |
1784 | N->getOperand(3), N->getOperand(4)), 0); |
1785 | } |
1786 | |
1787 | SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { |
1788 | assert(OpNo == 0 && "Don't know how to promote this operand!"); |
1789 | |
1790 | SDValue LHS = N->getOperand(0); |
1791 | SDValue RHS = N->getOperand(1); |
1792 | PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); |
1793 | |
1794 | |
1795 | return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); |
1796 | } |
1797 | |
1798 | SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { |
1799 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1800 | ZExtPromotedInteger(N->getOperand(1))), 0); |
1801 | } |
1802 | |
1803 | SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { |
1804 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1805 | SDLoc dl(N); |
1806 | Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
1807 | return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), |
1808 | Op, DAG.getValueType(N->getOperand(0).getValueType())); |
1809 | } |
1810 | |
1811 | SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { |
1812 | return SDValue(DAG.UpdateNodeOperands(N, |
1813 | SExtPromotedInteger(N->getOperand(0))), 0); |
1814 | } |
1815 | |
1816 | SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) { |
1817 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1818 | SExtPromotedInteger(N->getOperand(1))), 0); |
1819 | } |
1820 | |
1821 | SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ |
1822 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
1823 | SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); |
1824 | SDLoc dl(N); |
1825 | |
1826 | SDValue Val = GetPromotedInteger(N->getValue()); |
1827 | |
1828 | |
1829 | return DAG.getTruncStore(Ch, dl, Val, Ptr, |
1830 | N->getMemoryVT(), N->getMemOperand()); |
1831 | } |
1832 | |
1833 | SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, |
1834 | unsigned OpNo) { |
1835 | |
1836 | SDValue DataOp = N->getValue(); |
1837 | EVT DataVT = DataOp.getValueType(); |
1838 | SDValue Mask = N->getMask(); |
1839 | SDLoc dl(N); |
1840 | |
1841 | bool TruncateStore = false; |
1842 | if (OpNo == 4) { |
1843 | Mask = PromoteTargetBoolean(Mask, DataVT); |
1844 | |
1845 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
1846 | NewOps[4] = Mask; |
1847 | return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
1848 | } else { |
1849 | assert(OpNo == 1 && "Unexpected operand for promotion"); |
1850 | DataOp = GetPromotedInteger(DataOp); |
1851 | TruncateStore = true; |
1852 | } |
1853 | |
1854 | return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), |
1855 | N->getOffset(), Mask, N->getMemoryVT(), |
1856 | N->getMemOperand(), N->getAddressingMode(), |
1857 | TruncateStore, N->isCompressingStore()); |
1858 | } |
1859 | |
1860 | SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, |
1861 | unsigned OpNo) { |
1862 | assert(OpNo == 3 && "Only know how to promote the mask!"); |
1863 | EVT DataVT = N->getValueType(0); |
1864 | SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
1865 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
1866 | NewOps[OpNo] = Mask; |
1867 | SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); |
1868 | if (Res == N) |
1869 | return SDValue(Res, 0); |
1870 | |
1871 | |
1872 | ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); |
1873 | ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); |
1874 | return SDValue(); |
1875 | } |
1876 | |
1877 | SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, |
1878 | unsigned OpNo) { |
1879 | |
1880 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
1881 | if (OpNo == 2) { |
1882 | |
1883 | EVT DataVT = N->getValueType(0); |
1884 | NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
1885 | } else if (OpNo == 4) { |
1886 | |
1887 | if (N->isIndexSigned()) |
1888 | |
1889 | NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
1890 | else |
1891 | NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); |
1892 | } else |
1893 | NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
1894 | |
1895 | SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); |
1896 | if (Res == N) |
1897 | return SDValue(Res, 0); |
1898 | |
1899 | |
1900 | ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); |
1901 | ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); |
1902 | return SDValue(); |
1903 | } |
1904 | |
1905 | SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, |
1906 | unsigned OpNo) { |
1907 | bool TruncateStore = N->isTruncatingStore(); |
1908 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
1909 | if (OpNo == 2) { |
1910 | |
1911 | EVT DataVT = N->getValue().getValueType(); |
1912 | NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
1913 | } else if (OpNo == 4) { |
1914 | |
1915 | if (N->isIndexSigned()) |
1916 | |
1917 | NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo)); |
1918 | else |
1919 | NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo)); |
1920 | |
1921 | N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(), |
1922 | N->getMemoryVT(), NewOps[OpNo])); |
1923 | } else { |
1924 | NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
1925 | TruncateStore = true; |
1926 | } |
1927 | |
1928 | return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), |
1929 | SDLoc(N), NewOps, N->getMemOperand(), |
1930 | N->getIndexType(), TruncateStore); |
1931 | } |
1932 | |
1933 | SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { |
1934 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1935 | return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); |
1936 | } |
1937 | |
1938 | SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { |
1939 | return SDValue(DAG.UpdateNodeOperands(N, |
1940 | ZExtPromotedInteger(N->getOperand(0))), 0); |
1941 | } |
1942 | |
1943 | SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) { |
1944 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1945 | ZExtPromotedInteger(N->getOperand(1))), 0); |
1946 | } |
1947 | |
1948 | SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { |
1949 | SDLoc dl(N); |
1950 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1951 | Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
1952 | return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); |
1953 | } |
1954 | |
1955 | SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { |
1956 | assert(OpNo == 2 && "Don't know how to promote this operand!"); |
1957 | |
1958 | SDValue LHS = N->getOperand(0); |
1959 | SDValue RHS = N->getOperand(1); |
1960 | SDValue Carry = N->getOperand(2); |
1961 | SDLoc DL(N); |
1962 | |
1963 | Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); |
1964 | |
1965 | return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); |
1966 | } |
1967 | |
1968 | SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) { |
1969 | SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); |
1970 | return SDValue( |
1971 | DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); |
1972 | } |
1973 | |
1974 | SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) { |
1975 | |
1976 | SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
1977 | return SDValue(DAG.UpdateNodeOperands(N, Op), 0); |
1978 | } |
1979 | |
1980 | SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { |
1981 | assert(OpNo > 1 && "Don't know how to promote this operand!"); |
1982 | |
1983 | |
1984 | SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); |
1985 | SDValue Op3 = ZExtPromotedInteger(N->getOperand(3)); |
1986 | SDValue Op4 = ZExtPromotedInteger(N->getOperand(4)); |
1987 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), |
1988 | Op2, Op3, Op4), |
1989 | 0); |
1990 | } |
1991 | |
1992 | SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) { |
1993 | |
1994 | assert(N->getOpcode() == ISD::FPOWI && "No STRICT_FPOWI support here yet."); |
1995 | |
1996 | |
1997 | |
1998 | |
1999 | |
2000 | |
2001 | |
2002 | |
2003 | |
2004 | RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0)); |
2005 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); |
2006 | if (!TLI.getLibcallName(LC)) { |
2007 | |
2008 | |
2009 | DAG.getContext()->emitError("Don't know how to promote fpowi to fpow"); |
2010 | return DAG.getUNDEF(N->getValueType(0)); |
2011 | } |
2012 | |
2013 | assert(DAG.getLibInfo().getIntSize() == |
2014 | N->getOperand(1).getValueType().getSizeInBits() && |
2015 | "POWI exponent should match with sizeof(int) when doing the libcall."); |
2016 | TargetLowering::MakeLibCallOptions CallOptions; |
2017 | CallOptions.setSExt(true); |
2018 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
2019 | std::pair<SDValue, SDValue> Tmp = |
2020 | TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops, |
2021 | CallOptions, SDLoc(N), SDValue()); |
2022 | ReplaceValueWith(SDValue(N, 0), Tmp.first); |
2023 | return SDValue(); |
2024 | } |
2025 | |
2026 | SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { |
2027 | SDLoc dl(N); |
2028 | SDValue Op; |
2029 | switch (N->getOpcode()) { |
2030 | default: llvm_unreachable("Expected integer vector reduction"); |
2031 | case ISD::VECREDUCE_ADD: |
2032 | case ISD::VECREDUCE_MUL: |
2033 | case ISD::VECREDUCE_AND: |
2034 | case ISD::VECREDUCE_OR: |
2035 | case ISD::VECREDUCE_XOR: |
2036 | Op = GetPromotedInteger(N->getOperand(0)); |
2037 | break; |
2038 | case ISD::VECREDUCE_SMAX: |
2039 | case ISD::VECREDUCE_SMIN: |
2040 | Op = SExtPromotedInteger(N->getOperand(0)); |
2041 | break; |
2042 | case ISD::VECREDUCE_UMAX: |
2043 | case ISD::VECREDUCE_UMIN: |
2044 | Op = ZExtPromotedInteger(N->getOperand(0)); |
2045 | break; |
2046 | } |
2047 | |
2048 | EVT EltVT = Op.getValueType().getVectorElementType(); |
2049 | EVT VT = N->getValueType(0); |
2050 | if (VT.bitsGE(EltVT)) |
2051 | return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op); |
2052 | |
2053 | |
2054 | |
2055 | SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op); |
2056 | return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce); |
2057 | } |
2058 | |
2059 | SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) { |
2060 | SDValue Op = ZExtPromotedInteger(N->getOperand(1)); |
2061 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); |
2062 | } |
2063 | |
2064 | |
2065 | |
2066 | |
2067 | |
2068 | |
2069 | |
2070 | |
2071 | |
2072 | void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { |
2073 | LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); |
| 1 | Loop condition is false. Exiting loop | |
|
2074 | dbgs() << "\n"); |
2075 | SDValue Lo, Hi; |
2076 | Lo = Hi = SDValue(); |
2077 | |
2078 | |
2079 | if (CustomLowerNode(N, N->getValueType(ResNo), true)) |
| 2 | | Assuming the condition is false | |
|
| |
2080 | return; |
2081 | |
2082 | switch (N->getOpcode()) { |
| 4 | | Control jumps to 'case SHL:' at line 2202 | |
|
2083 | default: |
2084 | #ifndef NDEBUG |
2085 | dbgs() << "ExpandIntegerResult #" << ResNo << ": "; |
2086 | N->dump(&DAG); dbgs() << "\n"; |
2087 | #endif |
2088 | report_fatal_error("Do not know how to expand the result of this " |
2089 | "operator!"); |
2090 | |
2091 | case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
2092 | case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; |
2093 | case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
2094 | case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
2095 | case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break; |
2096 | |
2097 | case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; |
2098 | case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; |
2099 | case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; |
2100 | case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; |
2101 | case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; |
2102 | |
2103 | case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; |
2104 | case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; |
2105 | case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; |
2106 | case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; |
2107 | case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; |
2108 | case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break; |
2109 | case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; |
2110 | case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; |
2111 | case ISD::CTLZ_ZERO_UNDEF: |
2112 | case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; |
2113 | case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; |
2114 | case ISD::CTTZ_ZERO_UNDEF: |
2115 | case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; |
2116 | case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; |
2117 | case ISD::STRICT_FP_TO_SINT: |
2118 | case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; |
2119 | case ISD::STRICT_FP_TO_UINT: |
2120 | case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; |
2121 | case ISD::FP_TO_SINT_SAT: |
2122 | case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break; |
2123 | case ISD::STRICT_LLROUND: |
2124 | case ISD::STRICT_LLRINT: |
2125 | case ISD::LLROUND: |
2126 | case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break; |
2127 | case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; |
2128 | case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; |
2129 | case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; |
2130 | case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; |
2131 | case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; |
2132 | case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; |
2133 | case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; |
2134 | case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; |
2135 | case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; |
2136 | case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; |
2137 | case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; |
2138 | case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; |
2139 | |
2140 | case ISD::ATOMIC_LOAD_ADD: |
2141 | case ISD::ATOMIC_LOAD_SUB: |
2142 | case ISD::ATOMIC_LOAD_AND: |
2143 | case ISD::ATOMIC_LOAD_CLR: |
2144 | case ISD::ATOMIC_LOAD_OR: |
2145 | case ISD::ATOMIC_LOAD_XOR: |
2146 | case ISD::ATOMIC_LOAD_NAND: |
2147 | case ISD::ATOMIC_LOAD_MIN: |
2148 | case ISD::ATOMIC_LOAD_MAX: |
2149 | case ISD::ATOMIC_LOAD_UMIN: |
2150 | case ISD::ATOMIC_LOAD_UMAX: |
2151 | case ISD::ATOMIC_SWAP: |
2152 | case ISD::ATOMIC_CMP_SWAP: { |
2153 | std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); |
2154 | SplitInteger(Tmp.first, Lo, Hi); |
2155 | ReplaceValueWith(SDValue(N, 1), Tmp.second); |
2156 | break; |
2157 | } |
2158 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
2159 | AtomicSDNode *AN = cast<AtomicSDNode>(N); |
2160 | SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); |
2161 | SDValue Tmp = DAG.getAtomicCmpSwap( |
2162 | ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, |
2163 | N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), |
2164 | AN->getMemOperand()); |
2165 | |
2166 | |
2167 | |
2168 | |
2169 | SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, |
2170 | N->getOperand(2), ISD::SETEQ); |
2171 | |
2172 | SplitInteger(Tmp, Lo, Hi); |
2173 | ReplaceValueWith(SDValue(N, 1), Success); |
2174 | ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); |
2175 | break; |
2176 | } |
2177 | |
2178 | case ISD::AND: |
2179 | case ISD::OR: |
2180 | case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; |
2181 | |
2182 | case ISD::UMAX: |
2183 | case ISD::SMAX: |
2184 | case ISD::UMIN: |
2185 | case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; |
2186 | |
2187 | case ISD::ADD: |
2188 | case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; |
2189 | |
2190 | case ISD::ADDC: |
2191 | case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; |
2192 | |
2193 | case ISD::ADDE: |
2194 | case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; |
2195 | |
2196 | case ISD::ADDCARRY: |
2197 | case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; |
2198 | |
2199 | case ISD::SADDO_CARRY: |
2200 | case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break; |
2201 | |
2202 | case ISD::SHL: |
2203 | case ISD::SRA: |
2204 | case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; |
| 5 | | Calling 'DAGTypeLegalizer::ExpandIntRes_Shift' | |
|
2205 | |
2206 | case ISD::SADDO: |
2207 | case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; |
2208 | case ISD::UADDO: |
2209 | case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; |
2210 | case ISD::UMULO: |
2211 | case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; |
2212 | |
2213 | case ISD::SADDSAT: |
2214 | case ISD::UADDSAT: |
2215 | case ISD::SSUBSAT: |
2216 | case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; |
2217 | |
2218 | case ISD::SSHLSAT: |
2219 | case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break; |
2220 | |
2221 | case ISD::SMULFIX: |
2222 | case ISD::SMULFIXSAT: |
2223 | case ISD::UMULFIX: |
2224 | case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break; |
2225 | |
2226 | case ISD::SDIVFIX: |
2227 | case ISD::SDIVFIXSAT: |
2228 | case ISD::UDIVFIX: |
2229 | case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break; |
2230 | |
2231 | case ISD::VECREDUCE_ADD: |
2232 | case ISD::VECREDUCE_MUL: |
2233 | case ISD::VECREDUCE_AND: |
2234 | case ISD::VECREDUCE_OR: |
2235 | case ISD::VECREDUCE_XOR: |
2236 | case ISD::VECREDUCE_SMAX: |
2237 | case ISD::VECREDUCE_SMIN: |
2238 | case ISD::VECREDUCE_UMAX: |
2239 | case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; |
2240 | |
2241 | case ISD::ROTL: |
2242 | case ISD::ROTR: |
2243 | ExpandIntRes_Rotate(N, Lo, Hi); |
2244 | break; |
2245 | |
2246 | case ISD::FSHL: |
2247 | case ISD::FSHR: |
2248 | ExpandIntRes_FunnelShift(N, Lo, Hi); |
2249 | break; |
2250 | |
2251 | case ISD::VSCALE: |
2252 | ExpandIntRes_VSCALE(N, Lo, Hi); |
2253 | break; |
2254 | } |
2255 | |
2256 | |
2257 | if (Lo.getNode()) |
2258 | SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); |
2259 | } |
2260 | |
2261 | |
2262 | std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { |
2263 | unsigned Opc = Node->getOpcode(); |
2264 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); |
2265 | AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering(); |
2266 | |
2267 | |
2268 | RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT); |
2269 | EVT RetVT = Node->getValueType(0); |
2270 | TargetLowering::MakeLibCallOptions CallOptions; |
2271 | SmallVector<SDValue, 4> Ops; |
2272 | if (TLI.getLibcallName(LC)) { |
2273 | Ops.append(Node->op_begin() + 2, Node->op_end()); |
2274 | Ops.push_back(Node->getOperand(1)); |
2275 | } else { |
2276 | LC = RTLIB::getSYNC(Opc, VT); |
2277 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
2278 | "Unexpected atomic op or value type!"); |
2279 | Ops.append(Node->op_begin() + 1, Node->op_end()); |
2280 | } |
2281 | return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node), |
2282 | Node->getOperand(0)); |
2283 | } |
2284 | |
2285 | |
2286 | |
2287 | void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, |
2288 | SDValue &Lo, SDValue &Hi) { |
2289 | SDLoc DL(N); |
2290 | |
2291 | SDValue InL, InH; |
2292 | GetExpandedInteger(N->getOperand(0), InL, InH); |
2293 | |
2294 | |
2295 | |
2296 | if (!Amt) { |
2297 | Lo = InL; |
2298 | Hi = InH; |
2299 | return; |
2300 | } |
2301 | |
2302 | EVT NVT = InL.getValueType(); |
2303 | unsigned VTBits = N->getValueType(0).getSizeInBits(); |
2304 | unsigned NVTBits = NVT.getSizeInBits(); |
2305 | EVT ShTy = N->getOperand(1).getValueType(); |
2306 | |
2307 | if (N->getOpcode() == ISD::SHL) { |
2308 | if (Amt.ugt(VTBits)) { |
2309 | Lo = Hi = DAG.getConstant(0, DL, NVT); |
2310 | } else if (Amt.ugt(NVTBits)) { |
2311 | Lo = DAG.getConstant(0, DL, NVT); |
2312 | Hi = DAG.getNode(ISD::SHL, DL, |
2313 | NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
2314 | } else if (Amt == NVTBits) { |
2315 | Lo = DAG.getConstant(0, DL, NVT); |
2316 | Hi = InL; |
2317 | } else { |
2318 | Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); |
2319 | Hi = DAG.getNode(ISD::OR, DL, NVT, |
2320 | DAG.getNode(ISD::SHL, DL, NVT, InH, |
2321 | DAG.getConstant(Amt, DL, ShTy)), |
2322 | DAG.getNode(ISD::SRL, DL, NVT, InL, |
2323 | DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
2324 | } |
2325 | return; |
2326 | } |
2327 | |
2328 | if (N->getOpcode() == ISD::SRL) { |
2329 | if (Amt.ugt(VTBits)) { |
2330 | Lo = Hi = DAG.getConstant(0, DL, NVT); |
2331 | } else if (Amt.ugt(NVTBits)) { |
2332 | Lo = DAG.getNode(ISD::SRL, DL, |
2333 | NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
2334 | Hi = DAG.getConstant(0, DL, NVT); |
2335 | } else if (Amt == NVTBits) { |
2336 | Lo = InH; |
2337 | Hi = DAG.getConstant(0, DL, NVT); |
2338 | } else { |
2339 | Lo = DAG.getNode(ISD::OR, DL, NVT, |
2340 | DAG.getNode(ISD::SRL, DL, NVT, InL, |
2341 | DAG.getConstant(Amt, DL, ShTy)), |
2342 | DAG.getNode(ISD::SHL, DL, NVT, InH, |
2343 | DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
2344 | Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
2345 | } |
2346 | return; |
2347 | } |
2348 | |
2349 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
2350 | if (Amt.ugt(VTBits)) { |
2351 | Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
2352 | DAG.getConstant(NVTBits - 1, DL, ShTy)); |
2353 | } else if (Amt.ugt(NVTBits)) { |
2354 | Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
2355 | DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
2356 | Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
2357 | DAG.getConstant(NVTBits - 1, DL, ShTy)); |
2358 | } else if (Amt == NVTBits) { |
2359 | Lo = InH; |
2360 | Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
2361 | DAG.getConstant(NVTBits - 1, DL, ShTy)); |
2362 | } else { |
2363 | Lo = DAG.getNode(ISD::OR, DL, NVT, |
2364 | DAG.getNode(ISD::SRL, DL, NVT, InL, |
2365 | DAG.getConstant(Amt, DL, ShTy)), |
2366 | DAG.getNode(ISD::SHL, DL, NVT, InH, |
2367 | DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
2368 | Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
2369 | } |
2370 | } |
2371 | |
2372 | |
2373 | |
2374 | |
2375 | |
2376 | bool DAGTypeLegalizer:: |
2377 | ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2378 | SDValue Amt = N->getOperand(1); |
| 9 | | Value assigned to 'Amt.Node' | |
|
2379 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2380 | EVT ShTy = Amt.getValueType(); |
| 10 | | Calling 'SDValue::getValueType' | |
|
2381 | unsigned ShBits = ShTy.getScalarSizeInBits(); |
2382 | unsigned NVTBits = NVT.getScalarSizeInBits(); |
2383 | assert(isPowerOf2_32(NVTBits) && |
2384 | "Expanded integer type size not a power of two!"); |
2385 | SDLoc dl(N); |
2386 | |
2387 | APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); |
2388 | KnownBits Known = DAG.computeKnownBits(N->getOperand(1)); |
2389 | |
2390 | |
2391 | if (((Known.Zero|Known.One) & HighBitMask) == 0) |
2392 | return false; |
2393 | |
2394 | |
2395 | SDValue InL, InH; |
2396 | GetExpandedInteger(N->getOperand(0), InL, InH); |
2397 | |
2398 | |
2399 | |
2400 | if (Known.One.intersects(HighBitMask)) { |
2401 | |
2402 | Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, |
2403 | DAG.getConstant(~HighBitMask, dl, ShTy)); |
2404 | |
2405 | switch (N->getOpcode()) { |
2406 | default: llvm_unreachable("Unknown shift"); |
2407 | case ISD::SHL: |
2408 | Lo = DAG.getConstant(0, dl, NVT); |
2409 | Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); |
2410 | return true; |
2411 | case ISD::SRL: |
2412 | Hi = DAG.getConstant(0, dl, NVT); |
2413 | Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); |
2414 | return true; |
2415 | case ISD::SRA: |
2416 | Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, |
2417 | DAG.getConstant(NVTBits - 1, dl, ShTy)); |
2418 | Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); |
2419 | return true; |
2420 | } |
2421 | } |
2422 | |
2423 | |
2424 | |
2425 | if (HighBitMask.isSubsetOf(Known.Zero)) { |
2426 | |
2427 | |
2428 | |
2429 | SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, |
2430 | DAG.getConstant(NVTBits - 1, dl, ShTy)); |
2431 | |
2432 | unsigned Op1, Op2; |
2433 | switch (N->getOpcode()) { |
2434 | default: llvm_unreachable("Unknown shift"); |
2435 | case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; |
2436 | case ISD::SRL: |
2437 | case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; |
2438 | } |
2439 | |
2440 | |
2441 | if (N->getOpcode() != ISD::SHL) |
2442 | std::swap(InL, InH); |
2443 | |
2444 | |
2445 | |
2446 | SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); |
2447 | |
2448 | SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); |
2449 | |
2450 | Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); |
2451 | Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); |
2452 | |
2453 | if (N->getOpcode() != ISD::SHL) |
2454 | std::swap(Hi, Lo); |
2455 | return true; |
2456 | } |
2457 | |
2458 | return false; |
2459 | } |
2460 | |
2461 | |
2462 | |
2463 | bool DAGTypeLegalizer:: |
2464 | ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2465 | SDValue Amt = N->getOperand(1); |
2466 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2467 | EVT ShTy = Amt.getValueType(); |
2468 | unsigned NVTBits = NVT.getSizeInBits(); |
2469 | assert(isPowerOf2_32(NVTBits) && |
2470 | "Expanded integer type size not a power of two!"); |
2471 | SDLoc dl(N); |
2472 | |
2473 | |
2474 | SDValue InL, InH; |
2475 | GetExpandedInteger(N->getOperand(0), InL, InH); |
2476 | |
2477 | SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); |
2478 | SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); |
2479 | SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); |
2480 | SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
2481 | Amt, NVBitsNode, ISD::SETULT); |
2482 | SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
2483 | Amt, DAG.getConstant(0, dl, ShTy), |
2484 | ISD::SETEQ); |
2485 | |
2486 | SDValue LoS, HiS, LoL, HiL; |
2487 | switch (N->getOpcode()) { |
2488 | default: llvm_unreachable("Unknown shift"); |
2489 | case ISD::SHL: |
2490 | |
2491 | LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); |
2492 | HiS = DAG.getNode(ISD::OR, dl, NVT, |
2493 | DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), |
2494 | DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); |
2495 | |
2496 | |
2497 | LoL = DAG.getConstant(0, dl, NVT); |
2498 | HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); |
2499 | |
2500 | Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); |
2501 | Hi = DAG.getSelect(dl, NVT, isZero, InH, |
2502 | DAG.getSelect(dl, NVT, isShort, HiS, HiL)); |
2503 | return true; |
2504 | case ISD::SRL: |
2505 | |
2506 | HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); |
2507 | LoS = DAG.getNode(ISD::OR, dl, NVT, |
2508 | DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
2509 | |
2510 | |
2511 | DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
2512 | |
2513 | |
2514 | HiL = DAG.getConstant(0, dl, NVT); |
2515 | LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); |
2516 | |
2517 | Lo = DAG.getSelect(dl, NVT, isZero, InL, |
2518 | DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
2519 | Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
2520 | return true; |
2521 | case ISD::SRA: |
2522 | |
2523 | HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); |
2524 | LoS = DAG.getNode(ISD::OR, dl, NVT, |
2525 | DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
2526 | DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
2527 | |
2528 | |
2529 | HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, |
2530 | DAG.getConstant(NVTBits - 1, dl, ShTy)); |
2531 | LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); |
2532 | |
2533 | Lo = DAG.getSelect(dl, NVT, isZero, InL, |
2534 | DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
2535 | Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
2536 | return true; |
2537 | } |
2538 | } |
2539 | |
2540 | static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { |
2541 | |
2542 | switch (Op) { |
2543 | default: llvm_unreachable("invalid min/max opcode"); |
2544 | case ISD::SMAX: |
2545 | return std::make_pair(ISD::SETGT, ISD::UMAX); |
2546 | case ISD::UMAX: |
2547 | return std::make_pair(ISD::SETUGT, ISD::UMAX); |
2548 | case ISD::SMIN: |
2549 | return std::make_pair(ISD::SETLT, ISD::UMIN); |
2550 | case ISD::UMIN: |
2551 | return std::make_pair(ISD::SETULT, ISD::UMIN); |
2552 | } |
2553 | } |
2554 | |
2555 | void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, |
2556 | SDValue &Lo, SDValue &Hi) { |
2557 | SDLoc DL(N); |
2558 | ISD::NodeType LoOpc; |
2559 | ISD::CondCode CondC; |
2560 | std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); |
2561 | |
2562 | |
2563 | SDValue LHSL, LHSH, RHSL, RHSH; |
2564 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
2565 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
2566 | |
2567 | |
2568 | EVT NVT = LHSL.getValueType(); |
2569 | EVT CCT = getSetCCResultType(NVT); |
2570 | |
2571 | |
2572 | Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); |
2573 | |
2574 | |
2575 | |
2576 | SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); |
2577 | SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); |
2578 | |
2579 | |
2580 | SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); |
2581 | |
2582 | |
2583 | SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); |
2584 | |
2585 | Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); |
2586 | } |
2587 | |
2588 | void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, |
2589 | SDValue &Lo, SDValue &Hi) { |
2590 | SDLoc dl(N); |
2591 | |
2592 | SDValue LHSL, LHSH, RHSL, RHSH; |
2593 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
2594 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
2595 | |
2596 | EVT NVT = LHSL.getValueType(); |
2597 | SDValue LoOps[2] = { LHSL, RHSL }; |
2598 | SDValue HiOps[3] = { LHSH, RHSH }; |
2599 | |
2600 | bool HasOpCarry = TLI.isOperationLegalOrCustom( |
2601 | N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY, |
2602 | TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
2603 | if (HasOpCarry) { |
2604 | SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); |
2605 | if (N->getOpcode() == ISD::ADD) { |
2606 | Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
2607 | HiOps[2] = Lo.getValue(1); |
2608 | Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); |
2609 | } else { |
2610 | Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
2611 | HiOps[2] = Lo.getValue(1); |
2612 | Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); |
2613 | } |
2614 | return; |
2615 | } |
2616 | |
2617 | |
2618 | |
2619 | |
2620 | |
2621 | |
2622 | bool hasCarry = |
2623 | TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
2624 | ISD::ADDC : ISD::SUBC, |
2625 | TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
2626 | |
2627 | if (hasCarry) { |
2628 | SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); |
2629 | if (N->getOpcode() == ISD::ADD) { |
2630 | Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
2631 | HiOps[2] = Lo.getValue(1); |
2632 | Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
2633 | } else { |
2634 | Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
2635 | HiOps[2] = Lo.getValue(1); |
2636 | Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
2637 | } |
2638 | return; |
2639 | } |
2640 | |
2641 | bool hasOVF = |
2642 | TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
2643 | ISD::UADDO : ISD::USUBO, |
2644 | TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
2645 | TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); |
2646 | |
2647 | if (hasOVF) { |
2648 | EVT OvfVT = getSetCCResultType(NVT); |
2649 | SDVTList VTList = DAG.getVTList(NVT, OvfVT); |
2650 | int RevOpc; |
2651 | if (N->getOpcode() == ISD::ADD) { |
2652 | RevOpc = ISD::SUB; |
2653 | Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
2654 | Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); |
2655 | } else { |
2656 | RevOpc = ISD::ADD; |
2657 | Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
2658 | Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); |
2659 | } |
2660 | SDValue OVF = Lo.getValue(1); |
2661 | |
2662 | switch (BoolType) { |
2663 | case TargetLoweringBase::UndefinedBooleanContent: |
2664 | OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); |
2665 | LLVM_FALLTHROUGH; |
2666 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
2667 | OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); |
2668 | Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); |
2669 | break; |
2670 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
2671 | OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); |
2672 | Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); |
2673 | } |
2674 | return; |
2675 | } |
2676 | |
2677 | if (N->getOpcode() == ISD::ADD) { |
2678 | Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); |
2679 | Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); |
2680 | SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], |
2681 | ISD::SETULT); |
2682 | |
2683 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) { |
2684 | SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); |
2685 | Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); |
2686 | return; |
2687 | } |
2688 | |
2689 | SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, |
2690 | DAG.getConstant(1, dl, NVT), |
2691 | DAG.getConstant(0, dl, NVT)); |
2692 | SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], |
2693 | ISD::SETULT); |
2694 | SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, |
2695 | DAG.getConstant(1, dl, NVT), Carry1); |
2696 | Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); |
2697 | } else { |
2698 | Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); |
2699 | Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); |
2700 | SDValue Cmp = |
2701 | DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), |
2702 | LoOps[0], LoOps[1], ISD::SETULT); |
2703 | |
2704 | SDValue Borrow; |
2705 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
2706 | Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); |
2707 | else |
2708 | Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), |
2709 | DAG.getConstant(0, dl, NVT)); |
2710 | |
2711 | Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); |
2712 | } |
2713 | } |
2714 | |
2715 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, |
2716 | SDValue &Lo, SDValue &Hi) { |
2717 | |
2718 | SDValue LHSL, LHSH, RHSL, RHSH; |
2719 | SDLoc dl(N); |
2720 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
2721 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
2722 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
2723 | SDValue LoOps[2] = { LHSL, RHSL }; |
2724 | SDValue HiOps[3] = { LHSH, RHSH }; |
2725 | |
2726 | if (N->getOpcode() == ISD::ADDC) { |
2727 | Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
2728 | HiOps[2] = Lo.getValue(1); |
2729 | Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
2730 | } else { |
2731 | Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
2732 | HiOps[2] = Lo.getValue(1); |
2733 | Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
2734 | } |
2735 | |
2736 | |
2737 | |
2738 | ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
2739 | } |
2740 | |
2741 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, |
2742 | SDValue &Lo, SDValue &Hi) { |
2743 | |
2744 | SDValue LHSL, LHSH, RHSL, RHSH; |
2745 | SDLoc dl(N); |
2746 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
2747 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
2748 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
2749 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
2750 | SDValue HiOps[3] = { LHSH, RHSH }; |
2751 | |
2752 | Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
2753 | HiOps[2] = Lo.getValue(1); |
2754 | Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
2755 | |
2756 | |
2757 | |
2758 | ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
2759 | } |
2760 | |
2761 | void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, |
2762 | SDValue &Lo, SDValue &Hi) { |
2763 | SDValue LHS = N->getOperand(0); |
2764 | SDValue RHS = N->getOperand(1); |
2765 | SDLoc dl(N); |
2766 | |
2767 | SDValue Ovf; |
2768 | |
2769 | unsigned CarryOp, NoCarryOp; |
2770 | ISD::CondCode Cond; |
2771 | switch(N->getOpcode()) { |
2772 | case ISD::UADDO: |
2773 | CarryOp = ISD::ADDCARRY; |
2774 | NoCarryOp = ISD::ADD; |
2775 | Cond = ISD::SETULT; |
2776 | break; |
2777 | case ISD::USUBO: |
2778 | CarryOp = ISD::SUBCARRY; |
2779 | NoCarryOp = ISD::SUB; |
2780 | Cond = ISD::SETUGT; |
2781 | break; |
2782 | default: |
2783 | llvm_unreachable("Node has unexpected Opcode"); |
2784 | } |
2785 | |
2786 | bool HasCarryOp = TLI.isOperationLegalOrCustom( |
2787 | CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); |
2788 | |
2789 | if (HasCarryOp) { |
2790 | |
2791 | SDValue LHSL, LHSH, RHSL, RHSH; |
2792 | GetExpandedInteger(LHS, LHSL, LHSH); |
2793 | GetExpandedInteger(RHS, RHSL, RHSH); |
2794 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
2795 | SDValue LoOps[2] = { LHSL, RHSL }; |
2796 | SDValue HiOps[3] = { LHSH, RHSH }; |
2797 | |
2798 | Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
2799 | HiOps[2] = Lo.getValue(1); |
2800 | Hi = DAG.getNode(CarryOp, dl, VTList, HiOps); |
2801 | |
2802 | Ovf = Hi.getValue(1); |
2803 | } else { |
2804 | |
2805 | |
2806 | SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS); |
2807 | SplitInteger(Sum, Lo, Hi); |
2808 | |
2809 | |
2810 | |
2811 | Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); |
2812 | } |
2813 | |
2814 | |
2815 | |
2816 | ReplaceValueWith(SDValue(N, 1), Ovf); |
2817 | } |
2818 | |
2819 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, |
2820 | SDValue &Lo, SDValue &Hi) { |
2821 | |
2822 | SDValue LHSL, LHSH, RHSL, RHSH; |
2823 | SDLoc dl(N); |
2824 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
2825 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
2826 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
2827 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
2828 | SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; |
2829 | |
2830 | Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
2831 | HiOps[2] = Lo.getValue(1); |
2832 | Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
2833 | |
2834 | |
2835 | |
2836 | ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
2837 | } |
2838 | |
2839 | void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N, |
2840 | SDValue &Lo, SDValue &Hi) { |
2841 | |
2842 | SDValue LHSL, LHSH, RHSL, RHSH; |
2843 | SDLoc dl(N); |
2844 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
2845 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
2846 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
2847 | |
2848 | |
2849 | unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY |
2850 | : ISD::SUBCARRY; |
2851 | Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) }); |
2852 | Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); |
2853 | |
2854 | |
2855 | |
2856 | ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
2857 | } |
2858 | |
2859 | void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, |
2860 | SDValue &Lo, SDValue &Hi) { |
2861 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2862 | SDLoc dl(N); |
2863 | SDValue Op = N->getOperand(0); |
2864 | if (Op.getValueType().bitsLE(NVT)) { |
2865 | |
2866 | Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); |
2867 | Hi = DAG.getUNDEF(NVT); |
2868 | } else { |
2869 | |
2870 | |
2871 | assert(getTypeAction(Op.getValueType()) == |
2872 | TargetLowering::TypePromoteInteger && |
2873 | "Only know how to promote this result!"); |
2874 | SDValue Res = GetPromotedInteger(Op); |
2875 | assert(Res.getValueType() == N->getValueType(0) && |
2876 | "Operand over promoted?"); |
2877 | |
2878 | SplitInteger(Res, Lo, Hi); |
2879 | } |
2880 | } |
2881 | |
2882 | void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, |
2883 | SDValue &Lo, SDValue &Hi) { |
2884 | SDLoc dl(N); |
2885 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2886 | EVT NVT = Lo.getValueType(); |
2887 | EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); |
2888 | unsigned NVTBits = NVT.getSizeInBits(); |
2889 | unsigned EVTBits = EVT.getSizeInBits(); |
2890 | |
2891 | if (NVTBits < EVTBits) { |
2892 | Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, |
2893 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
2894 | EVTBits - NVTBits))); |
2895 | } else { |
2896 | Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); |
2897 | |
2898 | Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
2899 | DAG.getConstant(NVTBits - 1, dl, |
2900 | TLI.getPointerTy(DAG.getDataLayout()))); |
2901 | } |
2902 | } |
2903 | |
2904 | void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, |
2905 | SDValue &Lo, SDValue &Hi) { |
2906 | SDLoc dl(N); |
2907 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2908 | EVT NVT = Lo.getValueType(); |
2909 | EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); |
2910 | unsigned NVTBits = NVT.getSizeInBits(); |
2911 | unsigned EVTBits = EVT.getSizeInBits(); |
2912 | |
2913 | if (NVTBits < EVTBits) { |
2914 | Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, |
2915 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
2916 | EVTBits - NVTBits))); |
2917 | } else { |
2918 | Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); |
2919 | |
2920 | Hi = DAG.getConstant(0, dl, NVT); |
2921 | } |
2922 | } |
2923 | |
2924 | void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, |
2925 | SDValue &Lo, SDValue &Hi) { |
2926 | SDLoc dl(N); |
2927 | GetExpandedInteger(N->getOperand(0), Hi, Lo); |
2928 | Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); |
2929 | Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); |
2930 | } |
2931 | |
2932 | void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, |
2933 | SDValue &Lo, SDValue &Hi) { |
2934 | SDLoc dl(N); |
2935 | GetExpandedInteger(N->getOperand(0), Hi, Lo); |
2936 | Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); |
2937 | Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); |
2938 | } |
2939 | |
2940 | void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo, |
2941 | SDValue &Hi) { |
2942 | SDLoc dl(N); |
2943 | |
2944 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2945 | EVT NVT = Lo.getValueType(); |
2946 | Lo = |
2947 | DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi)); |
2948 | Hi = DAG.getConstant(0, dl, NVT); |
2949 | } |
2950 | |
2951 | void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, |
2952 | SDValue &Lo, SDValue &Hi) { |
2953 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2954 | unsigned NBitWidth = NVT.getSizeInBits(); |
2955 | auto Constant = cast<ConstantSDNode>(N); |
2956 | const APInt &Cst = Constant->getAPIntValue(); |
2957 | bool IsTarget = Constant->isTargetOpcode(); |
2958 | bool IsOpaque = Constant->isOpaque(); |
2959 | SDLoc dl(N); |
2960 | Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); |
2961 | Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, |
2962 | IsOpaque); |
2963 | } |
2964 | |
2965 | void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2966 | SDLoc dl(N); |
2967 | |
2968 | SDValue N0 = N->getOperand(0); |
2969 | GetExpandedInteger(N0, Lo, Hi); |
2970 | EVT NVT = Lo.getValueType(); |
2971 | |
2972 | |
2973 | |
2974 | |
2975 | |
2976 | |
2977 | |
2978 | bool HasAddCarry = TLI.isOperationLegalOrCustom( |
2979 | ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
2980 | if (HasAddCarry) { |
2981 | EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG); |
2982 | SDValue Sign = |
2983 | DAG.getNode(ISD::SRA, dl, NVT, Hi, |
2984 | DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy)); |
2985 | SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); |
2986 | Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign); |
2987 | Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1)); |
2988 | Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign); |
2989 | Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign); |
2990 | return; |
2991 | } |
2992 | |
2993 | |
2994 | EVT VT = N->getValueType(0); |
2995 | SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, |
2996 | DAG.getConstant(0, dl, VT), N0); |
2997 | SDValue NegLo, NegHi; |
2998 | SplitInteger(Neg, NegLo, NegHi); |
2999 | |
3000 | SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), |
3001 | DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT); |
3002 | Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); |
3003 | Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); |
3004 | } |
3005 | |
3006 | void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, |
3007 | SDValue &Lo, SDValue &Hi) { |
3008 | SDLoc dl(N); |
3009 | |
3010 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
3011 | EVT NVT = Lo.getValueType(); |
3012 | |
3013 | SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, |
3014 | DAG.getConstant(0, dl, NVT), ISD::SETNE); |
3015 | |
3016 | SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); |
3017 | SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); |
3018 | |
3019 | Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, |
3020 | DAG.getNode(ISD::ADD, dl, NVT, LoLZ, |
3021 | DAG.getConstant(NVT.getSizeInBits(), dl, |
3022 | NVT))); |
3023 | Hi = DAG.getConstant(0, dl, NVT); |
3024 | } |
3025 | |
3026 | void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, |
3027 | SDValue &Lo, SDValue &Hi) { |
3028 | SDLoc dl(N); |
3029 | |
3030 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
3031 | EVT NVT = Lo.getValueType(); |
3032 | Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), |
3033 | DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); |
3034 | Hi = DAG.getConstant(0, dl, NVT); |
3035 | } |
3036 | |
3037 | void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, |
3038 | SDValue &Lo, SDValue &Hi) { |
3039 | SDLoc dl(N); |
3040 | |
3041 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
3042 | EVT NVT = Lo.getValueType(); |
3043 | |
3044 | SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, |
3045 | DAG.getConstant(0, dl, NVT), ISD::SETNE); |
3046 | |
3047 | SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); |
3048 | SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); |
3049 | |
3050 | Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, |
3051 | DAG.getNode(ISD::ADD, dl, NVT, HiLZ, |
3052 | DAG.getConstant(NVT.getSizeInBits(), dl, |
3053 | NVT))); |
3054 | Hi = DAG.getConstant(0, dl, NVT); |
3055 | } |
3056 | |
3057 | void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, |
3058 | SDValue &Hi) { |
3059 | SDLoc dl(N); |
3060 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
3061 | unsigned NBitWidth = NVT.getSizeInBits(); |
3062 | |
3063 | EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); |
3064 | Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0)); |
3065 | SDValue Chain = Lo.getValue(1); |
3066 | |
3067 | Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
3068 | DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy)); |
3069 | |
3070 | |
3071 | |
3072 | ReplaceValueWith(SDValue(N, 1), Chain); |
3073 | } |
3074 | |
3075 | void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, |
3076 | SDValue &Hi) { |
3077 | SDLoc dl(N); |
3078 | EVT VT = N->getValueType(0); |
3079 | |
3080 | bool IsStrict = N->isStrictFPOpcode(); |
3081 | SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
3082 | SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
3083 | if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
3084 | Op = GetPromotedFloat(Op); |
3085 | |
3086 | if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
3087 | EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); |
3088 | Op = GetSoftPromotedHalf(Op); |
3089 | Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); |
3090 | } |
3091 | |
3092 | RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); |
3093 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); |
3094 | TargetLowering::MakeLibCallOptions CallOptions; |
3095 | CallOptions.setSExt(true); |
3096 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, |
3097 | CallOptions, dl, Chain); |
3098 | SplitInteger(Tmp.first, Lo, Hi); |
3099 | |
3100 | if (IsStrict) |
3101 | ReplaceValueWith(SDValue(N, 1), Tmp.second); |
3102 | } |
3103 | |
3104 | void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, |
3105 | SDValue &Hi) { |
3106 | SDLoc dl(N); |
3107 | EVT VT = N->getValueType(0); |
3108 | |
3109 | bool IsStrict = N->isStrictFPOpcode(); |
3110 | SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
3111 | SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
3112 | if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
3113 | Op = GetPromotedFloat(Op); |
3114 | |
3115 | if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) { |
3116 | EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); |
3117 | Op = GetSoftPromotedHalf(Op); |
3118 | Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op); |
3119 | } |
3120 | |
3121 | RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); |
3122 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); |
3123 | TargetLowering::MakeLibCallOptions CallOptions; |
3124 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op, |
3125 | CallOptions, dl, Chain); |
3126 | SplitInteger(Tmp.first, Lo, Hi); |
3127 | |
3128 | if (IsStrict) |
3129 | ReplaceValueWith(SDValue(N, 1), Tmp.second); |
3130 | } |
3131 | |
3132 | void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, |
3133 | SDValue &Hi) { |
3134 | SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); |
3135 | SplitInteger(Res, Lo, Hi); |
3136 | } |
3137 | |
3138 | void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo, |
3139 | SDValue &Hi) { |
3140 | SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); |
3141 | |
3142 | assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && |
3143 | "Input type needs to be promoted!"); |
3144 | |
3145 | EVT VT = Op.getValueType(); |
3146 | |
3147 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3148 | if (N->getOpcode() == ISD::LLROUND || |
3149 | N->getOpcode() == ISD::STRICT_LLROUND) { |
3150 | if (VT == MVT::f32) |
3151 | LC = RTLIB::LLROUND_F32; |
3152 | else if (VT == MVT::f64) |
3153 | LC = RTLIB::LLROUND_F64; |
3154 | else if (VT == MVT::f80) |
3155 | LC = RTLIB::LLROUND_F80; |
3156 | else if (VT == MVT::f128) |
3157 | LC = RTLIB::LLROUND_F128; |
3158 | else if (VT == MVT::ppcf128) |
3159 | LC = RTLIB::LLROUND_PPCF128; |
3160 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); |
3161 | } else if (N->getOpcode() == ISD::LLRINT || |
3162 | N->getOpcode() == ISD::STRICT_LLRINT) { |
3163 | if (VT == MVT::f32) |
3164 | LC = RTLIB::LLRINT_F32; |
3165 | else if (VT == MVT::f64) |
3166 | LC = RTLIB::LLRINT_F64; |
3167 | else if (VT == MVT::f80) |
3168 | LC = RTLIB::LLRINT_F80; |
3169 | else if (VT == MVT::f128) |
3170 | LC = RTLIB::LLRINT_F128; |
3171 | else if (VT == MVT::ppcf128) |
3172 | LC = RTLIB::LLRINT_PPCF128; |
3173 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); |
3174 | } else |
3175 | llvm_unreachable("Unexpected opcode!"); |
3176 | |
3177 | SDLoc dl(N); |
3178 | EVT RetVT = N->getValueType(0); |
3179 | SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue(); |
3180 | |
3181 | TargetLowering::MakeLibCallOptions CallOptions; |
3182 | CallOptions.setSExt(true); |
3183 | std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT, |
3184 | Op, CallOptions, dl, |
3185 | Chain); |
3186 | SplitInteger(Tmp.first, Lo, Hi); |
3187 | |
3188 | if (N->isStrictFPOpcode()) |
3189 | ReplaceValueWith(SDValue(N, 1), Tmp.second); |
3190 | } |
3191 | |
3192 | void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, |
3193 | SDValue &Lo, SDValue &Hi) { |
3194 | if (N->isAtomic()) { |
3195 | |
3196 | SDLoc dl(N); |
3197 | EVT VT = N->getMemoryVT(); |
3198 | SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
3199 | SDValue Zero = DAG.getConstant(0, dl, VT); |
3200 | SDValue Swap = DAG.getAtomicCmpSwap( |
3201 | ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
3202 | VT, VTs, N->getOperand(0), |
3203 | N->getOperand(1), Zero, Zero, N->getMemOperand()); |
3204 | ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); |
3205 | ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); |
3206 | return; |
3207 | } |
3208 | |
3209 | if (ISD::isNormalLoad(N)) { |
3210 | ExpandRes_NormalLoad(N, Lo, Hi); |
3211 | return; |
3212 | } |
3213 | |
3214 | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
3215 | |
3216 | EVT VT = N->getValueType(0); |
3217 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
3218 | SDValue Ch = N->getChain(); |
3219 | SDValue Ptr = N->getBasePtr(); |
3220 | ISD::LoadExtType ExtType = N->getExtensionType(); |
3221 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
3222 | AAMDNodes AAInfo = N->getAAInfo(); |
3223 | SDLoc dl(N); |
3224 | |
3225 | assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
3226 | |
3227 | if (N->getMemoryVT().bitsLE(NVT)) { |
3228 | EVT MemVT = N->getMemoryVT(); |
3229 | |
3230 | Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, |
3231 | N->getOriginalAlign(), MMOFlags, AAInfo); |
3232 | |
3233 | |
3234 | Ch = Lo.getValue(1); |
3235 | |
3236 | if (ExtType == ISD::SEXTLOAD) { |
3237 | |
3238 | |
3239 | unsigned LoSize = Lo.getValueSizeInBits(); |
3240 | Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
3241 | DAG.getConstant(LoSize - 1, dl, |
3242 | TLI.getPointerTy(DAG.getDataLayout()))); |
3243 | } else if (ExtType == ISD::ZEXTLOAD) { |
3244 | |
3245 | Hi = DAG.getConstant(0, dl, NVT); |
3246 | } else { |
3247 | assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); |
3248 | |
3249 | Hi = DAG.getUNDEF(NVT); |
3250 | } |
3251 | } else if (DAG.getDataLayout().isLittleEndian()) { |
3252 | |
3253 | Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), |
3254 | N->getOriginalAlign(), MMOFlags, AAInfo); |
3255 | |
3256 | unsigned ExcessBits = |
3257 | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
3258 | EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
3259 | |
3260 | |
3261 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3262 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
3263 | Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, |
3264 | N->getPointerInfo().getWithOffset(IncrementSize), NEVT, |
3265 | N->getOriginalAlign(), MMOFlags, AAInfo); |
3266 | |
3267 | |
3268 | |
3269 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
3270 | Hi.getValue(1)); |
3271 | } else { |
3272 | |
3273 | |
3274 | EVT MemVT = N->getMemoryVT(); |
3275 | unsigned EBytes = MemVT.getStoreSize(); |
3276 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3277 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
3278 | |
3279 | |
3280 | Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), |
3281 | EVT::getIntegerVT(*DAG.getContext(), |
3282 | MemVT.getSizeInBits() - ExcessBits), |
3283 | N->getOriginalAlign(), MMOFlags, AAInfo); |
3284 | |
3285 | |
3286 | Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl); |
3287 | |
3288 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, |
3289 | N->getPointerInfo().getWithOffset(IncrementSize), |
3290 | EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
3291 | N->getOriginalAlign(), MMOFlags, AAInfo); |
3292 | |
3293 | |
3294 | |
3295 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
3296 | Hi.getValue(1)); |
3297 | |
3298 | if (ExcessBits < NVT.getSizeInBits()) { |
3299 | |
3300 | Lo = DAG.getNode( |
3301 | ISD::OR, dl, NVT, Lo, |
3302 | DAG.getNode(ISD::SHL, dl, NVT, Hi, |
3303 | DAG.getConstant(ExcessBits, dl, |
3304 | TLI.getPointerTy(DAG.getDataLayout())))); |
3305 | |
3306 | Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT, |
3307 | Hi, |
3308 | DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
3309 | TLI.getPointerTy(DAG.getDataLayout()))); |
3310 | } |
3311 | } |
3312 | |
3313 | |
3314 | |
3315 | ReplaceValueWith(SDValue(N, 1), Ch); |
3316 | } |
3317 | |
3318 | void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, |
3319 | SDValue &Lo, SDValue &Hi) { |
3320 | SDLoc dl(N); |
3321 | SDValue LL, LH, RL, RH; |
3322 | GetExpandedInteger(N->getOperand(0), LL, LH); |
3323 | GetExpandedInteger(N->getOperand(1), RL, RH); |
3324 | Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); |
3325 | Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); |
3326 | } |
3327 | |
3328 | void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, |
3329 | SDValue &Lo, SDValue &Hi) { |
3330 | EVT VT = N->getValueType(0); |
3331 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
3332 | SDLoc dl(N); |
3333 | |
3334 | SDValue LL, LH, RL, RH; |
3335 | GetExpandedInteger(N->getOperand(0), LL, LH); |
3336 | GetExpandedInteger(N->getOperand(1), RL, RH); |
3337 | |
3338 | if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, |
3339 | TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
3340 | LL, LH, RL, RH)) |
3341 | return; |
3342 | |
3343 | |
3344 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3345 | if (VT == MVT::i16) |
3346 | LC = RTLIB::MUL_I16; |
3347 | else if (VT == MVT::i32) |
3348 | LC = RTLIB::MUL_I32; |
3349 | else if (VT == MVT::i64) |
3350 | LC = RTLIB::MUL_I64; |
3351 | else if (VT == MVT::i128) |
3352 | LC = RTLIB::MUL_I128; |
3353 | |
3354 | if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) { |
3355 | |
3356 | |
3357 | |
3358 | |
3359 | unsigned Bits = NVT.getSizeInBits(); |
3360 | unsigned HalfBits = Bits >> 1; |
3361 | SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, |
3362 | NVT); |
3363 | SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); |
3364 | SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); |
3365 | |
3366 | SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); |
3367 | SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); |
3368 | |
3369 | EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); |
3370 | if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) { |
3371 | |
3372 | |
3373 | ShiftAmtTy = MVT::i32; |
3374 | } |
3375 | SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy); |
3376 | SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); |
3377 | SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); |
3378 | SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); |
3379 | |
3380 | SDValue U = DAG.getNode(ISD::ADD, dl, NVT, |
3381 | DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); |
3382 | SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); |
3383 | SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); |
3384 | |
3385 | SDValue V = DAG.getNode(ISD::ADD, dl, NVT, |
3386 | DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); |
3387 | SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); |
3388 | |
3389 | SDValue W = DAG.getNode(ISD::ADD, dl, NVT, |
3390 | DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), |
3391 | DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); |
3392 | Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, |
3393 | DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); |
3394 | |
3395 | Hi = DAG.getNode(ISD::ADD, dl, NVT, W, |
3396 | DAG.getNode(ISD::ADD, dl, NVT, |
3397 | DAG.getNode(ISD::MUL, dl, NVT, RH, LL), |
3398 | DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); |
3399 | return; |
3400 | } |
3401 | |
3402 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
3403 | TargetLowering::MakeLibCallOptions CallOptions; |
3404 | CallOptions.setSExt(true); |
3405 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, |
3406 | Lo, Hi); |
3407 | } |
3408 | |
3409 | void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, |
3410 | SDValue &Hi) { |
3411 | SDLoc DL(N); |
3412 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
3413 | SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); |
3414 | SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); |
3415 | Lo = R.getValue(0); |
3416 | Hi = R.getValue(1); |
3417 | ReplaceValueWith(SDValue(N, 1), R.getValue(2)); |
3418 | } |
3419 | |
3420 | void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, |
3421 | SDValue &Hi) { |
3422 | SDValue Result = TLI.expandAddSubSat(N, DAG); |
3423 | SplitInteger(Result, Lo, Hi); |
3424 | } |
3425 | |
3426 | void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo, |
3427 | SDValue &Hi) { |
3428 | SDValue Result = TLI.expandShlSat(N, DAG); |
3429 | SplitInteger(Result, Lo, Hi); |
3430 | } |
3431 | |
3432 | |
3433 | |
3434 | |
3435 | |
3436 | |
3437 | void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, |
3438 | SDValue &Hi) { |
3439 | SDLoc dl(N); |
3440 | EVT VT = N->getValueType(0); |
3441 | unsigned VTSize = VT.getScalarSizeInBits(); |
3442 | SDValue LHS = N->getOperand(0); |
3443 | SDValue RHS = N->getOperand(1); |
3444 | uint64_t Scale = N->getConstantOperandVal(2); |
3445 | bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT || |
3446 | N->getOpcode() == ISD::UMULFIXSAT); |
3447 | bool Signed = (N->getOpcode() == ISD::SMULFIX || |
3448 | N->getOpcode() == ISD::SMULFIXSAT); |
3449 | |
3450 | |
3451 | if (!Scale) { |
3452 | SDValue Result; |
3453 | if (!Saturating) { |
3454 | Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); |
3455 | } else { |
3456 | EVT BoolVT = getSetCCResultType(VT); |
3457 | unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO; |
3458 | Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); |
3459 | SDValue Product = Result.getValue(0); |
3460 | SDValue Overflow = Result.getValue(1); |
3461 | if (Signed) { |
3462 | APInt MinVal = APInt::getSignedMinValue(VTSize); |
3463 | APInt MaxVal = APInt::getSignedMaxValue(VTSize); |
3464 | SDValue SatMin = DAG.getConstant(MinVal, dl, VT); |
3465 | SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); |
3466 | SDValue Zero = DAG.getConstant(0, dl, VT); |
3467 | |
3468 | |
3469 | SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); |
3470 | SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); |
3471 | Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); |
3472 | Result = DAG.getSelect(dl, VT, Overflow, Result, Product); |
3473 | } else { |
3474 | |
3475 | |
3476 | APInt MaxVal = APInt::getMaxValue(VTSize); |
3477 | SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); |
3478 | Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product); |
3479 | } |
3480 | } |
3481 | SplitInteger(Result, Lo, Hi); |
3482 | return; |
3483 | } |
3484 | |
3485 | |
3486 | |
3487 | assert(Scale <= VTSize && "Scale can't be larger than the value type size."); |
3488 | |
3489 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
3490 | SDValue LL, LH, RL, RH; |
3491 | GetExpandedInteger(LHS, LL, LH); |
3492 | GetExpandedInteger(RHS, RL, RH); |
3493 | SmallVector<SDValue, 4> Result; |
3494 | |
3495 | unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; |
3496 | if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, |
3497 | TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
3498 | LL, LH, RL, RH)) { |
3499 | report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); |
3500 | return; |
3501 | } |
3502 | |
3503 | unsigned NVTSize = NVT.getScalarSizeInBits(); |
3504 | assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " |
3505 | "the size of the current value type"); |
3506 | EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); |
3507 | |
3508 | |
3509 | |
3510 | |
3511 | |
3512 | |
3513 | |
3514 | |
3515 | |
3516 | |
3517 | |
3518 | |
3519 | |
3520 | |
3521 | |
3522 | |
3523 | |
3524 | |
3525 | |
3526 | |
3527 | uint64_t Part0 = Scale / NVTSize; |
3528 | if (Scale % NVTSize) { |
3529 | SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy); |
3530 | Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0], |
3531 | ShiftAmount); |
3532 | Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1], |
3533 | ShiftAmount); |
3534 | } else { |
3535 | Lo = Result[Part0]; |
3536 | Hi = Result[Part0 + 1]; |
3537 | } |
3538 | |
3539 | |
3540 | if (!Saturating) |
3541 | return; |
3542 | |
3543 | |
3544 | if (Scale == VTSize) |
3545 | return; |
3546 | |
3547 | |
3548 | |
3549 | |
3550 | |
3551 | |
3552 | |
3553 | |
3554 | |
3555 | |
3556 | |
3557 | |
3558 | |
3559 | SDValue ResultHL = Result[2]; |
3560 | SDValue ResultHH = Result[3]; |
3561 | |
3562 | SDValue SatMax, SatMin; |
3563 | SDValue NVTZero = DAG.getConstant(0, dl, NVT); |
3564 | SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); |
3565 | EVT BoolNVT = getSetCCResultType(NVT); |
3566 | |
3567 | if (!Signed) { |
3568 | if (Scale < NVTSize) { |
3569 | |
3570 | SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, |
3571 | DAG.getConstant(Scale, dl, ShiftTy)); |
3572 | SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH); |
3573 | SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE); |
3574 | } else if (Scale == NVTSize) { |
3575 | |
3576 | SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE); |
3577 | } else if (Scale < VTSize) { |
3578 | |
3579 | SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, |
3580 | DAG.getConstant(Scale - NVTSize, dl, |
3581 | ShiftTy)); |
3582 | SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE); |
3583 | } else |
3584 | llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT" |
3585 | "(and saturation can't happen with Scale==VTSize)."); |
3586 | |
3587 | Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi); |
3588 | Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo); |
3589 | return; |
3590 | } |
3591 | |
3592 | if (Scale < NVTSize) { |
3593 | |
3594 | |
3595 | |
3596 | |
3597 | unsigned OverflowBits = VTSize - Scale + 1; |
3598 | assert(OverflowBits <= VTSize && OverflowBits > NVTSize && |
3599 | "Extent of overflow bits must start within HL"); |
3600 | SDValue HLHiMask = DAG.getConstant( |
3601 | APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); |
3602 | SDValue HLLoMask = DAG.getConstant( |
3603 | APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); |
3604 | |
3605 | SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); |
3606 | SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); |
3607 | SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); |
3608 | SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, |
3609 | DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT)); |
3610 | |
3611 | SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); |
3612 | SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); |
3613 | SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); |
3614 | SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, |
3615 | DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT)); |
3616 | } else if (Scale == NVTSize) { |
3617 | |
3618 | SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); |
3619 | SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); |
3620 | SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); |
3621 | SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0, |
3622 | DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg)); |
3623 | |
3624 | SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); |
3625 | SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); |
3626 | SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); |
3627 | SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT, |
3628 | DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos)); |
3629 | } else if (Scale < VTSize) { |
3630 | |
3631 | |
3632 | unsigned OverflowBits = VTSize - Scale + 1; |
3633 | SDValue HHHiMask = DAG.getConstant( |
3634 | APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); |
3635 | SDValue HHLoMask = DAG.getConstant( |
3636 | APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); |
3637 | SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); |
3638 | SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); |
3639 | } else |
3640 | llvm_unreachable("Illegal scale for signed fixed point mul."); |
3641 | |
3642 | |
3643 | APInt MaxHi = APInt::getSignedMaxValue(NVTSize); |
3644 | APInt MaxLo = APInt::getAllOnesValue(NVTSize); |
3645 | Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi); |
3646 | Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo); |
3647 | |
3648 | APInt MinHi = APInt::getSignedMinValue(NVTSize); |
3649 | Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi); |
3650 | Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); |
3651 | } |
3652 | |
3653 | void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo, |
3654 | SDValue &Hi) { |
3655 | SDLoc dl(N); |
3656 | |
3657 | SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0), |
3658 | N->getOperand(1), |
3659 | N->getConstantOperandVal(2), DAG); |
3660 | |
3661 | if (!Res) |
3662 | Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1), |
3663 | N->getConstantOperandVal(2), TLI, DAG); |
3664 | SplitInteger(Res, Lo, Hi); |
3665 | } |
3666 | |
3667 | void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, |
3668 | SDValue &Lo, SDValue &Hi) { |
3669 | assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && |
3670 | "Node has unexpected Opcode"); |
3671 | SDValue LHS = Node->getOperand(0); |
3672 | SDValue RHS = Node->getOperand(1); |
3673 | SDLoc dl(Node); |
3674 | |
3675 | SDValue Ovf; |
3676 | |
3677 | bool IsAdd = Node->getOpcode() == ISD::SADDO; |
3678 | unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY; |
3679 | |
3680 | bool HasCarryOp = TLI.isOperationLegalOrCustom( |
3681 | CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); |
3682 | |
3683 | if (HasCarryOp) { |
3684 | |
3685 | SDValue LHSL, LHSH, RHSL, RHSH; |
3686 | GetExpandedInteger(LHS, LHSL, LHSH); |
3687 | GetExpandedInteger(RHS, RHSL, RHSH); |
3688 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1)); |
3689 | |
3690 | Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL}); |
3691 | Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) }); |
3692 | |
3693 | Ovf = Hi.getValue(1); |
3694 | } else { |
3695 | |
3696 | |
3697 | SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? |
3698 | ISD::ADD : ISD::SUB, dl, LHS.getValueType(), |
3699 | LHS, RHS); |
3700 | SplitInteger(Sum, Lo, Hi); |
3701 | |
3702 | |
3703 | |
3704 | |
3705 | |
3706 | |
3707 | |
3708 | |
3709 | |
3710 | |
3711 | |
3712 | |
3713 | |
3714 | |
3715 | |
3716 | |
3717 | |
3718 | |
3719 | |
3720 | |
3721 | |
3722 | |
3723 | |
3724 | |
3725 | EVT VT = LHS.getValueType(); |
3726 | SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); |
3727 | if (IsAdd) |
3728 | SignsMatch = DAG.getNOT(dl, SignsMatch, VT); |
3729 | |
3730 | SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum); |
3731 | Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE); |
3732 | EVT OType = Node->getValueType(1); |
3733 | Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT); |
3734 | } |
3735 | |
3736 | |
3737 | ReplaceValueWith(SDValue(Node, 1), Ovf); |
3738 | } |
3739 | |
3740 | void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, |
3741 | SDValue &Lo, SDValue &Hi) { |
3742 | EVT VT = N->getValueType(0); |
3743 | SDLoc dl(N); |
3744 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
3745 | |
3746 | if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { |
3747 | SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
3748 | SplitInteger(Res.getValue(0), Lo, Hi); |
3749 | return; |
3750 | } |
3751 | |
3752 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3753 | if (VT == MVT::i16) |
3754 | LC = RTLIB::SDIV_I16; |
3755 | else if (VT == MVT::i32) |
3756 | LC = RTLIB::SDIV_I32; |
3757 | else if (VT == MVT::i64) |
3758 | LC = RTLIB::SDIV_I64; |
3759 | else if (VT == MVT::i128) |
3760 | LC = RTLIB::SDIV_I128; |
3761 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); |
3762 | |
3763 | TargetLowering::MakeLibCallOptions CallOptions; |
3764 | CallOptions.setSExt(true); |
3765 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
3766 | } |
3767 | |
3768 | void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, |
3769 | SDValue &Lo, SDValue &Hi) { |
3770 | EVT VT = N->getValueType(0); |
3771 | SDLoc dl(N); |
3772 | |
3773 | |
3774 | |
3775 | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) |
| |
| |
3776 | return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); |
3777 | |
3778 | |
3779 | |
3780 | if (ExpandShiftWithKnownAmountBit(N, Lo, Hi)) |
| 8 | | Calling 'DAGTypeLegalizer::ExpandShiftWithKnownAmountBit' | |
|
3781 | return; |
3782 | |
3783 | |
3784 | unsigned PartsOpc; |
3785 | if (N->getOpcode() == ISD::SHL) { |
3786 | PartsOpc = ISD::SHL_PARTS; |
3787 | } else if (N->getOpcode() == ISD::SRL) { |
3788 | PartsOpc = ISD::SRL_PARTS; |
3789 | } else { |
3790 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
3791 | PartsOpc = ISD::SRA_PARTS; |
3792 | } |
3793 | |
3794 | |
3795 | |
3796 | |
3797 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
3798 | TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); |
3799 | const bool LegalOrCustom = |
3800 | (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || |
3801 | Action == TargetLowering::Custom; |
3802 | |
3803 | if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) { |
3804 | |
3805 | SDValue LHSL, LHSH; |
3806 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
3807 | EVT VT = LHSL.getValueType(); |
3808 | |
3809 | |
3810 | |
3811 | |
3812 | SDValue ShiftOp = N->getOperand(1); |
3813 | EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
3814 | assert(ShiftTy.getScalarSizeInBits() >= |
3815 | Log2_32_Ceil(VT.getScalarSizeInBits()) && |
3816 | "ShiftAmountTy is too small to cover the range of this type!"); |
3817 | if (ShiftOp.getValueType() != ShiftTy) |
3818 | ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); |
3819 | |
3820 | SDValue Ops[] = { LHSL, LHSH, ShiftOp }; |
3821 | Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); |
3822 | Hi = Lo.getValue(1); |
3823 | return; |
3824 | } |
3825 | |
3826 | |
3827 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3828 | bool isSigned; |
3829 | if (N->getOpcode() == ISD::SHL) { |
3830 | isSigned = false; |
3831 | if (VT == MVT::i16) |
3832 | LC = RTLIB::SHL_I16; |
3833 | else if (VT == MVT::i32) |
3834 | LC = RTLIB::SHL_I32; |
3835 | else if (VT == MVT::i64) |
3836 | LC = RTLIB::SHL_I64; |
3837 | else if (VT == MVT::i128) |
3838 | LC = RTLIB::SHL_I128; |
3839 | } else if (N->getOpcode() == ISD::SRL) { |
3840 | isSigned = false; |
3841 | if (VT == MVT::i16) |
3842 | LC = RTLIB::SRL_I16; |
3843 | else if (VT == MVT::i32) |
3844 | LC = RTLIB::SRL_I32; |
3845 | else if (VT == MVT::i64) |
3846 | LC = RTLIB::SRL_I64; |
3847 | else if (VT == MVT::i128) |
3848 | LC = RTLIB::SRL_I128; |
3849 | } else { |
3850 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
3851 | isSigned = true; |
3852 | if (VT == MVT::i16) |
3853 | LC = RTLIB::SRA_I16; |
3854 | else if (VT == MVT::i32) |
3855 | LC = RTLIB::SRA_I32; |
3856 | else if (VT == MVT::i64) |
3857 | LC = RTLIB::SRA_I64; |
3858 | else if (VT == MVT::i128) |
3859 | LC = RTLIB::SRA_I128; |
3860 | } |
3861 | |
3862 | if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) { |
3863 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
3864 | TargetLowering::MakeLibCallOptions CallOptions; |
3865 | CallOptions.setSExt(isSigned); |
3866 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
3867 | return; |
3868 | } |
3869 | |
3870 | if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)) |
3871 | llvm_unreachable("Unsupported shift!"); |
3872 | } |
3873 | |
3874 | void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, |
3875 | SDValue &Lo, SDValue &Hi) { |
3876 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
3877 | SDLoc dl(N); |
3878 | SDValue Op = N->getOperand(0); |
3879 | if (Op.getValueType().bitsLE(NVT)) { |
3880 | |
3881 | Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); |
3882 | |
3883 | unsigned LoSize = NVT.getSizeInBits(); |
3884 | Hi = DAG.getNode( |
3885 | ISD::SRA, dl, NVT, Lo, |
3886 | DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
3887 | } else { |
3888 | |
3889 | |
3890 | assert(getTypeAction(Op.getValueType()) == |
3891 | TargetLowering::TypePromoteInteger && |
3892 | "Only know how to promote this result!"); |
3893 | SDValue Res = GetPromotedInteger(Op); |
3894 | assert(Res.getValueType() == N->getValueType(0) && |
3895 | "Operand over promoted?"); |
3896 | |
3897 | SplitInteger(Res, Lo, Hi); |
3898 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
3899 | Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
3900 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
3901 | ExcessBits))); |
3902 | } |
3903 | } |
3904 | |
3905 | void DAGTypeLegalizer:: |
3906 | ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { |
3907 | SDLoc dl(N); |
3908 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
3909 | EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); |
3910 | |
3911 | if (EVT.bitsLE(Lo.getValueType())) { |
3912 | |
3913 | Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, |
3914 | N->getOperand(1)); |
3915 | |
3916 | |
3917 | |
3918 | Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, |
3919 | DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, |
3920 | TLI.getPointerTy(DAG.getDataLayout()))); |
3921 | } else { |
3922 | |
3923 | |
3924 | unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); |
3925 | Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
3926 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
3927 | ExcessBits))); |
3928 | } |
3929 | } |
3930 | |
3931 | void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, |
3932 | SDValue &Lo, SDValue &Hi) { |
3933 | EVT VT = N->getValueType(0); |
3934 | SDLoc dl(N); |
3935 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
3936 | |
3937 | if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) { |
3938 | SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
3939 | SplitInteger(Res.getValue(1), Lo, Hi); |
3940 | return; |
3941 | } |
3942 | |
3943 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3944 | if (VT == MVT::i16) |
3945 | LC = RTLIB::SREM_I16; |
3946 | else if (VT == MVT::i32) |
3947 | LC = RTLIB::SREM_I32; |
3948 | else if (VT == MVT::i64) |
3949 | LC = RTLIB::SREM_I64; |
3950 | else if (VT == MVT::i128) |
3951 | LC = RTLIB::SREM_I128; |
3952 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); |
3953 | |
3954 | TargetLowering::MakeLibCallOptions CallOptions; |
3955 | CallOptions.setSExt(true); |
3956 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
3957 | } |
3958 | |
3959 | void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, |
3960 | SDValue &Lo, SDValue &Hi) { |
3961 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
3962 | SDLoc dl(N); |
3963 | Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); |
3964 | Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), |
3965 | N->getOperand(0), |
3966 | DAG.getConstant(NVT.getSizeInBits(), dl, |
3967 | TLI.getPointerTy(DAG.getDataLayout()))); |
3968 | Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); |
3969 | } |
3970 | |
3971 | void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, |
3972 | SDValue &Lo, SDValue &Hi) { |
3973 | EVT VT = N->getValueType(0); |
3974 | SDLoc dl(N); |
3975 | |
3976 | if (N->getOpcode() == ISD::UMULO) { |
3977 | |
3978 | |
3979 | |
3980 | |
3981 | |
3982 | |
3983 | |
3984 | |
3985 | |
3986 | |
3987 | |
3988 | |
3989 | |
3990 | |
3991 | SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
3992 | SDValue LHSHigh, LHSLow, RHSHigh, RHSLow; |
3993 | GetExpandedInteger(LHS, LHSLow, LHSHigh); |
3994 | GetExpandedInteger(RHS, RHSLow, RHSHigh); |
3995 | EVT HalfVT = LHSLow.getValueType(); |
3996 | EVT BitVT = N->getValueType(1); |
3997 | SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT); |
3998 | |
3999 | SDValue HalfZero = DAG.getConstant(0, dl, HalfVT); |
4000 | SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT, |
4001 | DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE), |
4002 | DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE)); |
4003 | |
4004 | SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow); |
4005 | Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1)); |
4006 | |
4007 | SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow); |
4008 | Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1)); |
4009 | |
4010 | SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two); |
4011 | |
4012 | |
4013 | |
4014 | |
4015 | |
4016 | |
4017 | |
4018 | SDValue Three = DAG.getNode(ISD::MUL, dl, VT, |
4019 | DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow), |
4020 | DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow)); |
4021 | SplitInteger(Three, Lo, Hi); |
4022 | |
4023 | Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum); |
4024 | Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1)); |
4025 | ReplaceValueWith(SDValue(N, 1), Overflow); |
4026 | return; |
4027 | } |
4028 | |
4029 | Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); |
4030 | EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); |
4031 | Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); |
4032 | |
4033 | |
4034 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4035 | if (VT == MVT::i32) |
4036 | LC = RTLIB::MULO_I32; |
4037 | else if (VT == MVT::i64) |
4038 | LC = RTLIB::MULO_I64; |
4039 | else if (VT == MVT::i128) |
4040 | LC = RTLIB::MULO_I128; |
4041 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); |
4042 | |
4043 | SDValue Temp = DAG.CreateStackTemporary(PtrVT); |
4044 | |
4045 | SDValue Chain = |
4046 | DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, |
4047 | MachinePointerInfo()); |
4048 | |
4049 | TargetLowering::ArgListTy Args; |
4050 | TargetLowering::ArgListEntry Entry; |
4051 | for (const SDValue &Op : N->op_values()) { |
4052 | EVT ArgVT = Op.getValueType(); |
4053 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
4054 | Entry.Node = Op; |
4055 | Entry.Ty = ArgTy; |
4056 | Entry.IsSExt = true; |
4057 | Entry.IsZExt = false; |
4058 | Args.push_back(Entry); |
4059 | } |
4060 | |
4061 | |
4062 | Entry.Node = Temp; |
4063 | Entry.Ty = PtrTy->getPointerTo(); |
4064 | Entry.IsSExt = true; |
4065 | Entry.IsZExt = false; |
4066 | Args.push_back(Entry); |
4067 | |
4068 | SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); |
4069 | |
4070 | TargetLowering::CallLoweringInfo CLI(DAG); |
4071 | CLI.setDebugLoc(dl) |
4072 | .setChain(Chain) |
4073 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) |
4074 | .setSExtResult(); |
4075 | |
4076 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
4077 | |
4078 | SplitInteger(CallInfo.first, Lo, Hi); |
4079 | SDValue Temp2 = |
4080 | DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); |
4081 | SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, |
4082 | DAG.getConstant(0, dl, PtrVT), |
4083 | ISD::SETNE); |
4084 | |
4085 | ReplaceValueWith(SDValue(N, 1), Ofl); |
4086 | } |
4087 | |
4088 | void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, |
4089 | SDValue &Lo, SDValue &Hi) { |
4090 | EVT VT = N->getValueType(0); |
4091 | SDLoc dl(N); |
4092 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
4093 | |
4094 | if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { |
4095 | SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
4096 | SplitInteger(Res.getValue(0), Lo, Hi); |
4097 | return; |
4098 | } |
4099 | |
4100 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4101 | if (VT == MVT::i16) |
4102 | LC = RTLIB::UDIV_I16; |
4103 | else if (VT == MVT::i32) |
4104 | LC = RTLIB::UDIV_I32; |
4105 | else if (VT == MVT::i64) |
4106 | LC = RTLIB::UDIV_I64; |
4107 | else if (VT == MVT::i128) |
4108 | LC = RTLIB::UDIV_I128; |
4109 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); |
4110 | |
4111 | TargetLowering::MakeLibCallOptions CallOptions; |
4112 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
4113 | } |
4114 | |
4115 | void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, |
4116 | SDValue &Lo, SDValue &Hi) { |
4117 | EVT VT = N->getValueType(0); |
4118 | SDLoc dl(N); |
4119 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
4120 | |
4121 | if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) { |
4122 | SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
4123 | SplitInteger(Res.getValue(1), Lo, Hi); |
4124 | return; |
4125 | } |
4126 | |
4127 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
4128 | if (VT == MVT::i16) |
4129 | LC = RTLIB::UREM_I16; |
4130 | else if (VT == MVT::i32) |
4131 | LC = RTLIB::UREM_I32; |
4132 | else if (VT == MVT::i64) |
4133 | LC = RTLIB::UREM_I64; |
4134 | else if (VT == MVT::i128) |
4135 | LC = RTLIB::UREM_I128; |
4136 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); |
4137 | |
4138 | TargetLowering::MakeLibCallOptions CallOptions; |
4139 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi); |
4140 | } |
4141 | |
4142 | void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, |
4143 | SDValue &Lo, SDValue &Hi) { |
4144 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
4145 | SDLoc dl(N); |
4146 | SDValue Op = N->getOperand(0); |
4147 | if (Op.getValueType().bitsLE(NVT)) { |
4148 | |
4149 | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); |
4150 | Hi = DAG.getConstant(0, dl, NVT); |
4151 | } else { |
4152 | |
4153 | |
4154 | assert(getTypeAction(Op.getValueType()) == |
4155 | TargetLowering::TypePromoteInteger && |
4156 | "Only know how to promote this result!"); |
4157 | SDValue Res = GetPromotedInteger(Op); |
4158 | assert(Res.getValueType() == N->getValueType(0) && |
4159 | "Operand over promoted?"); |
4160 | |
4161 | SplitInteger(Res, Lo, Hi); |
4162 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
4163 | Hi = DAG.getZeroExtendInReg(Hi, dl, |
4164 | EVT::getIntegerVT(*DAG.getContext(), |
4165 | ExcessBits)); |
4166 | } |
4167 | } |
4168 | |
4169 | void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, |
4170 | SDValue &Lo, SDValue &Hi) { |
4171 | SDLoc dl(N); |
4172 | EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); |
4173 | SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
4174 | SDValue Zero = DAG.getConstant(0, dl, VT); |
4175 | SDValue Swap = DAG.getAtomicCmpSwap( |
4176 | ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
4177 | cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), |
4178 | N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); |
4179 | |
4180 | ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); |
4181 | ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); |
4182 | } |
4183 | |
4184 | void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, |
4185 | SDValue &Lo, SDValue &Hi) { |
4186 | |
4187 | |
4188 | SDValue Res = TLI.expandVecReduce(N, DAG); |
4189 | SplitInteger(Res, Lo, Hi); |
4190 | } |
4191 | |
4192 | void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N, |
4193 | SDValue &Lo, SDValue &Hi) { |
4194 | |
4195 | SDValue Res; |
4196 | TLI.expandROT(N, true , Res, DAG); |
4197 | SplitInteger(Res, Lo, Hi); |
4198 | } |
4199 | |
4200 | void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, |
4201 | SDValue &Lo, SDValue &Hi) { |
4202 | |
4203 | SDValue Res; |
4204 | TLI.expandFunnelShift(N, Res, DAG); |
4205 | SplitInteger(Res, Lo, Hi); |
4206 | } |
4207 | |
4208 | void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo, |
4209 | SDValue &Hi) { |
4210 | EVT VT = N->getValueType(0); |
4211 | EVT HalfVT = |
4212 | EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2); |
4213 | SDLoc dl(N); |
4214 | |
4215 | |
4216 | APInt One(HalfVT.getSizeInBits(), 1); |
4217 | SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One); |
4218 | VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase); |
4219 | SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0)); |
4220 | SplitInteger(Res, Lo, Hi); |
4221 | } |
4222 | |
4223 | |
4224 | |
4225 | |
4226 | |
4227 | |
4228 | |
4229 | |
4230 | |
4231 | bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { |
4232 | LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); |
4233 | dbgs() << "\n"); |
4234 | SDValue Res = SDValue(); |
4235 | |
4236 | if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) |
4237 | return false; |
4238 | |
4239 | switch (N->getOpcode()) { |
4240 | default: |
4241 | #ifndef NDEBUG |
4242 | dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; |
4243 | N->dump(&DAG); dbgs() << "\n"; |
4244 | #endif |
4245 | report_fatal_error("Do not know how to expand this operator's operand!"); |
4246 | |
4247 | case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; |
4248 | case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; |
4249 | case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; |
4250 | case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; |
4251 | case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; |
4252 | case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; |
4253 | case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break; |
4254 | case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; |
4255 | case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; |
4256 | case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; |
4257 | case ISD::STRICT_SINT_TO_FP: |
4258 | case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; |
4259 | case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; |
4260 | case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; |
4261 | case ISD::STRICT_UINT_TO_FP: |
4262 | case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; |
4263 | |
4264 | case ISD::SHL: |
4265 | case ISD::SRA: |
4266 | case ISD::SRL: |
4267 | case ISD::ROTL: |
4268 | case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; |
4269 | case ISD::RETURNADDR: |
4270 | case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; |
4271 | |
4272 | case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; |
4273 | } |
4274 | |
4275 | |
4276 | if (!Res.getNode()) return false; |
4277 | |
4278 | |
4279 | |
4280 | if (Res.getNode() == N) |
4281 | return true; |
4282 | |
4283 | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
4284 | "Invalid operand expansion"); |
4285 | |
4286 | ReplaceValueWith(SDValue(N, 0), Res); |
4287 | return false; |
4288 | } |
4289 | |
4290 | |
4291 | |
4292 | void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, |
4293 | SDValue &NewRHS, |
4294 | ISD::CondCode &CCCode, |
4295 | const SDLoc &dl) { |
4296 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
4297 | GetExpandedInteger(NewLHS, LHSLo, LHSHi); |
4298 | GetExpandedInteger(NewRHS, RHSLo, RHSHi); |
4299 | |
4300 | if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) { |
4301 | if (RHSLo == RHSHi) { |
4302 | if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) { |
4303 | if (RHSCST->isAllOnesValue()) { |
4304 | |
4305 | NewLHS = DAG.getNode(ISD::AND, dl, |
4306 | LHSLo.getValueType(), LHSLo, LHSHi); |
4307 | NewRHS = RHSLo; |
4308 | return; |
4309 | } |
4310 | } |
4311 | } |
4312 | |
4313 | NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); |
4314 | NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); |
4315 | NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); |
4316 | NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); |
4317 | return; |
4318 | } |
4319 | |
4320 | |
4321 | |
4322 | if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS)) |
4323 | if ((CCCode == ISD::SETLT && CST->isNullValue()) || |
4324 | (CCCode == ISD::SETGT && CST->isAllOnesValue())) { |
4325 | NewLHS = LHSHi; |
4326 | NewRHS = RHSHi; |
4327 | return; |
4328 | } |
4329 | |
4330 | |
4331 | ISD::CondCode LowCC; |
4332 | switch (CCCode) { |
4333 | default: llvm_unreachable("Unknown integer setcc!"); |
4334 | case ISD::SETLT: |
4335 | case ISD::SETULT: LowCC = ISD::SETULT; break; |
4336 | case ISD::SETGT: |
4337 | case ISD::SETUGT: LowCC = ISD::SETUGT; break; |
4338 | case ISD::SETLE: |
4339 | case ISD::SETULE: LowCC = ISD::SETULE; break; |
4340 | case ISD::SETGE: |
4341 | case ISD::SETUGE: LowCC = ISD::SETUGE; break; |
4342 | } |
4343 | |
4344 | |
4345 | |
4346 | |
4347 | |
4348 | |
4349 | |
4350 | TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, |
4351 | nullptr); |
4352 | SDValue LoCmp, HiCmp; |
4353 | if (TLI.isTypeLegal(LHSLo.getValueType()) && |
4354 | TLI.isTypeLegal(RHSLo.getValueType())) |
4355 | LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, |
4356 | RHSLo, LowCC, false, DagCombineInfo, dl); |
4357 | if (!LoCmp.getNode()) |
4358 | LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, |
4359 | RHSLo, LowCC); |
4360 | if (TLI.isTypeLegal(LHSHi.getValueType()) && |
4361 | TLI.isTypeLegal(RHSHi.getValueType())) |
4362 | HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, |
4363 | RHSHi, CCCode, false, DagCombineInfo, dl); |
4364 | if (!HiCmp.getNode()) |
4365 | HiCmp = |
4366 | DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), |
4367 | LHSHi, RHSHi, DAG.getCondCode(CCCode)); |
4368 | |
4369 | ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); |
4370 | ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); |
4371 | |
4372 | bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || |
4373 | CCCode == ISD::SETUGE || CCCode == ISD::SETULE); |
4374 | |
4375 | if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) || |
4376 | (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) || |
4377 | (LoCmpC && LoCmpC->isNullValue())))) { |
4378 | |
4379 | |
4380 | |
4381 | NewLHS = HiCmp; |
4382 | NewRHS = SDValue(); |
4383 | return; |
4384 | } |
4385 | |
4386 | if (LHSHi == RHSHi) { |
4387 | |
4388 | NewLHS = LoCmp; |
4389 | NewRHS = SDValue(); |
4390 | return; |
4391 | } |
4392 | |
4393 | |
4394 | EVT HiVT = LHSHi.getValueType(); |
4395 | EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); |
4396 | bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); |
4397 | |
4398 | |
4399 | if (HasSETCCCARRY) { |
4400 | |
4401 | |
4402 | bool FlipOperands = false; |
4403 | switch (CCCode) { |
4404 | case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; |
4405 | case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; |
4406 | case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; |
4407 | case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; |
4408 | default: break; |
4409 | } |
4410 | if (FlipOperands) { |
4411 | std::swap(LHSLo, RHSLo); |
4412 | std::swap(LHSHi, RHSHi); |
4413 | } |
4414 | |
4415 | |
4416 | |
4417 | |
4418 | EVT LoVT = LHSLo.getValueType(); |
4419 | SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT)); |
4420 | SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo); |
4421 | SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT), |
4422 | LHSHi, RHSHi, LowCmp.getValue(1), |
4423 | DAG.getCondCode(CCCode)); |
4424 | NewLHS = Res; |
4425 | NewRHS = SDValue(); |
4426 | return; |
4427 | } |
4428 | |
4429 | NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, |
4430 | false, DagCombineInfo, dl); |
4431 | if (!NewLHS.getNode()) |
4432 | NewLHS = |
4433 | DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); |
4434 | NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); |
4435 | NewRHS = SDValue(); |
4436 | } |
4437 | |
4438 | SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { |
4439 | SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); |
4440 | ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); |
4441 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
4442 | |
4443 | |
4444 | |
4445 | if (!NewRHS.getNode()) { |
4446 | NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
4447 | CCCode = ISD::SETNE; |
4448 | } |
4449 | |
4450 | |
4451 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
4452 | DAG.getCondCode(CCCode), NewLHS, NewRHS, |
4453 | N->getOperand(4)), 0); |
4454 | } |
4455 | |
4456 | SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { |
4457 | SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
4458 | ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); |
4459 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
4460 | |
4461 | |
4462 | |
4463 | if (!NewRHS.getNode()) { |
4464 | NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
4465 | CCCode = ISD::SETNE; |
4466 | } |
4467 | |
4468 | |
4469 | return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, |
4470 | N->getOperand(2), N->getOperand(3), |
4471 | DAG.getCondCode(CCCode)), 0); |
4472 | } |
4473 | |
4474 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { |
4475 | SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
4476 | ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); |
4477 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
4478 | |
4479 | |
4480 | if (!NewRHS.getNode()) { |
4481 | assert(NewLHS.getValueType() == N->getValueType(0) && |
4482 | "Unexpected setcc expansion!"); |
4483 | return NewLHS; |
4484 | } |
4485 | |
4486 | |
4487 | return SDValue( |
4488 | DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); |
4489 | } |
4490 | |
4491 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { |
4492 | SDValue LHS = N->getOperand(0); |
4493 | SDValue RHS = N->getOperand(1); |
4494 | SDValue Carry = N->getOperand(2); |
4495 | SDValue Cond = N->getOperand(3); |
4496 | SDLoc dl = SDLoc(N); |
4497 | |
4498 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
4499 | GetExpandedInteger(LHS, LHSLo, LHSHi); |
4500 | GetExpandedInteger(RHS, RHSLo, RHSHi); |
4501 | |
4502 | |
4503 | SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); |
4504 | SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); |
4505 | return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, |
4506 | LowCmp.getValue(1), Cond); |
4507 | } |
4508 | |
4509 | SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) { |
4510 | |
4511 | SDValue Lo, Hi; |
4512 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
4513 | return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo, |
4514 | Hi); |
4515 | } |
4516 | |
4517 | SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { |
4518 | |
4519 | |
4520 | |
4521 | SDValue Lo, Hi; |
4522 | GetExpandedInteger(N->getOperand(1), Lo, Hi); |
4523 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); |
4524 | } |
4525 | |
4526 | SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { |
4527 | |
4528 | |
4529 | |
4530 | SDValue Lo, Hi; |
4531 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
4532 | return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); |
4533 | } |
4534 | |
4535 | SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { |
4536 | bool IsStrict = N->isStrictFPOpcode(); |
4537 | SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
4538 | SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
4539 | EVT DstVT = N->getValueType(0); |
4540 | RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); |
4541 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
4542 | "Don't know how to expand this SINT_TO_FP!"); |
4543 | TargetLowering::MakeLibCallOptions CallOptions; |
4544 | CallOptions.setSExt(true); |
4545 | std::pair<SDValue, SDValue> Tmp = |
4546 | TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); |
4547 | |
4548 | if (!IsStrict) |
4549 | return Tmp.first; |
4550 | |
4551 | ReplaceValueWith(SDValue(N, 1), Tmp.second); |
4552 | ReplaceValueWith(SDValue(N, 0), Tmp.first); |
4553 | return SDValue(); |
4554 | } |
4555 | |
4556 | SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { |
4557 | if (N->isAtomic()) { |
4558 | |
4559 | SDLoc dl(N); |
4560 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
4561 | N->getMemoryVT(), |
4562 | N->getOperand(0), N->getOperand(2), |
4563 | N->getOperand(1), |
4564 | N->getMemOperand()); |
4565 | return Swap.getValue(1); |
4566 | } |
4567 | if (ISD::isNormalStore(N)) |
4568 | return ExpandOp_NormalStore(N, OpNo); |
4569 | |
4570 | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
4571 | assert(OpNo == 1 && "Can only expand the stored value so far"); |
4572 | |
4573 | EVT VT = N->getOperand(1).getValueType(); |
4574 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
4575 | SDValue Ch = N->getChain(); |
4576 | SDValue Ptr = N->getBasePtr(); |
4577 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
4578 | AAMDNodes AAInfo = N->getAAInfo(); |
4579 | SDLoc dl(N); |
4580 | SDValue Lo, Hi; |
4581 | |
4582 | assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
4583 | |
4584 | if (N->getMemoryVT().bitsLE(NVT)) { |
4585 | GetExpandedInteger(N->getValue(), Lo, Hi); |
4586 | return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), |
4587 | N->getMemoryVT(), N->getOriginalAlign(), MMOFlags, |
4588 | AAInfo); |
4589 | } |
4590 | |
4591 | if (DAG.getDataLayout().isLittleEndian()) { |
4592 | |
4593 | GetExpandedInteger(N->getValue(), Lo, Hi); |
4594 | |
4595 | Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), |
4596 | N->getOriginalAlign(), MMOFlags, AAInfo); |
4597 | |
4598 | unsigned ExcessBits = |
4599 | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
4600 | EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
4601 | |
4602 | |
4603 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
4604 | Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); |
4605 | Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, |
4606 | N->getPointerInfo().getWithOffset(IncrementSize), |
4607 | NEVT, N->getOriginalAlign(), MMOFlags, AAInfo); |
4608 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
4609 | } |
4610 | |
4611 | |
4612 | |
4613 | GetExpandedInteger(N->getValue(), Lo, Hi); |
4614 | |
4615 | EVT ExtVT = N->getMemoryVT(); |
4616 | unsigned EBytes = ExtVT.getStoreSize(); |
4617 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
4618 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
4619 | EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), |
4620 | ExtVT.getSizeInBits() - ExcessBits); |
4621 | |
4622 | if (ExcessBits < NVT.getSizeInBits()) { |
4623 | |
4624 | Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, |
4625 | DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
4626 | TLI.getPointerTy(DAG.getDataLayout()))); |
4627 | Hi = DAG.getNode( |
4628 | ISD::OR, dl, NVT, Hi, |
4629 | DAG.getNode(ISD::SRL, dl, NVT, Lo, |
4630 | DAG.getConstant(ExcessBits, dl, |
4631 | TLI.getPointerTy(DAG.getDataLayout())))); |
4632 | } |
4633 | |
4634 | |
4635 | Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, |
4636 | N->getOriginalAlign(), MMOFlags, AAInfo); |
4637 | |
4638 | |
4639 | Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize)); |
4640 | |
4641 | Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, |
4642 | N->getPointerInfo().getWithOffset(IncrementSize), |
4643 | EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
4644 | N->getOriginalAlign(), MMOFlags, AAInfo); |
4645 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
4646 | } |
4647 | |
4648 | SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { |
4649 | SDValue InL, InH; |
4650 | GetExpandedInteger(N->getOperand(0), InL, InH); |
4651 | |
4652 | return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); |
4653 | } |
4654 | |
4655 | SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { |
4656 | bool IsStrict = N->isStrictFPOpcode(); |
4657 | SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); |
4658 | SDValue Op = N->getOperand(IsStrict ? 1 : 0); |
4659 | EVT DstVT = N->getValueType(0); |
4660 | RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT); |
4661 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
4662 | "Don't know how to expand this UINT_TO_FP!"); |
4663 | TargetLowering::MakeLibCallOptions CallOptions; |
4664 | CallOptions.setSExt(true); |
4665 | std::pair<SDValue, SDValue> Tmp = |
4666 | TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain); |
4667 | |
4668 | if (!IsStrict) |
4669 | return Tmp.first; |
4670 | |
4671 | ReplaceValueWith(SDValue(N, 1), Tmp.second); |
4672 | ReplaceValueWith(SDValue(N, 0), Tmp.first); |
4673 | return SDValue(); |
4674 | } |
4675 | |
4676 | SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { |
4677 | SDLoc dl(N); |
4678 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
4679 | cast<AtomicSDNode>(N)->getMemoryVT(), |
4680 | N->getOperand(0), |
4681 | N->getOperand(1), N->getOperand(2), |
4682 | cast<AtomicSDNode>(N)->getMemOperand()); |
4683 | return Swap.getValue(1); |
4684 | } |
4685 | |
4686 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) { |
4687 | SDLoc dl(N); |
4688 | |
4689 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
4690 | SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
4691 | EVT OutVT = V0.getValueType(); |
4692 | |
4693 | return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2)); |
4694 | } |
4695 | |
4696 | SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { |
4697 | |
4698 | EVT OutVT = N->getValueType(0); |
4699 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4700 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
4701 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
4702 | |
4703 | SDLoc dl(N); |
4704 | SDValue BaseIdx = N->getOperand(1); |
4705 | |
4706 | |
4707 | |
4708 | if (OutVT.isScalableVector()) { |
4709 | SDValue InOp0 = N->getOperand(0); |
4710 | EVT InVT = InOp0.getValueType(); |
4711 | |
4712 | |
4713 | |
4714 | if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { |
4715 | |
4716 | SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx }; |
4717 | |
4718 | EVT PromEltVT = Ops[0].getValueType().getVectorElementType(); |
4719 | assert(PromEltVT.bitsLE(NOutVTElem) && |
4720 | "Promoted operand has an element type greater than result"); |
4721 | |
4722 | EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT); |
4723 | SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops); |
4724 | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext); |
4725 | } |
4726 | } |
4727 | |
4728 | if (OutVT.isScalableVector()) |
4729 | report_fatal_error("Unable to promote scalable types using BUILD_VECTOR"); |
4730 | |
4731 | SDValue InOp0 = N->getOperand(0); |
4732 | if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) |
4733 | InOp0 = GetPromotedInteger(N->getOperand(0)); |
4734 | |
4735 | EVT InVT = InOp0.getValueType(); |
4736 | |
4737 | unsigned OutNumElems = OutVT.getVectorNumElements(); |
4738 | SmallVector<SDValue, 8> Ops; |
4739 | Ops.reserve(OutNumElems); |
4740 | for (unsigned i = 0; i != OutNumElems; ++i) { |
4741 | |
4742 | |
4743 | SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), |
4744 | BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); |
4745 | SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
4746 | InVT.getVectorElementType(), N->getOperand(0), Index); |
4747 | |
4748 | SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); |
4749 | |
4750 | Ops.push_back(Op); |
4751 | } |
4752 | |
4753 | return DAG.getBuildVector(NOutVT, dl, Ops); |
4754 | } |
4755 | |
4756 | SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) { |
4757 | EVT OutVT = N->getValueType(0); |
4758 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4759 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
4760 | |
4761 | SDLoc dl(N); |
4762 | SDValue Vec = N->getOperand(0); |
4763 | SDValue SubVec = N->getOperand(1); |
4764 | SDValue Idx = N->getOperand(2); |
4765 | |
4766 | EVT SubVecVT = SubVec.getValueType(); |
4767 | EVT NSubVT = |
4768 | EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(), |
4769 | SubVecVT.getVectorElementCount()); |
4770 | |
4771 | Vec = GetPromotedInteger(Vec); |
4772 | SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec); |
4773 | |
4774 | return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx); |
4775 | } |
4776 | |
4777 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) { |
4778 | SDLoc dl(N); |
4779 | |
4780 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
4781 | EVT OutVT = V0.getValueType(); |
4782 | |
4783 | return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0); |
4784 | } |
4785 | |
4786 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { |
4787 | ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); |
4788 | EVT VT = N->getValueType(0); |
4789 | SDLoc dl(N); |
4790 | |
4791 | ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); |
4792 | |
4793 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
4794 | SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
4795 | EVT OutVT = V0.getValueType(); |
4796 | |
4797 | return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); |
4798 | } |
4799 | |
4800 | |
4801 | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { |
4802 | EVT OutVT = N->getValueType(0); |
4803 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4804 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
4805 | unsigned NumElems = N->getNumOperands(); |
4806 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
4807 | |
4808 | SDLoc dl(N); |
4809 | |
4810 | SmallVector<SDValue, 8> Ops; |
4811 | Ops.reserve(NumElems); |
4812 | for (unsigned i = 0; i != NumElems; ++i) { |
4813 | SDValue Op; |
4814 | |
4815 | |
4816 | |
4817 | |
4818 | if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) |
4819 | Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); |
4820 | else |
4821 | Op = N->getOperand(i); |
4822 | Ops.push_back(Op); |
4823 | } |
4824 | |
4825 | return DAG.getBuildVector(NOutVT, dl, Ops); |
4826 | } |
4827 | |
4828 | SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { |
4829 | |
4830 | SDLoc dl(N); |
4831 | |
4832 | assert(!N->getOperand(0).getValueType().isVector() && |
4833 | "Input must be a scalar"); |
4834 | |
4835 | EVT OutVT = N->getValueType(0); |
4836 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4837 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
4838 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
4839 | |
4840 | SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); |
4841 | |
4842 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); |
4843 | } |
4844 | |
4845 | SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) { |
4846 | SDLoc dl(N); |
4847 | |
4848 | SDValue SplatVal = N->getOperand(0); |
4849 | |
4850 | assert(!SplatVal.getValueType().isVector() && "Input must be a scalar"); |
4851 | |
4852 | EVT OutVT = N->getValueType(0); |
4853 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4854 | assert(NOutVT.isVector() && "Type must be promoted to a vector type"); |
4855 | EVT NOutElemVT = NOutVT.getVectorElementType(); |
4856 | |
4857 | SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal); |
4858 | |
4859 | return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op); |
4860 | } |
4861 | |
4862 | SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) { |
4863 | SDLoc dl(N); |
4864 | EVT OutVT = N->getValueType(0); |
4865 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4866 | assert(NOutVT.isVector() && "Type must be promoted to a vector type"); |
4867 | APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue(); |
4868 | return DAG.getStepVector(dl, NOutVT, |
4869 | StepVal.sext(NOutVT.getScalarSizeInBits())); |
4870 | } |
4871 | |
4872 | SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { |
4873 | SDLoc dl(N); |
4874 | |
4875 | EVT OutVT = N->getValueType(0); |
4876 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4877 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
4878 | |
4879 | EVT OutElemTy = NOutVT.getVectorElementType(); |
4880 | |
4881 | unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); |
4882 | unsigned NumOutElem = NOutVT.getVectorNumElements(); |
4883 | unsigned NumOperands = N->getNumOperands(); |
4884 | assert(NumElem * NumOperands == NumOutElem && |
4885 | "Unexpected number of elements"); |
4886 | |
4887 | |
4888 | SmallVector<SDValue, 8> Ops(NumOutElem); |
4889 | for (unsigned i = 0; i < NumOperands; ++i) { |
4890 | SDValue Op = N->getOperand(i); |
4891 | if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger) |
4892 | Op = GetPromotedInteger(Op); |
4893 | EVT SclrTy = Op.getValueType().getVectorElementType(); |
4894 | assert(NumElem == Op.getValueType().getVectorNumElements() && |
4895 | "Unexpected number of elements"); |
4896 | |
4897 | for (unsigned j = 0; j < NumElem; ++j) { |
4898 | SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op, |
4899 | DAG.getVectorIdxConstant(j, dl)); |
4900 | Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy); |
4901 | } |
4902 | } |
4903 | |
4904 | return DAG.getBuildVector(NOutVT, dl, Ops); |
4905 | } |
4906 | |
4907 | SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { |
4908 | EVT VT = N->getValueType(0); |
4909 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
4910 | assert(NVT.isVector() && "This type must be promoted to a vector type"); |
4911 | |
4912 | SDLoc dl(N); |
4913 | |
4914 | |
4915 | |
4916 | |
4917 | |
4918 | if (getTypeAction(N->getOperand(0).getValueType()) |
4919 | == TargetLowering::TypePromoteInteger) { |
4920 | SDValue Promoted; |
4921 | |
4922 | switch(N->getOpcode()) { |
4923 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
4924 | Promoted = SExtPromotedInteger(N->getOperand(0)); |
4925 | break; |
4926 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
4927 | Promoted = ZExtPromotedInteger(N->getOperand(0)); |
4928 | break; |
4929 | case ISD::ANY_EXTEND_VECTOR_INREG: |
4930 | Promoted = GetPromotedInteger(N->getOperand(0)); |
4931 | break; |
4932 | default: |
4933 | llvm_unreachable("Node has unexpected Opcode"); |
4934 | } |
4935 | return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); |
4936 | } |
4937 | |
4938 | |
4939 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
4940 | } |
4941 | |
4942 | SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { |
4943 | EVT OutVT = N->getValueType(0); |
4944 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
4945 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
4946 | |
4947 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
4948 | |
4949 | SDLoc dl(N); |
4950 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
4951 | |
4952 | SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, |
4953 | NOutVTElem, N->getOperand(1)); |
4954 | return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, |
4955 | V0, ConvElem, N->getOperand(2)); |
4956 | } |
4957 | |
4958 | SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { |
4959 | |
4960 | |
4961 | SDLoc dl(N); |
4962 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
4963 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
4964 | } |
4965 | |
4966 | SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { |
4967 | SDLoc dl(N); |
4968 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
4969 | SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, |
4970 | TLI.getVectorIdxTy(DAG.getDataLayout())); |
4971 | SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
4972 | V0->getValueType(0).getScalarType(), V0, V1); |
4973 | |
4974 | |
4975 | |
4976 | |
4977 | return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); |
4978 | } |
4979 | |
4980 | SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { |
4981 | SDLoc dl(N); |
4982 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
4983 | MVT InVT = V0.getValueType().getSimpleVT(); |
4984 | MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), |
4985 | N->getValueType(0).getVectorNumElements()); |
4986 | SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); |
4987 | return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); |
4988 | } |
4989 | |
4990 | SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { |
4991 | SDLoc dl(N); |
4992 | |
4993 | EVT ResVT = N->getValueType(0); |
4994 | unsigned NumElems = N->getNumOperands(); |
4995 | |
4996 | if (ResVT.isScalableVector()) { |
4997 | SDValue ResVec = DAG.getUNDEF(ResVT); |
4998 | |
4999 | for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) { |
5000 | SDValue Op = N->getOperand(OpIdx); |
5001 | unsigned OpNumElts = Op.getValueType().getVectorMinNumElements(); |
5002 | ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op, |
5003 | DAG.getIntPtrConstant(OpIdx * OpNumElts, dl)); |
5004 | } |
5005 | |
5006 | return ResVec; |
5007 | } |
5008 | |
5009 | EVT RetSclrTy = N->getValueType(0).getVectorElementType(); |
5010 | |
5011 | SmallVector<SDValue, 8> NewOps; |
5012 | NewOps.reserve(NumElems); |
5013 | |
5014 | |
5015 | for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) { |
5016 | SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); |
5017 | EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); |
5018 | unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); |
5019 | |
5020 | for (unsigned i=0; i<NumElem; ++i) { |
5021 | |
5022 | SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, |
5023 | DAG.getVectorIdxConstant(i, dl)); |
5024 | SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); |
5025 | NewOps.push_back(Tr); |
5026 | } |
5027 | } |
5028 | |
5029 | return DAG.getBuildVector(N->getValueType(0), dl, NewOps); |
5030 | } |