| File: | src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/include/llvm/CodeGen/SelectionDAGNodes.h |
| Warning: | line 1110, column 10 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | /// \file |
| 10 | /// Custom DAG lowering for SI |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "SIISelLowering.h" |
| 15 | #include "AMDGPU.h" |
| 16 | #include "AMDGPUInstrInfo.h" |
| 17 | #include "AMDGPUTargetMachine.h" |
| 18 | #include "SIMachineFunctionInfo.h" |
| 19 | #include "SIRegisterInfo.h" |
| 20 | #include "llvm/ADT/Statistic.h" |
| 21 | #include "llvm/Analysis/LegacyDivergenceAnalysis.h" |
| 22 | #include "llvm/BinaryFormat/ELF.h" |
| 23 | #include "llvm/CodeGen/Analysis.h" |
| 24 | #include "llvm/CodeGen/FunctionLoweringInfo.h" |
| 25 | #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" |
| 26 | #include "llvm/CodeGen/MachineLoopInfo.h" |
| 27 | #include "llvm/IR/DiagnosticInfo.h" |
| 28 | #include "llvm/IR/IntrinsicInst.h" |
| 29 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
| 30 | #include "llvm/IR/IntrinsicsR600.h" |
| 31 | #include "llvm/Support/CommandLine.h" |
| 32 | #include "llvm/Support/KnownBits.h" |
| 33 | |
| 34 | using namespace llvm; |
| 35 | |
| 36 | #define DEBUG_TYPE"si-lower" "si-lower" |
| 37 | |
| 38 | STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"si-lower", "NumTailCalls" , "Number of tail calls"}; |
| 39 | |
| 40 | static cl::opt<bool> DisableLoopAlignment( |
| 41 | "amdgpu-disable-loop-alignment", |
| 42 | cl::desc("Do not align and prefetch loops"), |
| 43 | cl::init(false)); |
| 44 | |
| 45 | static cl::opt<bool> VGPRReserveforSGPRSpill( |
| 46 | "amdgpu-reserve-vgpr-for-sgpr-spill", |
| 47 | cl::desc("Allocates one VGPR for future SGPR Spill"), cl::init(true)); |
| 48 | |
| 49 | static cl::opt<bool> UseDivergentRegisterIndexing( |
| 50 | "amdgpu-use-divergent-register-indexing", |
| 51 | cl::Hidden, |
| 52 | cl::desc("Use indirect register addressing for divergent indexes"), |
| 53 | cl::init(false)); |
| 54 | |
| 55 | static bool hasFP32Denormals(const MachineFunction &MF) { |
| 56 | const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 57 | return Info->getMode().allFP32Denormals(); |
| 58 | } |
| 59 | |
| 60 | static bool hasFP64FP16Denormals(const MachineFunction &MF) { |
| 61 | const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 62 | return Info->getMode().allFP64FP16Denormals(); |
| 63 | } |
| 64 | |
| 65 | static unsigned findFirstFreeSGPR(CCState &CCInfo) { |
| 66 | unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); |
| 67 | for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) { |
| 68 | if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) { |
| 69 | return AMDGPU::SGPR0 + Reg; |
| 70 | } |
| 71 | } |
| 72 | llvm_unreachable("Cannot allocate sgpr")__builtin_unreachable(); |
| 73 | } |
| 74 | |
| 75 | SITargetLowering::SITargetLowering(const TargetMachine &TM, |
| 76 | const GCNSubtarget &STI) |
| 77 | : AMDGPUTargetLowering(TM, STI), |
| 78 | Subtarget(&STI) { |
| 79 | addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass); |
| 80 | addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); |
| 81 | |
| 82 | addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass); |
| 83 | addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass); |
| 84 | |
| 85 | addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass); |
| 86 | |
| 87 | const SIRegisterInfo *TRI = STI.getRegisterInfo(); |
| 88 | const TargetRegisterClass *V64RegClass = TRI->getVGPR64Class(); |
| 89 | |
| 90 | addRegisterClass(MVT::f64, V64RegClass); |
| 91 | addRegisterClass(MVT::v2f32, V64RegClass); |
| 92 | |
| 93 | addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass); |
| 94 | addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96)); |
| 95 | |
| 96 | addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass); |
| 97 | addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass); |
| 98 | |
| 99 | addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass); |
| 100 | addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128)); |
| 101 | |
| 102 | addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass); |
| 103 | addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160)); |
| 104 | |
| 105 | addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass); |
| 106 | addRegisterClass(MVT::v6f32, TRI->getVGPRClassForBitWidth(192)); |
| 107 | |
| 108 | addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass); |
| 109 | addRegisterClass(MVT::v3f64, TRI->getVGPRClassForBitWidth(192)); |
| 110 | |
| 111 | addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass); |
| 112 | addRegisterClass(MVT::v7f32, TRI->getVGPRClassForBitWidth(224)); |
| 113 | |
| 114 | addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass); |
| 115 | addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256)); |
| 116 | |
| 117 | addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass); |
| 118 | addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256)); |
| 119 | |
| 120 | addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass); |
| 121 | addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512)); |
| 122 | |
| 123 | addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass); |
| 124 | addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512)); |
| 125 | |
| 126 | addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass); |
| 127 | addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024)); |
| 128 | |
| 129 | if (Subtarget->has16BitInsts()) { |
| 130 | addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass); |
| 131 | addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass); |
| 132 | |
| 133 | // Unless there are also VOP3P operations, not operations are really legal. |
| 134 | addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass); |
| 135 | addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass); |
| 136 | addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass); |
| 137 | addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass); |
| 138 | } |
| 139 | |
| 140 | addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass); |
| 141 | addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024)); |
| 142 | |
| 143 | computeRegisterProperties(Subtarget->getRegisterInfo()); |
| 144 | |
| 145 | // The boolean content concept here is too inflexible. Compares only ever |
| 146 | // really produce a 1-bit result. Any copy/extend from these will turn into a |
| 147 | // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as |
| 148 | // it's what most targets use. |
| 149 | setBooleanContents(ZeroOrOneBooleanContent); |
| 150 | setBooleanVectorContents(ZeroOrOneBooleanContent); |
| 151 | |
| 152 | // We need to custom lower vector stores from local memory |
| 153 | setOperationAction(ISD::LOAD, MVT::v2i32, Custom); |
| 154 | setOperationAction(ISD::LOAD, MVT::v3i32, Custom); |
| 155 | setOperationAction(ISD::LOAD, MVT::v4i32, Custom); |
| 156 | setOperationAction(ISD::LOAD, MVT::v5i32, Custom); |
| 157 | setOperationAction(ISD::LOAD, MVT::v6i32, Custom); |
| 158 | setOperationAction(ISD::LOAD, MVT::v7i32, Custom); |
| 159 | setOperationAction(ISD::LOAD, MVT::v8i32, Custom); |
| 160 | setOperationAction(ISD::LOAD, MVT::v16i32, Custom); |
| 161 | setOperationAction(ISD::LOAD, MVT::i1, Custom); |
| 162 | setOperationAction(ISD::LOAD, MVT::v32i32, Custom); |
| 163 | |
| 164 | setOperationAction(ISD::STORE, MVT::v2i32, Custom); |
| 165 | setOperationAction(ISD::STORE, MVT::v3i32, Custom); |
| 166 | setOperationAction(ISD::STORE, MVT::v4i32, Custom); |
| 167 | setOperationAction(ISD::STORE, MVT::v5i32, Custom); |
| 168 | setOperationAction(ISD::STORE, MVT::v6i32, Custom); |
| 169 | setOperationAction(ISD::STORE, MVT::v7i32, Custom); |
| 170 | setOperationAction(ISD::STORE, MVT::v8i32, Custom); |
| 171 | setOperationAction(ISD::STORE, MVT::v16i32, Custom); |
| 172 | setOperationAction(ISD::STORE, MVT::i1, Custom); |
| 173 | setOperationAction(ISD::STORE, MVT::v32i32, Custom); |
| 174 | |
| 175 | setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); |
| 176 | setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand); |
| 177 | setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); |
| 178 | setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand); |
| 179 | setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand); |
| 180 | setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand); |
| 181 | setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand); |
| 182 | setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand); |
| 183 | setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand); |
| 184 | setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand); |
| 185 | setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand); |
| 186 | setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand); |
| 187 | setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand); |
| 188 | setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand); |
| 189 | setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand); |
| 190 | setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand); |
| 191 | |
| 192 | setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); |
| 193 | setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); |
| 194 | setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand); |
| 195 | setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand); |
| 196 | setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand); |
| 197 | setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand); |
| 198 | setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand); |
| 199 | |
| 200 | setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); |
| 201 | setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); |
| 202 | |
| 203 | setOperationAction(ISD::SELECT, MVT::i1, Promote); |
| 204 | setOperationAction(ISD::SELECT, MVT::i64, Custom); |
| 205 | setOperationAction(ISD::SELECT, MVT::f64, Promote); |
| 206 | AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64); |
| 207 | |
| 208 | setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); |
| 209 | setOperationAction(ISD::SELECT_CC, MVT::i32, Expand); |
| 210 | setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); |
| 211 | setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); |
| 212 | setOperationAction(ISD::SELECT_CC, MVT::i1, Expand); |
| 213 | |
| 214 | setOperationAction(ISD::SETCC, MVT::i1, Promote); |
| 215 | setOperationAction(ISD::SETCC, MVT::v2i1, Expand); |
| 216 | setOperationAction(ISD::SETCC, MVT::v4i1, Expand); |
| 217 | AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32); |
| 218 | |
| 219 | setOperationAction(ISD::TRUNCATE, MVT::v2i32, Expand); |
| 220 | setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); |
| 221 | setOperationAction(ISD::TRUNCATE, MVT::v3i32, Expand); |
| 222 | setOperationAction(ISD::FP_ROUND, MVT::v3f32, Expand); |
| 223 | setOperationAction(ISD::TRUNCATE, MVT::v4i32, Expand); |
| 224 | setOperationAction(ISD::FP_ROUND, MVT::v4f32, Expand); |
| 225 | setOperationAction(ISD::TRUNCATE, MVT::v5i32, Expand); |
| 226 | setOperationAction(ISD::FP_ROUND, MVT::v5f32, Expand); |
| 227 | setOperationAction(ISD::TRUNCATE, MVT::v6i32, Expand); |
| 228 | setOperationAction(ISD::FP_ROUND, MVT::v6f32, Expand); |
| 229 | setOperationAction(ISD::TRUNCATE, MVT::v7i32, Expand); |
| 230 | setOperationAction(ISD::FP_ROUND, MVT::v7f32, Expand); |
| 231 | setOperationAction(ISD::TRUNCATE, MVT::v8i32, Expand); |
| 232 | setOperationAction(ISD::FP_ROUND, MVT::v8f32, Expand); |
| 233 | setOperationAction(ISD::TRUNCATE, MVT::v16i32, Expand); |
| 234 | setOperationAction(ISD::FP_ROUND, MVT::v16f32, Expand); |
| 235 | |
| 236 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Custom); |
| 237 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Custom); |
| 238 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom); |
| 239 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Custom); |
| 240 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom); |
| 241 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v3i16, Custom); |
| 242 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Custom); |
| 243 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Custom); |
| 244 | |
| 245 | setOperationAction(ISD::BRCOND, MVT::Other, Custom); |
| 246 | setOperationAction(ISD::BR_CC, MVT::i1, Expand); |
| 247 | setOperationAction(ISD::BR_CC, MVT::i32, Expand); |
| 248 | setOperationAction(ISD::BR_CC, MVT::i64, Expand); |
| 249 | setOperationAction(ISD::BR_CC, MVT::f32, Expand); |
| 250 | setOperationAction(ISD::BR_CC, MVT::f64, Expand); |
| 251 | |
| 252 | setOperationAction(ISD::UADDO, MVT::i32, Legal); |
| 253 | setOperationAction(ISD::USUBO, MVT::i32, Legal); |
| 254 | |
| 255 | setOperationAction(ISD::ADDCARRY, MVT::i32, Legal); |
| 256 | setOperationAction(ISD::SUBCARRY, MVT::i32, Legal); |
| 257 | |
| 258 | setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); |
| 259 | setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); |
| 260 | setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); |
| 261 | |
| 262 | #if 0 |
| 263 | setOperationAction(ISD::ADDCARRY, MVT::i64, Legal); |
| 264 | setOperationAction(ISD::SUBCARRY, MVT::i64, Legal); |
| 265 | #endif |
| 266 | |
| 267 | // We only support LOAD/STORE and vector manipulation ops for vectors |
| 268 | // with > 4 elements. |
| 269 | for (MVT VT : { MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32, |
| 270 | MVT::v2i64, MVT::v2f64, MVT::v4i16, MVT::v4f16, |
| 271 | MVT::v3i64, MVT::v3f64, MVT::v6i32, MVT::v6f32, |
| 272 | MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, |
| 273 | MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32 }) { |
| 274 | for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) { |
| 275 | switch (Op) { |
| 276 | case ISD::LOAD: |
| 277 | case ISD::STORE: |
| 278 | case ISD::BUILD_VECTOR: |
| 279 | case ISD::BITCAST: |
| 280 | case ISD::EXTRACT_VECTOR_ELT: |
| 281 | case ISD::INSERT_VECTOR_ELT: |
| 282 | case ISD::EXTRACT_SUBVECTOR: |
| 283 | case ISD::SCALAR_TO_VECTOR: |
| 284 | break; |
| 285 | case ISD::INSERT_SUBVECTOR: |
| 286 | case ISD::CONCAT_VECTORS: |
| 287 | setOperationAction(Op, VT, Custom); |
| 288 | break; |
| 289 | default: |
| 290 | setOperationAction(Op, VT, Expand); |
| 291 | break; |
| 292 | } |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand); |
| 297 | |
| 298 | // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that |
| 299 | // is expanded to avoid having two separate loops in case the index is a VGPR. |
| 300 | |
| 301 | // Most operations are naturally 32-bit vector operations. We only support |
| 302 | // load and store of i64 vectors, so promote v2i64 vector operations to v4i32. |
| 303 | for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) { |
| 304 | setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); |
| 305 | AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32); |
| 306 | |
| 307 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); |
| 308 | AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32); |
| 309 | |
| 310 | setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); |
| 311 | AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32); |
| 312 | |
| 313 | setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); |
| 314 | AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32); |
| 315 | } |
| 316 | |
| 317 | for (MVT Vec64 : { MVT::v3i64, MVT::v3f64 }) { |
| 318 | setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); |
| 319 | AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32); |
| 320 | |
| 321 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); |
| 322 | AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32); |
| 323 | |
| 324 | setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); |
| 325 | AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32); |
| 326 | |
| 327 | setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); |
| 328 | AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32); |
| 329 | } |
| 330 | |
| 331 | for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) { |
| 332 | setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); |
| 333 | AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32); |
| 334 | |
| 335 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); |
| 336 | AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32); |
| 337 | |
| 338 | setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); |
| 339 | AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32); |
| 340 | |
| 341 | setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); |
| 342 | AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32); |
| 343 | } |
| 344 | |
| 345 | for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) { |
| 346 | setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); |
| 347 | AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32); |
| 348 | |
| 349 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); |
| 350 | AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32); |
| 351 | |
| 352 | setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); |
| 353 | AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32); |
| 354 | |
| 355 | setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); |
| 356 | AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32); |
| 357 | } |
| 358 | |
| 359 | for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) { |
| 360 | setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote); |
| 361 | AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32); |
| 362 | |
| 363 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote); |
| 364 | AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32); |
| 365 | |
| 366 | setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote); |
| 367 | AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32); |
| 368 | |
| 369 | setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote); |
| 370 | AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32); |
| 371 | } |
| 372 | |
| 373 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i32, Expand); |
| 374 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Expand); |
| 375 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i32, Expand); |
| 376 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16f32, Expand); |
| 377 | |
| 378 | setOperationAction(ISD::BUILD_VECTOR, MVT::v4f16, Custom); |
| 379 | setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); |
| 380 | |
| 381 | // Avoid stack access for these. |
| 382 | // TODO: Generalize to more vector types. |
| 383 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); |
| 384 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); |
| 385 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Custom); |
| 386 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Custom); |
| 387 | |
| 388 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i8, Custom); |
| 389 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i8, Custom); |
| 390 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i8, Custom); |
| 391 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i8, Custom); |
| 392 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i8, Custom); |
| 393 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i8, Custom); |
| 394 | |
| 395 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i16, Custom); |
| 396 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f16, Custom); |
| 397 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); |
| 398 | setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom); |
| 399 | |
| 400 | // Deal with vec3 vector operations when widened to vec4. |
| 401 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3i32, Custom); |
| 402 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3f32, Custom); |
| 403 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i32, Custom); |
| 404 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4f32, Custom); |
| 405 | |
| 406 | // Deal with vec5/6/7 vector operations when widened to vec8. |
| 407 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5i32, Custom); |
| 408 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5f32, Custom); |
| 409 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v6i32, Custom); |
| 410 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v6f32, Custom); |
| 411 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v7i32, Custom); |
| 412 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v7f32, Custom); |
| 413 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i32, Custom); |
| 414 | setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8f32, Custom); |
| 415 | |
| 416 | // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, |
| 417 | // and output demarshalling |
| 418 | setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); |
| 419 | setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); |
| 420 | |
| 421 | // We can't return success/failure, only the old value, |
| 422 | // let LLVM add the comparison |
| 423 | setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand); |
| 424 | setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Expand); |
| 425 | |
| 426 | if (Subtarget->hasFlatAddressSpace()) { |
| 427 | setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom); |
| 428 | setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom); |
| 429 | } |
| 430 | |
| 431 | setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); |
| 432 | setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); |
| 433 | |
| 434 | // FIXME: This should be narrowed to i32, but that only happens if i64 is |
| 435 | // illegal. |
| 436 | // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32. |
| 437 | setOperationAction(ISD::BSWAP, MVT::i64, Legal); |
| 438 | setOperationAction(ISD::BSWAP, MVT::i32, Legal); |
| 439 | |
| 440 | // On SI this is s_memtime and s_memrealtime on VI. |
| 441 | setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); |
| 442 | setOperationAction(ISD::TRAP, MVT::Other, Custom); |
| 443 | setOperationAction(ISD::DEBUGTRAP, MVT::Other, Custom); |
| 444 | |
| 445 | if (Subtarget->has16BitInsts()) { |
| 446 | setOperationAction(ISD::FPOW, MVT::f16, Promote); |
| 447 | setOperationAction(ISD::FPOWI, MVT::f16, Promote); |
| 448 | setOperationAction(ISD::FLOG, MVT::f16, Custom); |
| 449 | setOperationAction(ISD::FEXP, MVT::f16, Custom); |
| 450 | setOperationAction(ISD::FLOG10, MVT::f16, Custom); |
| 451 | } |
| 452 | |
| 453 | if (Subtarget->hasMadMacF32Insts()) |
| 454 | setOperationAction(ISD::FMAD, MVT::f32, Legal); |
| 455 | |
| 456 | if (!Subtarget->hasBFI()) { |
| 457 | // fcopysign can be done in a single instruction with BFI. |
| 458 | setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); |
| 459 | setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); |
| 460 | } |
| 461 | |
| 462 | if (!Subtarget->hasBCNT(32)) |
| 463 | setOperationAction(ISD::CTPOP, MVT::i32, Expand); |
| 464 | |
| 465 | if (!Subtarget->hasBCNT(64)) |
| 466 | setOperationAction(ISD::CTPOP, MVT::i64, Expand); |
| 467 | |
| 468 | if (Subtarget->hasFFBH()) |
| 469 | setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom); |
| 470 | |
| 471 | if (Subtarget->hasFFBL()) |
| 472 | setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom); |
| 473 | |
| 474 | // We only really have 32-bit BFE instructions (and 16-bit on VI). |
| 475 | // |
| 476 | // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any |
| 477 | // effort to match them now. We want this to be false for i64 cases when the |
| 478 | // extraction isn't restricted to the upper or lower half. Ideally we would |
| 479 | // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that |
| 480 | // span the midpoint are probably relatively rare, so don't worry about them |
| 481 | // for now. |
| 482 | if (Subtarget->hasBFE()) |
| 483 | setHasExtractBitsInsn(true); |
| 484 | |
| 485 | // Clamp modifier on add/sub |
| 486 | if (Subtarget->hasIntClamp()) { |
| 487 | setOperationAction(ISD::UADDSAT, MVT::i32, Legal); |
| 488 | setOperationAction(ISD::USUBSAT, MVT::i32, Legal); |
| 489 | } |
| 490 | |
| 491 | if (Subtarget->hasAddNoCarry()) { |
| 492 | setOperationAction(ISD::SADDSAT, MVT::i16, Legal); |
| 493 | setOperationAction(ISD::SSUBSAT, MVT::i16, Legal); |
| 494 | setOperationAction(ISD::SADDSAT, MVT::i32, Legal); |
| 495 | setOperationAction(ISD::SSUBSAT, MVT::i32, Legal); |
| 496 | } |
| 497 | |
| 498 | setOperationAction(ISD::FMINNUM, MVT::f32, Custom); |
| 499 | setOperationAction(ISD::FMAXNUM, MVT::f32, Custom); |
| 500 | setOperationAction(ISD::FMINNUM, MVT::f64, Custom); |
| 501 | setOperationAction(ISD::FMAXNUM, MVT::f64, Custom); |
| 502 | |
| 503 | |
| 504 | // These are really only legal for ieee_mode functions. We should be avoiding |
| 505 | // them for functions that don't have ieee_mode enabled, so just say they are |
| 506 | // legal. |
| 507 | setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); |
| 508 | setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); |
| 509 | setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); |
| 510 | setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); |
| 511 | |
| 512 | |
| 513 | if (Subtarget->haveRoundOpsF64()) { |
| 514 | setOperationAction(ISD::FTRUNC, MVT::f64, Legal); |
| 515 | setOperationAction(ISD::FCEIL, MVT::f64, Legal); |
| 516 | setOperationAction(ISD::FRINT, MVT::f64, Legal); |
| 517 | } else { |
| 518 | setOperationAction(ISD::FCEIL, MVT::f64, Custom); |
| 519 | setOperationAction(ISD::FTRUNC, MVT::f64, Custom); |
| 520 | setOperationAction(ISD::FRINT, MVT::f64, Custom); |
| 521 | setOperationAction(ISD::FFLOOR, MVT::f64, Custom); |
| 522 | } |
| 523 | |
| 524 | setOperationAction(ISD::FFLOOR, MVT::f64, Legal); |
| 525 | |
| 526 | setOperationAction(ISD::FSIN, MVT::f32, Custom); |
| 527 | setOperationAction(ISD::FCOS, MVT::f32, Custom); |
| 528 | setOperationAction(ISD::FDIV, MVT::f32, Custom); |
| 529 | setOperationAction(ISD::FDIV, MVT::f64, Custom); |
| 530 | |
| 531 | if (Subtarget->has16BitInsts()) { |
| 532 | setOperationAction(ISD::Constant, MVT::i16, Legal); |
| 533 | |
| 534 | setOperationAction(ISD::SMIN, MVT::i16, Legal); |
| 535 | setOperationAction(ISD::SMAX, MVT::i16, Legal); |
| 536 | |
| 537 | setOperationAction(ISD::UMIN, MVT::i16, Legal); |
| 538 | setOperationAction(ISD::UMAX, MVT::i16, Legal); |
| 539 | |
| 540 | setOperationAction(ISD::SIGN_EXTEND, MVT::i16, Promote); |
| 541 | AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32); |
| 542 | |
| 543 | setOperationAction(ISD::ROTR, MVT::i16, Expand); |
| 544 | setOperationAction(ISD::ROTL, MVT::i16, Expand); |
| 545 | |
| 546 | setOperationAction(ISD::SDIV, MVT::i16, Promote); |
| 547 | setOperationAction(ISD::UDIV, MVT::i16, Promote); |
| 548 | setOperationAction(ISD::SREM, MVT::i16, Promote); |
| 549 | setOperationAction(ISD::UREM, MVT::i16, Promote); |
| 550 | setOperationAction(ISD::UADDSAT, MVT::i16, Legal); |
| 551 | setOperationAction(ISD::USUBSAT, MVT::i16, Legal); |
| 552 | |
| 553 | setOperationAction(ISD::BITREVERSE, MVT::i16, Promote); |
| 554 | |
| 555 | setOperationAction(ISD::CTTZ, MVT::i16, Promote); |
| 556 | setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Promote); |
| 557 | setOperationAction(ISD::CTLZ, MVT::i16, Promote); |
| 558 | setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Promote); |
| 559 | setOperationAction(ISD::CTPOP, MVT::i16, Promote); |
| 560 | |
| 561 | setOperationAction(ISD::SELECT_CC, MVT::i16, Expand); |
| 562 | |
| 563 | setOperationAction(ISD::BR_CC, MVT::i16, Expand); |
| 564 | |
| 565 | setOperationAction(ISD::LOAD, MVT::i16, Custom); |
| 566 | |
| 567 | setTruncStoreAction(MVT::i64, MVT::i16, Expand); |
| 568 | |
| 569 | setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote); |
| 570 | AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32); |
| 571 | setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote); |
| 572 | AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32); |
| 573 | |
| 574 | setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom); |
| 575 | setOperationAction(ISD::FP_TO_UINT, MVT::i16, Custom); |
| 576 | |
| 577 | // F16 - Constant Actions. |
| 578 | setOperationAction(ISD::ConstantFP, MVT::f16, Legal); |
| 579 | |
| 580 | // F16 - Load/Store Actions. |
| 581 | setOperationAction(ISD::LOAD, MVT::f16, Promote); |
| 582 | AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16); |
| 583 | setOperationAction(ISD::STORE, MVT::f16, Promote); |
| 584 | AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16); |
| 585 | |
| 586 | // F16 - VOP1 Actions. |
| 587 | setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); |
| 588 | setOperationAction(ISD::FCOS, MVT::f16, Custom); |
| 589 | setOperationAction(ISD::FSIN, MVT::f16, Custom); |
| 590 | |
| 591 | setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom); |
| 592 | setOperationAction(ISD::UINT_TO_FP, MVT::i16, Custom); |
| 593 | |
| 594 | setOperationAction(ISD::FP_TO_SINT, MVT::f16, Promote); |
| 595 | setOperationAction(ISD::FP_TO_UINT, MVT::f16, Promote); |
| 596 | setOperationAction(ISD::SINT_TO_FP, MVT::f16, Promote); |
| 597 | setOperationAction(ISD::UINT_TO_FP, MVT::f16, Promote); |
| 598 | setOperationAction(ISD::FROUND, MVT::f16, Custom); |
| 599 | |
| 600 | // F16 - VOP2 Actions. |
| 601 | setOperationAction(ISD::BR_CC, MVT::f16, Expand); |
| 602 | setOperationAction(ISD::SELECT_CC, MVT::f16, Expand); |
| 603 | |
| 604 | setOperationAction(ISD::FDIV, MVT::f16, Custom); |
| 605 | |
| 606 | // F16 - VOP3 Actions. |
| 607 | setOperationAction(ISD::FMA, MVT::f16, Legal); |
| 608 | if (STI.hasMadF16()) |
| 609 | setOperationAction(ISD::FMAD, MVT::f16, Legal); |
| 610 | |
| 611 | for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) { |
| 612 | for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) { |
| 613 | switch (Op) { |
| 614 | case ISD::LOAD: |
| 615 | case ISD::STORE: |
| 616 | case ISD::BUILD_VECTOR: |
| 617 | case ISD::BITCAST: |
| 618 | case ISD::EXTRACT_VECTOR_ELT: |
| 619 | case ISD::INSERT_VECTOR_ELT: |
| 620 | case ISD::INSERT_SUBVECTOR: |
| 621 | case ISD::EXTRACT_SUBVECTOR: |
| 622 | case ISD::SCALAR_TO_VECTOR: |
| 623 | break; |
| 624 | case ISD::CONCAT_VECTORS: |
| 625 | setOperationAction(Op, VT, Custom); |
| 626 | break; |
| 627 | default: |
| 628 | setOperationAction(Op, VT, Expand); |
| 629 | break; |
| 630 | } |
| 631 | } |
| 632 | } |
| 633 | |
| 634 | // v_perm_b32 can handle either of these. |
| 635 | setOperationAction(ISD::BSWAP, MVT::i16, Legal); |
| 636 | setOperationAction(ISD::BSWAP, MVT::v2i16, Legal); |
| 637 | setOperationAction(ISD::BSWAP, MVT::v4i16, Custom); |
| 638 | |
| 639 | // XXX - Do these do anything? Vector constants turn into build_vector. |
| 640 | setOperationAction(ISD::Constant, MVT::v2i16, Legal); |
| 641 | setOperationAction(ISD::ConstantFP, MVT::v2f16, Legal); |
| 642 | |
| 643 | setOperationAction(ISD::UNDEF, MVT::v2i16, Legal); |
| 644 | setOperationAction(ISD::UNDEF, MVT::v2f16, Legal); |
| 645 | |
| 646 | setOperationAction(ISD::STORE, MVT::v2i16, Promote); |
| 647 | AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32); |
| 648 | setOperationAction(ISD::STORE, MVT::v2f16, Promote); |
| 649 | AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32); |
| 650 | |
| 651 | setOperationAction(ISD::LOAD, MVT::v2i16, Promote); |
| 652 | AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32); |
| 653 | setOperationAction(ISD::LOAD, MVT::v2f16, Promote); |
| 654 | AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32); |
| 655 | |
| 656 | setOperationAction(ISD::AND, MVT::v2i16, Promote); |
| 657 | AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32); |
| 658 | setOperationAction(ISD::OR, MVT::v2i16, Promote); |
| 659 | AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32); |
| 660 | setOperationAction(ISD::XOR, MVT::v2i16, Promote); |
| 661 | AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32); |
| 662 | |
| 663 | setOperationAction(ISD::LOAD, MVT::v4i16, Promote); |
| 664 | AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32); |
| 665 | setOperationAction(ISD::LOAD, MVT::v4f16, Promote); |
| 666 | AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32); |
| 667 | |
| 668 | setOperationAction(ISD::STORE, MVT::v4i16, Promote); |
| 669 | AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); |
| 670 | setOperationAction(ISD::STORE, MVT::v4f16, Promote); |
| 671 | AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); |
| 672 | |
| 673 | setOperationAction(ISD::ANY_EXTEND, MVT::v2i32, Expand); |
| 674 | setOperationAction(ISD::ZERO_EXTEND, MVT::v2i32, Expand); |
| 675 | setOperationAction(ISD::SIGN_EXTEND, MVT::v2i32, Expand); |
| 676 | setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand); |
| 677 | |
| 678 | setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Expand); |
| 679 | setOperationAction(ISD::ZERO_EXTEND, MVT::v4i32, Expand); |
| 680 | setOperationAction(ISD::SIGN_EXTEND, MVT::v4i32, Expand); |
| 681 | |
| 682 | if (!Subtarget->hasVOP3PInsts()) { |
| 683 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom); |
| 684 | setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom); |
| 685 | } |
| 686 | |
| 687 | setOperationAction(ISD::FNEG, MVT::v2f16, Legal); |
| 688 | // This isn't really legal, but this avoids the legalizer unrolling it (and |
| 689 | // allows matching fneg (fabs x) patterns) |
| 690 | setOperationAction(ISD::FABS, MVT::v2f16, Legal); |
| 691 | |
| 692 | setOperationAction(ISD::FMAXNUM, MVT::f16, Custom); |
| 693 | setOperationAction(ISD::FMINNUM, MVT::f16, Custom); |
| 694 | setOperationAction(ISD::FMAXNUM_IEEE, MVT::f16, Legal); |
| 695 | setOperationAction(ISD::FMINNUM_IEEE, MVT::f16, Legal); |
| 696 | |
| 697 | setOperationAction(ISD::FMINNUM_IEEE, MVT::v4f16, Custom); |
| 698 | setOperationAction(ISD::FMAXNUM_IEEE, MVT::v4f16, Custom); |
| 699 | |
| 700 | setOperationAction(ISD::FMINNUM, MVT::v4f16, Expand); |
| 701 | setOperationAction(ISD::FMAXNUM, MVT::v4f16, Expand); |
| 702 | } |
| 703 | |
| 704 | if (Subtarget->hasVOP3PInsts()) { |
| 705 | setOperationAction(ISD::ADD, MVT::v2i16, Legal); |
| 706 | setOperationAction(ISD::SUB, MVT::v2i16, Legal); |
| 707 | setOperationAction(ISD::MUL, MVT::v2i16, Legal); |
| 708 | setOperationAction(ISD::SHL, MVT::v2i16, Legal); |
| 709 | setOperationAction(ISD::SRL, MVT::v2i16, Legal); |
| 710 | setOperationAction(ISD::SRA, MVT::v2i16, Legal); |
| 711 | setOperationAction(ISD::SMIN, MVT::v2i16, Legal); |
| 712 | setOperationAction(ISD::UMIN, MVT::v2i16, Legal); |
| 713 | setOperationAction(ISD::SMAX, MVT::v2i16, Legal); |
| 714 | setOperationAction(ISD::UMAX, MVT::v2i16, Legal); |
| 715 | |
| 716 | setOperationAction(ISD::UADDSAT, MVT::v2i16, Legal); |
| 717 | setOperationAction(ISD::USUBSAT, MVT::v2i16, Legal); |
| 718 | setOperationAction(ISD::SADDSAT, MVT::v2i16, Legal); |
| 719 | setOperationAction(ISD::SSUBSAT, MVT::v2i16, Legal); |
| 720 | |
| 721 | setOperationAction(ISD::FADD, MVT::v2f16, Legal); |
| 722 | setOperationAction(ISD::FMUL, MVT::v2f16, Legal); |
| 723 | setOperationAction(ISD::FMA, MVT::v2f16, Legal); |
| 724 | |
| 725 | setOperationAction(ISD::FMINNUM_IEEE, MVT::v2f16, Legal); |
| 726 | setOperationAction(ISD::FMAXNUM_IEEE, MVT::v2f16, Legal); |
| 727 | |
| 728 | setOperationAction(ISD::FCANONICALIZE, MVT::v2f16, Legal); |
| 729 | |
| 730 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); |
| 731 | setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); |
| 732 | |
| 733 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f16, Custom); |
| 734 | setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); |
| 735 | |
| 736 | setOperationAction(ISD::SHL, MVT::v4i16, Custom); |
| 737 | setOperationAction(ISD::SRA, MVT::v4i16, Custom); |
| 738 | setOperationAction(ISD::SRL, MVT::v4i16, Custom); |
| 739 | setOperationAction(ISD::ADD, MVT::v4i16, Custom); |
| 740 | setOperationAction(ISD::SUB, MVT::v4i16, Custom); |
| 741 | setOperationAction(ISD::MUL, MVT::v4i16, Custom); |
| 742 | |
| 743 | setOperationAction(ISD::SMIN, MVT::v4i16, Custom); |
| 744 | setOperationAction(ISD::SMAX, MVT::v4i16, Custom); |
| 745 | setOperationAction(ISD::UMIN, MVT::v4i16, Custom); |
| 746 | setOperationAction(ISD::UMAX, MVT::v4i16, Custom); |
| 747 | |
| 748 | setOperationAction(ISD::UADDSAT, MVT::v4i16, Custom); |
| 749 | setOperationAction(ISD::SADDSAT, MVT::v4i16, Custom); |
| 750 | setOperationAction(ISD::USUBSAT, MVT::v4i16, Custom); |
| 751 | setOperationAction(ISD::SSUBSAT, MVT::v4i16, Custom); |
| 752 | |
| 753 | setOperationAction(ISD::FADD, MVT::v4f16, Custom); |
| 754 | setOperationAction(ISD::FMUL, MVT::v4f16, Custom); |
| 755 | setOperationAction(ISD::FMA, MVT::v4f16, Custom); |
| 756 | |
| 757 | setOperationAction(ISD::FMAXNUM, MVT::v2f16, Custom); |
| 758 | setOperationAction(ISD::FMINNUM, MVT::v2f16, Custom); |
| 759 | |
| 760 | setOperationAction(ISD::FMINNUM, MVT::v4f16, Custom); |
| 761 | setOperationAction(ISD::FMAXNUM, MVT::v4f16, Custom); |
| 762 | setOperationAction(ISD::FCANONICALIZE, MVT::v4f16, Custom); |
| 763 | |
| 764 | setOperationAction(ISD::FEXP, MVT::v2f16, Custom); |
| 765 | setOperationAction(ISD::SELECT, MVT::v4i16, Custom); |
| 766 | setOperationAction(ISD::SELECT, MVT::v4f16, Custom); |
| 767 | |
| 768 | if (Subtarget->hasPackedFP32Ops()) { |
| 769 | setOperationAction(ISD::FADD, MVT::v2f32, Legal); |
| 770 | setOperationAction(ISD::FMUL, MVT::v2f32, Legal); |
| 771 | setOperationAction(ISD::FMA, MVT::v2f32, Legal); |
| 772 | setOperationAction(ISD::FNEG, MVT::v2f32, Legal); |
| 773 | |
| 774 | for (MVT VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32 }) { |
| 775 | setOperationAction(ISD::FADD, VT, Custom); |
| 776 | setOperationAction(ISD::FMUL, VT, Custom); |
| 777 | setOperationAction(ISD::FMA, VT, Custom); |
| 778 | } |
| 779 | } |
| 780 | } |
| 781 | |
| 782 | setOperationAction(ISD::FNEG, MVT::v4f16, Custom); |
| 783 | setOperationAction(ISD::FABS, MVT::v4f16, Custom); |
| 784 | |
| 785 | if (Subtarget->has16BitInsts()) { |
| 786 | setOperationAction(ISD::SELECT, MVT::v2i16, Promote); |
| 787 | AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32); |
| 788 | setOperationAction(ISD::SELECT, MVT::v2f16, Promote); |
| 789 | AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32); |
| 790 | } else { |
| 791 | // Legalization hack. |
| 792 | setOperationAction(ISD::SELECT, MVT::v2i16, Custom); |
| 793 | setOperationAction(ISD::SELECT, MVT::v2f16, Custom); |
| 794 | |
| 795 | setOperationAction(ISD::FNEG, MVT::v2f16, Custom); |
| 796 | setOperationAction(ISD::FABS, MVT::v2f16, Custom); |
| 797 | } |
| 798 | |
| 799 | for (MVT VT : { MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8 }) { |
| 800 | setOperationAction(ISD::SELECT, VT, Custom); |
| 801 | } |
| 802 | |
| 803 | setOperationAction(ISD::SMULO, MVT::i64, Custom); |
| 804 | setOperationAction(ISD::UMULO, MVT::i64, Custom); |
| 805 | |
| 806 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); |
| 807 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom); |
| 808 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom); |
| 809 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); |
| 810 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f16, Custom); |
| 811 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2i16, Custom); |
| 812 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2f16, Custom); |
| 813 | |
| 814 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2f16, Custom); |
| 815 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2i16, Custom); |
| 816 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v3f16, Custom); |
| 817 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v3i16, Custom); |
| 818 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4f16, Custom); |
| 819 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4i16, Custom); |
| 820 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v8f16, Custom); |
| 821 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); |
| 822 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::f16, Custom); |
| 823 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom); |
| 824 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom); |
| 825 | |
| 826 | setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); |
| 827 | setOperationAction(ISD::INTRINSIC_VOID, MVT::v2i16, Custom); |
| 828 | setOperationAction(ISD::INTRINSIC_VOID, MVT::v2f16, Custom); |
| 829 | setOperationAction(ISD::INTRINSIC_VOID, MVT::v3i16, Custom); |
| 830 | setOperationAction(ISD::INTRINSIC_VOID, MVT::v3f16, Custom); |
| 831 | setOperationAction(ISD::INTRINSIC_VOID, MVT::v4f16, Custom); |
| 832 | setOperationAction(ISD::INTRINSIC_VOID, MVT::v4i16, Custom); |
| 833 | setOperationAction(ISD::INTRINSIC_VOID, MVT::f16, Custom); |
| 834 | setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); |
| 835 | setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); |
| 836 | |
| 837 | setTargetDAGCombine(ISD::ADD); |
| 838 | setTargetDAGCombine(ISD::ADDCARRY); |
| 839 | setTargetDAGCombine(ISD::SUB); |
| 840 | setTargetDAGCombine(ISD::SUBCARRY); |
| 841 | setTargetDAGCombine(ISD::FADD); |
| 842 | setTargetDAGCombine(ISD::FSUB); |
| 843 | setTargetDAGCombine(ISD::FMINNUM); |
| 844 | setTargetDAGCombine(ISD::FMAXNUM); |
| 845 | setTargetDAGCombine(ISD::FMINNUM_IEEE); |
| 846 | setTargetDAGCombine(ISD::FMAXNUM_IEEE); |
| 847 | setTargetDAGCombine(ISD::FMA); |
| 848 | setTargetDAGCombine(ISD::SMIN); |
| 849 | setTargetDAGCombine(ISD::SMAX); |
| 850 | setTargetDAGCombine(ISD::UMIN); |
| 851 | setTargetDAGCombine(ISD::UMAX); |
| 852 | setTargetDAGCombine(ISD::SETCC); |
| 853 | setTargetDAGCombine(ISD::AND); |
| 854 | setTargetDAGCombine(ISD::OR); |
| 855 | setTargetDAGCombine(ISD::XOR); |
| 856 | setTargetDAGCombine(ISD::SINT_TO_FP); |
| 857 | setTargetDAGCombine(ISD::UINT_TO_FP); |
| 858 | setTargetDAGCombine(ISD::FCANONICALIZE); |
| 859 | setTargetDAGCombine(ISD::SCALAR_TO_VECTOR); |
| 860 | setTargetDAGCombine(ISD::ZERO_EXTEND); |
| 861 | setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); |
| 862 | setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); |
| 863 | setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); |
| 864 | |
| 865 | // All memory operations. Some folding on the pointer operand is done to help |
| 866 | // matching the constant offsets in the addressing modes. |
| 867 | setTargetDAGCombine(ISD::LOAD); |
| 868 | setTargetDAGCombine(ISD::STORE); |
| 869 | setTargetDAGCombine(ISD::ATOMIC_LOAD); |
| 870 | setTargetDAGCombine(ISD::ATOMIC_STORE); |
| 871 | setTargetDAGCombine(ISD::ATOMIC_CMP_SWAP); |
| 872 | setTargetDAGCombine(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
| 873 | setTargetDAGCombine(ISD::ATOMIC_SWAP); |
| 874 | setTargetDAGCombine(ISD::ATOMIC_LOAD_ADD); |
| 875 | setTargetDAGCombine(ISD::ATOMIC_LOAD_SUB); |
| 876 | setTargetDAGCombine(ISD::ATOMIC_LOAD_AND); |
| 877 | setTargetDAGCombine(ISD::ATOMIC_LOAD_OR); |
| 878 | setTargetDAGCombine(ISD::ATOMIC_LOAD_XOR); |
| 879 | setTargetDAGCombine(ISD::ATOMIC_LOAD_NAND); |
| 880 | setTargetDAGCombine(ISD::ATOMIC_LOAD_MIN); |
| 881 | setTargetDAGCombine(ISD::ATOMIC_LOAD_MAX); |
| 882 | setTargetDAGCombine(ISD::ATOMIC_LOAD_UMIN); |
| 883 | setTargetDAGCombine(ISD::ATOMIC_LOAD_UMAX); |
| 884 | setTargetDAGCombine(ISD::ATOMIC_LOAD_FADD); |
| 885 | setTargetDAGCombine(ISD::INTRINSIC_VOID); |
| 886 | setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); |
| 887 | |
| 888 | // FIXME: In other contexts we pretend this is a per-function property. |
| 889 | setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32); |
| 890 | |
| 891 | setSchedulingPreference(Sched::RegPressure); |
| 892 | } |
| 893 | |
| 894 | const GCNSubtarget *SITargetLowering::getSubtarget() const { |
| 895 | return Subtarget; |
| 896 | } |
| 897 | |
| 898 | //===----------------------------------------------------------------------===// |
| 899 | // TargetLowering queries |
| 900 | //===----------------------------------------------------------------------===// |
| 901 | |
| 902 | // v_mad_mix* support a conversion from f16 to f32. |
| 903 | // |
| 904 | // There is only one special case when denormals are enabled we don't currently, |
| 905 | // where this is OK to use. |
| 906 | bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, |
| 907 | EVT DestVT, EVT SrcVT) const { |
| 908 | return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) || |
| 909 | (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) && |
| 910 | DestVT.getScalarType() == MVT::f32 && |
| 911 | SrcVT.getScalarType() == MVT::f16 && |
| 912 | // TODO: This probably only requires no input flushing? |
| 913 | !hasFP32Denormals(DAG.getMachineFunction()); |
| 914 | } |
| 915 | |
| 916 | bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const { |
| 917 | // SI has some legal vector types, but no legal vector operations. Say no |
| 918 | // shuffles are legal in order to prefer scalarizing some vector operations. |
| 919 | return false; |
| 920 | } |
| 921 | |
| 922 | MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, |
| 923 | CallingConv::ID CC, |
| 924 | EVT VT) const { |
| 925 | if (CC == CallingConv::AMDGPU_KERNEL) |
| 926 | return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); |
| 927 | |
| 928 | if (VT.isVector()) { |
| 929 | EVT ScalarVT = VT.getScalarType(); |
| 930 | unsigned Size = ScalarVT.getSizeInBits(); |
| 931 | if (Size == 16) { |
| 932 | if (Subtarget->has16BitInsts()) |
| 933 | return VT.isInteger() ? MVT::v2i16 : MVT::v2f16; |
| 934 | return VT.isInteger() ? MVT::i32 : MVT::f32; |
| 935 | } |
| 936 | |
| 937 | if (Size < 16) |
| 938 | return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32; |
| 939 | return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32; |
| 940 | } |
| 941 | |
| 942 | if (VT.getSizeInBits() > 32) |
| 943 | return MVT::i32; |
| 944 | |
| 945 | return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); |
| 946 | } |
| 947 | |
| 948 | unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context, |
| 949 | CallingConv::ID CC, |
| 950 | EVT VT) const { |
| 951 | if (CC == CallingConv::AMDGPU_KERNEL) |
| 952 | return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT); |
| 953 | |
| 954 | if (VT.isVector()) { |
| 955 | unsigned NumElts = VT.getVectorNumElements(); |
| 956 | EVT ScalarVT = VT.getScalarType(); |
| 957 | unsigned Size = ScalarVT.getSizeInBits(); |
| 958 | |
| 959 | // FIXME: Should probably promote 8-bit vectors to i16. |
| 960 | if (Size == 16 && Subtarget->has16BitInsts()) |
| 961 | return (NumElts + 1) / 2; |
| 962 | |
| 963 | if (Size <= 32) |
| 964 | return NumElts; |
| 965 | |
| 966 | if (Size > 32) |
| 967 | return NumElts * ((Size + 31) / 32); |
| 968 | } else if (VT.getSizeInBits() > 32) |
| 969 | return (VT.getSizeInBits() + 31) / 32; |
| 970 | |
| 971 | return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT); |
| 972 | } |
| 973 | |
| 974 | unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv( |
| 975 | LLVMContext &Context, CallingConv::ID CC, |
| 976 | EVT VT, EVT &IntermediateVT, |
| 977 | unsigned &NumIntermediates, MVT &RegisterVT) const { |
| 978 | if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) { |
| 979 | unsigned NumElts = VT.getVectorNumElements(); |
| 980 | EVT ScalarVT = VT.getScalarType(); |
| 981 | unsigned Size = ScalarVT.getSizeInBits(); |
| 982 | // FIXME: We should fix the ABI to be the same on targets without 16-bit |
| 983 | // support, but unless we can properly handle 3-vectors, it will be still be |
| 984 | // inconsistent. |
| 985 | if (Size == 16 && Subtarget->has16BitInsts()) { |
| 986 | RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16; |
| 987 | IntermediateVT = RegisterVT; |
| 988 | NumIntermediates = (NumElts + 1) / 2; |
| 989 | return NumIntermediates; |
| 990 | } |
| 991 | |
| 992 | if (Size == 32) { |
| 993 | RegisterVT = ScalarVT.getSimpleVT(); |
| 994 | IntermediateVT = RegisterVT; |
| 995 | NumIntermediates = NumElts; |
| 996 | return NumIntermediates; |
| 997 | } |
| 998 | |
| 999 | if (Size < 16 && Subtarget->has16BitInsts()) { |
| 1000 | // FIXME: Should probably form v2i16 pieces |
| 1001 | RegisterVT = MVT::i16; |
| 1002 | IntermediateVT = ScalarVT; |
| 1003 | NumIntermediates = NumElts; |
| 1004 | return NumIntermediates; |
| 1005 | } |
| 1006 | |
| 1007 | |
| 1008 | if (Size != 16 && Size <= 32) { |
| 1009 | RegisterVT = MVT::i32; |
| 1010 | IntermediateVT = ScalarVT; |
| 1011 | NumIntermediates = NumElts; |
| 1012 | return NumIntermediates; |
| 1013 | } |
| 1014 | |
| 1015 | if (Size > 32) { |
| 1016 | RegisterVT = MVT::i32; |
| 1017 | IntermediateVT = RegisterVT; |
| 1018 | NumIntermediates = NumElts * ((Size + 31) / 32); |
| 1019 | return NumIntermediates; |
| 1020 | } |
| 1021 | } |
| 1022 | |
| 1023 | return TargetLowering::getVectorTypeBreakdownForCallingConv( |
| 1024 | Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT); |
| 1025 | } |
| 1026 | |
| 1027 | static EVT memVTFromImageData(Type *Ty, unsigned DMaskLanes) { |
| 1028 | assert(DMaskLanes != 0)((void)0); |
| 1029 | |
| 1030 | if (auto *VT = dyn_cast<FixedVectorType>(Ty)) { |
| 1031 | unsigned NumElts = std::min(DMaskLanes, VT->getNumElements()); |
| 1032 | return EVT::getVectorVT(Ty->getContext(), |
| 1033 | EVT::getEVT(VT->getElementType()), |
| 1034 | NumElts); |
| 1035 | } |
| 1036 | |
| 1037 | return EVT::getEVT(Ty); |
| 1038 | } |
| 1039 | |
| 1040 | // Peek through TFE struct returns to only use the data size. |
| 1041 | static EVT memVTFromImageReturn(Type *Ty, unsigned DMaskLanes) { |
| 1042 | auto *ST = dyn_cast<StructType>(Ty); |
| 1043 | if (!ST) |
| 1044 | return memVTFromImageData(Ty, DMaskLanes); |
| 1045 | |
| 1046 | // Some intrinsics return an aggregate type - special case to work out the |
| 1047 | // correct memVT. |
| 1048 | // |
| 1049 | // Only limited forms of aggregate type currently expected. |
| 1050 | if (ST->getNumContainedTypes() != 2 || |
| 1051 | !ST->getContainedType(1)->isIntegerTy(32)) |
| 1052 | return EVT(); |
| 1053 | return memVTFromImageData(ST->getContainedType(0), DMaskLanes); |
| 1054 | } |
| 1055 | |
| 1056 | bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, |
| 1057 | const CallInst &CI, |
| 1058 | MachineFunction &MF, |
| 1059 | unsigned IntrID) const { |
| 1060 | if (const AMDGPU::RsrcIntrinsic *RsrcIntr = |
| 1061 | AMDGPU::lookupRsrcIntrinsic(IntrID)) { |
| 1062 | AttributeList Attr = Intrinsic::getAttributes(CI.getContext(), |
| 1063 | (Intrinsic::ID)IntrID); |
| 1064 | if (Attr.hasFnAttribute(Attribute::ReadNone)) |
| 1065 | return false; |
| 1066 | |
| 1067 | SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| 1068 | |
| 1069 | if (RsrcIntr->IsImage) { |
| 1070 | Info.ptrVal = |
| 1071 | MFI->getImagePSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); |
| 1072 | Info.align.reset(); |
| 1073 | } else { |
| 1074 | Info.ptrVal = |
| 1075 | MFI->getBufferPSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); |
| 1076 | } |
| 1077 | |
| 1078 | Info.flags = MachineMemOperand::MODereferenceable; |
| 1079 | if (Attr.hasFnAttribute(Attribute::ReadOnly)) { |
| 1080 | unsigned DMaskLanes = 4; |
| 1081 | |
| 1082 | if (RsrcIntr->IsImage) { |
| 1083 | const AMDGPU::ImageDimIntrinsicInfo *Intr |
| 1084 | = AMDGPU::getImageDimIntrinsicInfo(IntrID); |
| 1085 | const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = |
| 1086 | AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); |
| 1087 | |
| 1088 | if (!BaseOpcode->Gather4) { |
| 1089 | // If this isn't a gather, we may have excess loaded elements in the |
| 1090 | // IR type. Check the dmask for the real number of elements loaded. |
| 1091 | unsigned DMask |
| 1092 | = cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue(); |
| 1093 | DMaskLanes = DMask == 0 ? 1 : countPopulation(DMask); |
| 1094 | } |
| 1095 | |
| 1096 | Info.memVT = memVTFromImageReturn(CI.getType(), DMaskLanes); |
| 1097 | } else |
| 1098 | Info.memVT = EVT::getEVT(CI.getType()); |
| 1099 | |
| 1100 | // FIXME: What does alignment mean for an image? |
| 1101 | Info.opc = ISD::INTRINSIC_W_CHAIN; |
| 1102 | Info.flags |= MachineMemOperand::MOLoad; |
| 1103 | } else if (Attr.hasFnAttribute(Attribute::WriteOnly)) { |
| 1104 | Info.opc = ISD::INTRINSIC_VOID; |
| 1105 | |
| 1106 | Type *DataTy = CI.getArgOperand(0)->getType(); |
| 1107 | if (RsrcIntr->IsImage) { |
| 1108 | unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue(); |
| 1109 | unsigned DMaskLanes = DMask == 0 ? 1 : countPopulation(DMask); |
| 1110 | Info.memVT = memVTFromImageData(DataTy, DMaskLanes); |
| 1111 | } else |
| 1112 | Info.memVT = EVT::getEVT(DataTy); |
| 1113 | |
| 1114 | Info.flags |= MachineMemOperand::MOStore; |
| 1115 | } else { |
| 1116 | // Atomic |
| 1117 | Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID : |
| 1118 | ISD::INTRINSIC_W_CHAIN; |
| 1119 | Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType()); |
| 1120 | Info.flags = MachineMemOperand::MOLoad | |
| 1121 | MachineMemOperand::MOStore | |
| 1122 | MachineMemOperand::MODereferenceable; |
| 1123 | |
| 1124 | // XXX - Should this be volatile without known ordering? |
| 1125 | Info.flags |= MachineMemOperand::MOVolatile; |
| 1126 | } |
| 1127 | return true; |
| 1128 | } |
| 1129 | |
| 1130 | switch (IntrID) { |
| 1131 | case Intrinsic::amdgcn_atomic_inc: |
| 1132 | case Intrinsic::amdgcn_atomic_dec: |
| 1133 | case Intrinsic::amdgcn_ds_ordered_add: |
| 1134 | case Intrinsic::amdgcn_ds_ordered_swap: |
| 1135 | case Intrinsic::amdgcn_ds_fadd: |
| 1136 | case Intrinsic::amdgcn_ds_fmin: |
| 1137 | case Intrinsic::amdgcn_ds_fmax: { |
| 1138 | Info.opc = ISD::INTRINSIC_W_CHAIN; |
| 1139 | Info.memVT = MVT::getVT(CI.getType()); |
| 1140 | Info.ptrVal = CI.getOperand(0); |
| 1141 | Info.align.reset(); |
| 1142 | Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; |
| 1143 | |
| 1144 | const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4)); |
| 1145 | if (!Vol->isZero()) |
| 1146 | Info.flags |= MachineMemOperand::MOVolatile; |
| 1147 | |
| 1148 | return true; |
| 1149 | } |
| 1150 | case Intrinsic::amdgcn_buffer_atomic_fadd: { |
| 1151 | SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| 1152 | |
| 1153 | Info.opc = ISD::INTRINSIC_W_CHAIN; |
| 1154 | Info.memVT = MVT::getVT(CI.getOperand(0)->getType()); |
| 1155 | Info.ptrVal = |
| 1156 | MFI->getBufferPSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); |
| 1157 | Info.align.reset(); |
| 1158 | Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; |
| 1159 | |
| 1160 | const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4)); |
| 1161 | if (!Vol || !Vol->isZero()) |
| 1162 | Info.flags |= MachineMemOperand::MOVolatile; |
| 1163 | |
| 1164 | return true; |
| 1165 | } |
| 1166 | case Intrinsic::amdgcn_ds_append: |
| 1167 | case Intrinsic::amdgcn_ds_consume: { |
| 1168 | Info.opc = ISD::INTRINSIC_W_CHAIN; |
| 1169 | Info.memVT = MVT::getVT(CI.getType()); |
| 1170 | Info.ptrVal = CI.getOperand(0); |
| 1171 | Info.align.reset(); |
| 1172 | Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; |
| 1173 | |
| 1174 | const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1)); |
| 1175 | if (!Vol->isZero()) |
| 1176 | Info.flags |= MachineMemOperand::MOVolatile; |
| 1177 | |
| 1178 | return true; |
| 1179 | } |
| 1180 | case Intrinsic::amdgcn_global_atomic_csub: { |
| 1181 | Info.opc = ISD::INTRINSIC_W_CHAIN; |
| 1182 | Info.memVT = MVT::getVT(CI.getType()); |
| 1183 | Info.ptrVal = CI.getOperand(0); |
| 1184 | Info.align.reset(); |
| 1185 | Info.flags = MachineMemOperand::MOLoad | |
| 1186 | MachineMemOperand::MOStore | |
| 1187 | MachineMemOperand::MOVolatile; |
| 1188 | return true; |
| 1189 | } |
| 1190 | case Intrinsic::amdgcn_image_bvh_intersect_ray: { |
| 1191 | SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| 1192 | Info.opc = ISD::INTRINSIC_W_CHAIN; |
| 1193 | Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT? |
| 1194 | Info.ptrVal = |
| 1195 | MFI->getImagePSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); |
| 1196 | Info.align.reset(); |
| 1197 | Info.flags = MachineMemOperand::MOLoad | |
| 1198 | MachineMemOperand::MODereferenceable; |
| 1199 | return true; |
| 1200 | } |
| 1201 | case Intrinsic::amdgcn_global_atomic_fadd: |
| 1202 | case Intrinsic::amdgcn_global_atomic_fmin: |
| 1203 | case Intrinsic::amdgcn_global_atomic_fmax: |
| 1204 | case Intrinsic::amdgcn_flat_atomic_fadd: |
| 1205 | case Intrinsic::amdgcn_flat_atomic_fmin: |
| 1206 | case Intrinsic::amdgcn_flat_atomic_fmax: { |
| 1207 | Info.opc = ISD::INTRINSIC_W_CHAIN; |
| 1208 | Info.memVT = MVT::getVT(CI.getType()); |
| 1209 | Info.ptrVal = CI.getOperand(0); |
| 1210 | Info.align.reset(); |
| 1211 | Info.flags = MachineMemOperand::MOLoad | |
| 1212 | MachineMemOperand::MOStore | |
| 1213 | MachineMemOperand::MODereferenceable | |
| 1214 | MachineMemOperand::MOVolatile; |
| 1215 | return true; |
| 1216 | } |
| 1217 | case Intrinsic::amdgcn_ds_gws_init: |
| 1218 | case Intrinsic::amdgcn_ds_gws_barrier: |
| 1219 | case Intrinsic::amdgcn_ds_gws_sema_v: |
| 1220 | case Intrinsic::amdgcn_ds_gws_sema_br: |
| 1221 | case Intrinsic::amdgcn_ds_gws_sema_p: |
| 1222 | case Intrinsic::amdgcn_ds_gws_sema_release_all: { |
| 1223 | Info.opc = ISD::INTRINSIC_VOID; |
| 1224 | |
| 1225 | SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| 1226 | Info.ptrVal = |
| 1227 | MFI->getGWSPSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()); |
| 1228 | |
| 1229 | // This is an abstract access, but we need to specify a type and size. |
| 1230 | Info.memVT = MVT::i32; |
| 1231 | Info.size = 4; |
| 1232 | Info.align = Align(4); |
| 1233 | |
| 1234 | Info.flags = MachineMemOperand::MOStore; |
| 1235 | if (IntrID == Intrinsic::amdgcn_ds_gws_barrier) |
| 1236 | Info.flags = MachineMemOperand::MOLoad; |
| 1237 | return true; |
| 1238 | } |
| 1239 | default: |
| 1240 | return false; |
| 1241 | } |
| 1242 | } |
| 1243 | |
| 1244 | bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II, |
| 1245 | SmallVectorImpl<Value*> &Ops, |
| 1246 | Type *&AccessTy) const { |
| 1247 | switch (II->getIntrinsicID()) { |
| 1248 | case Intrinsic::amdgcn_atomic_inc: |
| 1249 | case Intrinsic::amdgcn_atomic_dec: |
| 1250 | case Intrinsic::amdgcn_ds_ordered_add: |
| 1251 | case Intrinsic::amdgcn_ds_ordered_swap: |
| 1252 | case Intrinsic::amdgcn_ds_append: |
| 1253 | case Intrinsic::amdgcn_ds_consume: |
| 1254 | case Intrinsic::amdgcn_ds_fadd: |
| 1255 | case Intrinsic::amdgcn_ds_fmin: |
| 1256 | case Intrinsic::amdgcn_ds_fmax: |
| 1257 | case Intrinsic::amdgcn_global_atomic_fadd: |
| 1258 | case Intrinsic::amdgcn_flat_atomic_fadd: |
| 1259 | case Intrinsic::amdgcn_flat_atomic_fmin: |
| 1260 | case Intrinsic::amdgcn_flat_atomic_fmax: |
| 1261 | case Intrinsic::amdgcn_global_atomic_csub: { |
| 1262 | Value *Ptr = II->getArgOperand(0); |
| 1263 | AccessTy = II->getType(); |
| 1264 | Ops.push_back(Ptr); |
| 1265 | return true; |
| 1266 | } |
| 1267 | default: |
| 1268 | return false; |
| 1269 | } |
| 1270 | } |
| 1271 | |
| 1272 | bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const { |
| 1273 | if (!Subtarget->hasFlatInstOffsets()) { |
| 1274 | // Flat instructions do not have offsets, and only have the register |
| 1275 | // address. |
| 1276 | return AM.BaseOffs == 0 && AM.Scale == 0; |
| 1277 | } |
| 1278 | |
| 1279 | return AM.Scale == 0 && |
| 1280 | (AM.BaseOffs == 0 || |
| 1281 | Subtarget->getInstrInfo()->isLegalFLATOffset( |
| 1282 | AM.BaseOffs, AMDGPUAS::FLAT_ADDRESS, SIInstrFlags::FLAT)); |
| 1283 | } |
| 1284 | |
| 1285 | bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const { |
| 1286 | if (Subtarget->hasFlatGlobalInsts()) |
| 1287 | return AM.Scale == 0 && |
| 1288 | (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset( |
| 1289 | AM.BaseOffs, AMDGPUAS::GLOBAL_ADDRESS, |
| 1290 | SIInstrFlags::FlatGlobal)); |
| 1291 | |
| 1292 | if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) { |
| 1293 | // Assume the we will use FLAT for all global memory accesses |
| 1294 | // on VI. |
| 1295 | // FIXME: This assumption is currently wrong. On VI we still use |
| 1296 | // MUBUF instructions for the r + i addressing mode. As currently |
| 1297 | // implemented, the MUBUF instructions only work on buffer < 4GB. |
| 1298 | // It may be possible to support > 4GB buffers with MUBUF instructions, |
| 1299 | // by setting the stride value in the resource descriptor which would |
| 1300 | // increase the size limit to (stride * 4GB). However, this is risky, |
| 1301 | // because it has never been validated. |
| 1302 | return isLegalFlatAddressingMode(AM); |
| 1303 | } |
| 1304 | |
| 1305 | return isLegalMUBUFAddressingMode(AM); |
| 1306 | } |
| 1307 | |
| 1308 | bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const { |
| 1309 | // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and |
| 1310 | // additionally can do r + r + i with addr64. 32-bit has more addressing |
| 1311 | // mode options. Depending on the resource constant, it can also do |
| 1312 | // (i64 r0) + (i32 r1) * (i14 i). |
| 1313 | // |
| 1314 | // Private arrays end up using a scratch buffer most of the time, so also |
| 1315 | // assume those use MUBUF instructions. Scratch loads / stores are currently |
| 1316 | // implemented as mubuf instructions with offen bit set, so slightly |
| 1317 | // different than the normal addr64. |
| 1318 | if (!SIInstrInfo::isLegalMUBUFImmOffset(AM.BaseOffs)) |
| 1319 | return false; |
| 1320 | |
| 1321 | // FIXME: Since we can split immediate into soffset and immediate offset, |
| 1322 | // would it make sense to allow any immediate? |
| 1323 | |
| 1324 | switch (AM.Scale) { |
| 1325 | case 0: // r + i or just i, depending on HasBaseReg. |
| 1326 | return true; |
| 1327 | case 1: |
| 1328 | return true; // We have r + r or r + i. |
| 1329 | case 2: |
| 1330 | if (AM.HasBaseReg) { |
| 1331 | // Reject 2 * r + r. |
| 1332 | return false; |
| 1333 | } |
| 1334 | |
| 1335 | // Allow 2 * r as r + r |
| 1336 | // Or 2 * r + i is allowed as r + r + i. |
| 1337 | return true; |
| 1338 | default: // Don't allow n * r |
| 1339 | return false; |
| 1340 | } |
| 1341 | } |
| 1342 | |
| 1343 | bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL, |
| 1344 | const AddrMode &AM, Type *Ty, |
| 1345 | unsigned AS, Instruction *I) const { |
| 1346 | // No global is ever allowed as a base. |
| 1347 | if (AM.BaseGV) |
| 1348 | return false; |
| 1349 | |
| 1350 | if (AS == AMDGPUAS::GLOBAL_ADDRESS) |
| 1351 | return isLegalGlobalAddressingMode(AM); |
| 1352 | |
| 1353 | if (AS == AMDGPUAS::CONSTANT_ADDRESS || |
| 1354 | AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT || |
| 1355 | AS == AMDGPUAS::BUFFER_FAT_POINTER) { |
| 1356 | // If the offset isn't a multiple of 4, it probably isn't going to be |
| 1357 | // correctly aligned. |
| 1358 | // FIXME: Can we get the real alignment here? |
| 1359 | if (AM.BaseOffs % 4 != 0) |
| 1360 | return isLegalMUBUFAddressingMode(AM); |
| 1361 | |
| 1362 | // There are no SMRD extloads, so if we have to do a small type access we |
| 1363 | // will use a MUBUF load. |
| 1364 | // FIXME?: We also need to do this if unaligned, but we don't know the |
| 1365 | // alignment here. |
| 1366 | if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4) |
| 1367 | return isLegalGlobalAddressingMode(AM); |
| 1368 | |
| 1369 | if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) { |
| 1370 | // SMRD instructions have an 8-bit, dword offset on SI. |
| 1371 | if (!isUInt<8>(AM.BaseOffs / 4)) |
| 1372 | return false; |
| 1373 | } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) { |
| 1374 | // On CI+, this can also be a 32-bit literal constant offset. If it fits |
| 1375 | // in 8-bits, it can use a smaller encoding. |
| 1376 | if (!isUInt<32>(AM.BaseOffs / 4)) |
| 1377 | return false; |
| 1378 | } else if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { |
| 1379 | // On VI, these use the SMEM format and the offset is 20-bit in bytes. |
| 1380 | if (!isUInt<20>(AM.BaseOffs)) |
| 1381 | return false; |
| 1382 | } else |
| 1383 | llvm_unreachable("unhandled generation")__builtin_unreachable(); |
| 1384 | |
| 1385 | if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg. |
| 1386 | return true; |
| 1387 | |
| 1388 | if (AM.Scale == 1 && AM.HasBaseReg) |
| 1389 | return true; |
| 1390 | |
| 1391 | return false; |
| 1392 | |
| 1393 | } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) { |
| 1394 | return isLegalMUBUFAddressingMode(AM); |
| 1395 | } else if (AS == AMDGPUAS::LOCAL_ADDRESS || |
| 1396 | AS == AMDGPUAS::REGION_ADDRESS) { |
| 1397 | // Basic, single offset DS instructions allow a 16-bit unsigned immediate |
| 1398 | // field. |
| 1399 | // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have |
| 1400 | // an 8-bit dword offset but we don't know the alignment here. |
| 1401 | if (!isUInt<16>(AM.BaseOffs)) |
| 1402 | return false; |
| 1403 | |
| 1404 | if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg. |
| 1405 | return true; |
| 1406 | |
| 1407 | if (AM.Scale == 1 && AM.HasBaseReg) |
| 1408 | return true; |
| 1409 | |
| 1410 | return false; |
| 1411 | } else if (AS == AMDGPUAS::FLAT_ADDRESS || |
| 1412 | AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) { |
| 1413 | // For an unknown address space, this usually means that this is for some |
| 1414 | // reason being used for pure arithmetic, and not based on some addressing |
| 1415 | // computation. We don't have instructions that compute pointers with any |
| 1416 | // addressing modes, so treat them as having no offset like flat |
| 1417 | // instructions. |
| 1418 | return isLegalFlatAddressingMode(AM); |
| 1419 | } |
| 1420 | |
| 1421 | // Assume a user alias of global for unknown address spaces. |
| 1422 | return isLegalGlobalAddressingMode(AM); |
| 1423 | } |
| 1424 | |
| 1425 | bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT, |
| 1426 | const SelectionDAG &DAG) const { |
| 1427 | if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) { |
| 1428 | return (MemVT.getSizeInBits() <= 4 * 32); |
| 1429 | } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) { |
| 1430 | unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize(); |
| 1431 | return (MemVT.getSizeInBits() <= MaxPrivateBits); |
| 1432 | } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) { |
| 1433 | return (MemVT.getSizeInBits() <= 2 * 32); |
| 1434 | } |
| 1435 | return true; |
| 1436 | } |
| 1437 | |
| 1438 | bool SITargetLowering::allowsMisalignedMemoryAccessesImpl( |
| 1439 | unsigned Size, unsigned AddrSpace, Align Alignment, |
| 1440 | MachineMemOperand::Flags Flags, bool *IsFast) const { |
| 1441 | if (IsFast) |
| 1442 | *IsFast = false; |
| 1443 | |
| 1444 | if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS || |
| 1445 | AddrSpace == AMDGPUAS::REGION_ADDRESS) { |
| 1446 | // Check if alignment requirements for ds_read/write instructions are |
| 1447 | // disabled. |
| 1448 | if (Subtarget->hasUnalignedDSAccessEnabled() && |
| 1449 | !Subtarget->hasLDSMisalignedBug()) { |
| 1450 | if (IsFast) |
| 1451 | *IsFast = Alignment != Align(2); |
| 1452 | return true; |
| 1453 | } |
| 1454 | |
| 1455 | // Either, the alignment requirements are "enabled", or there is an |
| 1456 | // unaligned LDS access related hardware bug though alignment requirements |
| 1457 | // are "disabled". In either case, we need to check for proper alignment |
| 1458 | // requirements. |
| 1459 | // |
| 1460 | if (Size == 64) { |
| 1461 | // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we |
| 1462 | // can do a 4 byte aligned, 8 byte access in a single operation using |
| 1463 | // ds_read2/write2_b32 with adjacent offsets. |
| 1464 | bool AlignedBy4 = Alignment >= Align(4); |
| 1465 | if (IsFast) |
| 1466 | *IsFast = AlignedBy4; |
| 1467 | |
| 1468 | return AlignedBy4; |
| 1469 | } |
| 1470 | if (Size == 96) { |
| 1471 | // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on |
| 1472 | // gfx8 and older. |
| 1473 | bool AlignedBy16 = Alignment >= Align(16); |
| 1474 | if (IsFast) |
| 1475 | *IsFast = AlignedBy16; |
| 1476 | |
| 1477 | return AlignedBy16; |
| 1478 | } |
| 1479 | if (Size == 128) { |
| 1480 | // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on |
| 1481 | // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a |
| 1482 | // single operation using ds_read2/write2_b64. |
| 1483 | bool AlignedBy8 = Alignment >= Align(8); |
| 1484 | if (IsFast) |
| 1485 | *IsFast = AlignedBy8; |
| 1486 | |
| 1487 | return AlignedBy8; |
| 1488 | } |
| 1489 | } |
| 1490 | |
| 1491 | if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) { |
| 1492 | bool AlignedBy4 = Alignment >= Align(4); |
| 1493 | if (IsFast) |
| 1494 | *IsFast = AlignedBy4; |
| 1495 | |
| 1496 | return AlignedBy4 || |
| 1497 | Subtarget->enableFlatScratch() || |
| 1498 | Subtarget->hasUnalignedScratchAccess(); |
| 1499 | } |
| 1500 | |
| 1501 | // FIXME: We have to be conservative here and assume that flat operations |
| 1502 | // will access scratch. If we had access to the IR function, then we |
| 1503 | // could determine if any private memory was used in the function. |
| 1504 | if (AddrSpace == AMDGPUAS::FLAT_ADDRESS && |
| 1505 | !Subtarget->hasUnalignedScratchAccess()) { |
| 1506 | bool AlignedBy4 = Alignment >= Align(4); |
| 1507 | if (IsFast) |
| 1508 | *IsFast = AlignedBy4; |
| 1509 | |
| 1510 | return AlignedBy4; |
| 1511 | } |
| 1512 | |
| 1513 | if (Subtarget->hasUnalignedBufferAccessEnabled() && |
| 1514 | !(AddrSpace == AMDGPUAS::LOCAL_ADDRESS || |
| 1515 | AddrSpace == AMDGPUAS::REGION_ADDRESS)) { |
| 1516 | // If we have an uniform constant load, it still requires using a slow |
| 1517 | // buffer instruction if unaligned. |
| 1518 | if (IsFast) { |
| 1519 | // Accesses can really be issued as 1-byte aligned or 4-byte aligned, so |
| 1520 | // 2-byte alignment is worse than 1 unless doing a 2-byte accesss. |
| 1521 | *IsFast = (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS || |
| 1522 | AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ? |
| 1523 | Alignment >= Align(4) : Alignment != Align(2); |
| 1524 | } |
| 1525 | |
| 1526 | return true; |
| 1527 | } |
| 1528 | |
| 1529 | // Smaller than dword value must be aligned. |
| 1530 | if (Size < 32) |
| 1531 | return false; |
| 1532 | |
| 1533 | // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the |
| 1534 | // byte-address are ignored, thus forcing Dword alignment. |
| 1535 | // This applies to private, global, and constant memory. |
| 1536 | if (IsFast) |
| 1537 | *IsFast = true; |
| 1538 | |
| 1539 | return Size >= 32 && Alignment >= Align(4); |
| 1540 | } |
| 1541 | |
| 1542 | bool SITargetLowering::allowsMisalignedMemoryAccesses( |
| 1543 | EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, |
| 1544 | bool *IsFast) const { |
| 1545 | if (IsFast) |
| 1546 | *IsFast = false; |
| 1547 | |
| 1548 | // TODO: I think v3i32 should allow unaligned accesses on CI with DS_READ_B96, |
| 1549 | // which isn't a simple VT. |
| 1550 | // Until MVT is extended to handle this, simply check for the size and |
| 1551 | // rely on the condition below: allow accesses if the size is a multiple of 4. |
| 1552 | if (VT == MVT::Other || (VT != MVT::Other && VT.getSizeInBits() > 1024 && |
| 1553 | VT.getStoreSize() > 16)) { |
| 1554 | return false; |
| 1555 | } |
| 1556 | |
| 1557 | return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace, |
| 1558 | Alignment, Flags, IsFast); |
| 1559 | } |
| 1560 | |
| 1561 | EVT SITargetLowering::getOptimalMemOpType( |
| 1562 | const MemOp &Op, const AttributeList &FuncAttributes) const { |
| 1563 | // FIXME: Should account for address space here. |
| 1564 | |
| 1565 | // The default fallback uses the private pointer size as a guess for a type to |
| 1566 | // use. Make sure we switch these to 64-bit accesses. |
| 1567 | |
| 1568 | if (Op.size() >= 16 && |
| 1569 | Op.isDstAligned(Align(4))) // XXX: Should only do for global |
| 1570 | return MVT::v4i32; |
| 1571 | |
| 1572 | if (Op.size() >= 8 && Op.isDstAligned(Align(4))) |
| 1573 | return MVT::v2i32; |
| 1574 | |
| 1575 | // Use the default. |
| 1576 | return MVT::Other; |
| 1577 | } |
| 1578 | |
| 1579 | bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const { |
| 1580 | const MemSDNode *MemNode = cast<MemSDNode>(N); |
| 1581 | const Value *Ptr = MemNode->getMemOperand()->getValue(); |
| 1582 | const Instruction *I = dyn_cast_or_null<Instruction>(Ptr); |
| 1583 | return I && I->getMetadata("amdgpu.noclobber"); |
| 1584 | } |
| 1585 | |
| 1586 | bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) { |
| 1587 | return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS || |
| 1588 | AS == AMDGPUAS::PRIVATE_ADDRESS; |
| 1589 | } |
| 1590 | |
| 1591 | bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS, |
| 1592 | unsigned DestAS) const { |
| 1593 | // Flat -> private/local is a simple truncate. |
| 1594 | // Flat -> global is no-op |
| 1595 | if (SrcAS == AMDGPUAS::FLAT_ADDRESS) |
| 1596 | return true; |
| 1597 | |
| 1598 | const GCNTargetMachine &TM = |
| 1599 | static_cast<const GCNTargetMachine &>(getTargetMachine()); |
| 1600 | return TM.isNoopAddrSpaceCast(SrcAS, DestAS); |
| 1601 | } |
| 1602 | |
| 1603 | bool SITargetLowering::isMemOpUniform(const SDNode *N) const { |
| 1604 | const MemSDNode *MemNode = cast<MemSDNode>(N); |
| 1605 | |
| 1606 | return AMDGPUInstrInfo::isUniformMMO(MemNode->getMemOperand()); |
| 1607 | } |
| 1608 | |
| 1609 | TargetLoweringBase::LegalizeTypeAction |
| 1610 | SITargetLowering::getPreferredVectorAction(MVT VT) const { |
| 1611 | if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 && |
| 1612 | VT.getScalarType().bitsLE(MVT::i16)) |
| 1613 | return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector; |
| 1614 | return TargetLoweringBase::getPreferredVectorAction(VT); |
| 1615 | } |
| 1616 | |
| 1617 | bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, |
| 1618 | Type *Ty) const { |
| 1619 | // FIXME: Could be smarter if called for vector constants. |
| 1620 | return true; |
| 1621 | } |
| 1622 | |
| 1623 | bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const { |
| 1624 | if (Subtarget->has16BitInsts() && VT == MVT::i16) { |
| 1625 | switch (Op) { |
| 1626 | case ISD::LOAD: |
| 1627 | case ISD::STORE: |
| 1628 | |
| 1629 | // These operations are done with 32-bit instructions anyway. |
| 1630 | case ISD::AND: |
| 1631 | case ISD::OR: |
| 1632 | case ISD::XOR: |
| 1633 | case ISD::SELECT: |
| 1634 | // TODO: Extensions? |
| 1635 | return true; |
| 1636 | default: |
| 1637 | return false; |
| 1638 | } |
| 1639 | } |
| 1640 | |
| 1641 | // SimplifySetCC uses this function to determine whether or not it should |
| 1642 | // create setcc with i1 operands. We don't have instructions for i1 setcc. |
| 1643 | if (VT == MVT::i1 && Op == ISD::SETCC) |
| 1644 | return false; |
| 1645 | |
| 1646 | return TargetLowering::isTypeDesirableForOp(Op, VT); |
| 1647 | } |
| 1648 | |
| 1649 | SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG, |
| 1650 | const SDLoc &SL, |
| 1651 | SDValue Chain, |
| 1652 | uint64_t Offset) const { |
| 1653 | const DataLayout &DL = DAG.getDataLayout(); |
| 1654 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1655 | const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 1656 | |
| 1657 | const ArgDescriptor *InputPtrReg; |
| 1658 | const TargetRegisterClass *RC; |
| 1659 | LLT ArgTy; |
| 1660 | |
| 1661 | std::tie(InputPtrReg, RC, ArgTy) = |
| 1662 | Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR); |
| 1663 | |
| 1664 | MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); |
| 1665 | MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); |
| 1666 | SDValue BasePtr = DAG.getCopyFromReg(Chain, SL, |
| 1667 | MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT); |
| 1668 | |
| 1669 | return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Offset)); |
| 1670 | } |
| 1671 | |
| 1672 | SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG, |
| 1673 | const SDLoc &SL) const { |
| 1674 | uint64_t Offset = getImplicitParameterOffset(DAG.getMachineFunction(), |
| 1675 | FIRST_IMPLICIT); |
| 1676 | return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset); |
| 1677 | } |
| 1678 | |
| 1679 | SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT, |
| 1680 | const SDLoc &SL, SDValue Val, |
| 1681 | bool Signed, |
| 1682 | const ISD::InputArg *Arg) const { |
| 1683 | // First, if it is a widened vector, narrow it. |
| 1684 | if (VT.isVector() && |
| 1685 | VT.getVectorNumElements() != MemVT.getVectorNumElements()) { |
| 1686 | EVT NarrowedVT = |
| 1687 | EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), |
| 1688 | VT.getVectorNumElements()); |
| 1689 | Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val, |
| 1690 | DAG.getConstant(0, SL, MVT::i32)); |
| 1691 | } |
| 1692 | |
| 1693 | // Then convert the vector elements or scalar value. |
| 1694 | if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && |
| 1695 | VT.bitsLT(MemVT)) { |
| 1696 | unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext; |
| 1697 | Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT)); |
| 1698 | } |
| 1699 | |
| 1700 | if (MemVT.isFloatingPoint()) |
| 1701 | Val = getFPExtOrFPRound(DAG, Val, SL, VT); |
| 1702 | else if (Signed) |
| 1703 | Val = DAG.getSExtOrTrunc(Val, SL, VT); |
| 1704 | else |
| 1705 | Val = DAG.getZExtOrTrunc(Val, SL, VT); |
| 1706 | |
| 1707 | return Val; |
| 1708 | } |
| 1709 | |
| 1710 | SDValue SITargetLowering::lowerKernargMemParameter( |
| 1711 | SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain, |
| 1712 | uint64_t Offset, Align Alignment, bool Signed, |
| 1713 | const ISD::InputArg *Arg) const { |
| 1714 | MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); |
| 1715 | |
| 1716 | // Try to avoid using an extload by loading earlier than the argument address, |
| 1717 | // and extracting the relevant bits. The load should hopefully be merged with |
| 1718 | // the previous argument. |
| 1719 | if (MemVT.getStoreSize() < 4 && Alignment < 4) { |
| 1720 | // TODO: Handle align < 4 and size >= 4 (can happen with packed structs). |
| 1721 | int64_t AlignDownOffset = alignDown(Offset, 4); |
| 1722 | int64_t OffsetDiff = Offset - AlignDownOffset; |
| 1723 | |
| 1724 | EVT IntVT = MemVT.changeTypeToInteger(); |
| 1725 | |
| 1726 | // TODO: If we passed in the base kernel offset we could have a better |
| 1727 | // alignment than 4, but we don't really need it. |
| 1728 | SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset); |
| 1729 | SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4), |
| 1730 | MachineMemOperand::MODereferenceable | |
| 1731 | MachineMemOperand::MOInvariant); |
| 1732 | |
| 1733 | SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); |
| 1734 | SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); |
| 1735 | |
| 1736 | SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract); |
| 1737 | ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal); |
| 1738 | ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg); |
| 1739 | |
| 1740 | |
| 1741 | return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL); |
| 1742 | } |
| 1743 | |
| 1744 | SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset); |
| 1745 | SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment, |
| 1746 | MachineMemOperand::MODereferenceable | |
| 1747 | MachineMemOperand::MOInvariant); |
| 1748 | |
| 1749 | SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg); |
| 1750 | return DAG.getMergeValues({ Val, Load.getValue(1) }, SL); |
| 1751 | } |
| 1752 | |
| 1753 | SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA, |
| 1754 | const SDLoc &SL, SDValue Chain, |
| 1755 | const ISD::InputArg &Arg) const { |
| 1756 | MachineFunction &MF = DAG.getMachineFunction(); |
| 1757 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 1758 | |
| 1759 | if (Arg.Flags.isByVal()) { |
| 1760 | unsigned Size = Arg.Flags.getByValSize(); |
| 1761 | int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false); |
| 1762 | return DAG.getFrameIndex(FrameIdx, MVT::i32); |
| 1763 | } |
| 1764 | |
| 1765 | unsigned ArgOffset = VA.getLocMemOffset(); |
| 1766 | unsigned ArgSize = VA.getValVT().getStoreSize(); |
| 1767 | |
| 1768 | int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true); |
| 1769 | |
| 1770 | // Create load nodes to retrieve arguments from the stack. |
| 1771 | SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); |
| 1772 | SDValue ArgValue; |
| 1773 | |
| 1774 | // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT) |
| 1775 | ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; |
| 1776 | MVT MemVT = VA.getValVT(); |
| 1777 | |
| 1778 | switch (VA.getLocInfo()) { |
| 1779 | default: |
| 1780 | break; |
| 1781 | case CCValAssign::BCvt: |
| 1782 | MemVT = VA.getLocVT(); |
| 1783 | break; |
| 1784 | case CCValAssign::SExt: |
| 1785 | ExtType = ISD::SEXTLOAD; |
| 1786 | break; |
| 1787 | case CCValAssign::ZExt: |
| 1788 | ExtType = ISD::ZEXTLOAD; |
| 1789 | break; |
| 1790 | case CCValAssign::AExt: |
| 1791 | ExtType = ISD::EXTLOAD; |
| 1792 | break; |
| 1793 | } |
| 1794 | |
| 1795 | ArgValue = DAG.getExtLoad( |
| 1796 | ExtType, SL, VA.getLocVT(), Chain, FIN, |
| 1797 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), |
| 1798 | MemVT); |
| 1799 | return ArgValue; |
| 1800 | } |
| 1801 | |
| 1802 | SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG, |
| 1803 | const SIMachineFunctionInfo &MFI, |
| 1804 | EVT VT, |
| 1805 | AMDGPUFunctionArgInfo::PreloadedValue PVID) const { |
| 1806 | const ArgDescriptor *Reg; |
| 1807 | const TargetRegisterClass *RC; |
| 1808 | LLT Ty; |
| 1809 | |
| 1810 | std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID); |
| 1811 | return CreateLiveInRegister(DAG, RC, Reg->getRegister(), VT); |
| 1812 | } |
| 1813 | |
| 1814 | static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits, |
| 1815 | CallingConv::ID CallConv, |
| 1816 | ArrayRef<ISD::InputArg> Ins, BitVector &Skipped, |
| 1817 | FunctionType *FType, |
| 1818 | SIMachineFunctionInfo *Info) { |
| 1819 | for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) { |
| 1820 | const ISD::InputArg *Arg = &Ins[I]; |
| 1821 | |
| 1822 | assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&((void)0) |
| 1823 | "vector type argument should have been split")((void)0); |
| 1824 | |
| 1825 | // First check if it's a PS input addr. |
| 1826 | if (CallConv == CallingConv::AMDGPU_PS && |
| 1827 | !Arg->Flags.isInReg() && PSInputNum <= 15) { |
| 1828 | bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum); |
| 1829 | |
| 1830 | // Inconveniently only the first part of the split is marked as isSplit, |
| 1831 | // so skip to the end. We only want to increment PSInputNum once for the |
| 1832 | // entire split argument. |
| 1833 | if (Arg->Flags.isSplit()) { |
| 1834 | while (!Arg->Flags.isSplitEnd()) { |
| 1835 | assert((!Arg->VT.isVector() ||((void)0) |
| 1836 | Arg->VT.getScalarSizeInBits() == 16) &&((void)0) |
| 1837 | "unexpected vector split in ps argument type")((void)0); |
| 1838 | if (!SkipArg) |
| 1839 | Splits.push_back(*Arg); |
| 1840 | Arg = &Ins[++I]; |
| 1841 | } |
| 1842 | } |
| 1843 | |
| 1844 | if (SkipArg) { |
| 1845 | // We can safely skip PS inputs. |
| 1846 | Skipped.set(Arg->getOrigArgIndex()); |
| 1847 | ++PSInputNum; |
| 1848 | continue; |
| 1849 | } |
| 1850 | |
| 1851 | Info->markPSInputAllocated(PSInputNum); |
| 1852 | if (Arg->Used) |
| 1853 | Info->markPSInputEnabled(PSInputNum); |
| 1854 | |
| 1855 | ++PSInputNum; |
| 1856 | } |
| 1857 | |
| 1858 | Splits.push_back(*Arg); |
| 1859 | } |
| 1860 | } |
| 1861 | |
| 1862 | // Allocate special inputs passed in VGPRs. |
| 1863 | void SITargetLowering::allocateSpecialEntryInputVGPRs(CCState &CCInfo, |
| 1864 | MachineFunction &MF, |
| 1865 | const SIRegisterInfo &TRI, |
| 1866 | SIMachineFunctionInfo &Info) const { |
| 1867 | const LLT S32 = LLT::scalar(32); |
| 1868 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 1869 | |
| 1870 | if (Info.hasWorkItemIDX()) { |
| 1871 | Register Reg = AMDGPU::VGPR0; |
| 1872 | MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32); |
| 1873 | |
| 1874 | CCInfo.AllocateReg(Reg); |
| 1875 | unsigned Mask = (Subtarget->hasPackedTID() && |
| 1876 | Info.hasWorkItemIDY()) ? 0x3ff : ~0u; |
| 1877 | Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask)); |
| 1878 | } |
| 1879 | |
| 1880 | if (Info.hasWorkItemIDY()) { |
| 1881 | assert(Info.hasWorkItemIDX())((void)0); |
| 1882 | if (Subtarget->hasPackedTID()) { |
| 1883 | Info.setWorkItemIDY(ArgDescriptor::createRegister(AMDGPU::VGPR0, |
| 1884 | 0x3ff << 10)); |
| 1885 | } else { |
| 1886 | unsigned Reg = AMDGPU::VGPR1; |
| 1887 | MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32); |
| 1888 | |
| 1889 | CCInfo.AllocateReg(Reg); |
| 1890 | Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg)); |
| 1891 | } |
| 1892 | } |
| 1893 | |
| 1894 | if (Info.hasWorkItemIDZ()) { |
| 1895 | assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY())((void)0); |
| 1896 | if (Subtarget->hasPackedTID()) { |
| 1897 | Info.setWorkItemIDZ(ArgDescriptor::createRegister(AMDGPU::VGPR0, |
| 1898 | 0x3ff << 20)); |
| 1899 | } else { |
| 1900 | unsigned Reg = AMDGPU::VGPR2; |
| 1901 | MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32); |
| 1902 | |
| 1903 | CCInfo.AllocateReg(Reg); |
| 1904 | Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg)); |
| 1905 | } |
| 1906 | } |
| 1907 | } |
| 1908 | |
| 1909 | // Try to allocate a VGPR at the end of the argument list, or if no argument |
| 1910 | // VGPRs are left allocating a stack slot. |
| 1911 | // If \p Mask is is given it indicates bitfield position in the register. |
| 1912 | // If \p Arg is given use it with new ]p Mask instead of allocating new. |
| 1913 | static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u, |
| 1914 | ArgDescriptor Arg = ArgDescriptor()) { |
| 1915 | if (Arg.isSet()) |
| 1916 | return ArgDescriptor::createArg(Arg, Mask); |
| 1917 | |
| 1918 | ArrayRef<MCPhysReg> ArgVGPRs |
| 1919 | = makeArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32); |
| 1920 | unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs); |
| 1921 | if (RegIdx == ArgVGPRs.size()) { |
| 1922 | // Spill to stack required. |
| 1923 | int64_t Offset = CCInfo.AllocateStack(4, Align(4)); |
| 1924 | |
| 1925 | return ArgDescriptor::createStack(Offset, Mask); |
| 1926 | } |
| 1927 | |
| 1928 | unsigned Reg = ArgVGPRs[RegIdx]; |
| 1929 | Reg = CCInfo.AllocateReg(Reg); |
| 1930 | assert(Reg != AMDGPU::NoRegister)((void)0); |
| 1931 | |
| 1932 | MachineFunction &MF = CCInfo.getMachineFunction(); |
| 1933 | Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass); |
| 1934 | MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32)); |
| 1935 | return ArgDescriptor::createRegister(Reg, Mask); |
| 1936 | } |
| 1937 | |
| 1938 | static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo, |
| 1939 | const TargetRegisterClass *RC, |
| 1940 | unsigned NumArgRegs) { |
| 1941 | ArrayRef<MCPhysReg> ArgSGPRs = makeArrayRef(RC->begin(), 32); |
| 1942 | unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs); |
| 1943 | if (RegIdx == ArgSGPRs.size()) |
| 1944 | report_fatal_error("ran out of SGPRs for arguments"); |
| 1945 | |
| 1946 | unsigned Reg = ArgSGPRs[RegIdx]; |
| 1947 | Reg = CCInfo.AllocateReg(Reg); |
| 1948 | assert(Reg != AMDGPU::NoRegister)((void)0); |
| 1949 | |
| 1950 | MachineFunction &MF = CCInfo.getMachineFunction(); |
| 1951 | MF.addLiveIn(Reg, RC); |
| 1952 | return ArgDescriptor::createRegister(Reg); |
| 1953 | } |
| 1954 | |
| 1955 | // If this has a fixed position, we still should allocate the register in the |
| 1956 | // CCInfo state. Technically we could get away with this for values passed |
| 1957 | // outside of the normal argument range. |
| 1958 | static void allocateFixedSGPRInputImpl(CCState &CCInfo, |
| 1959 | const TargetRegisterClass *RC, |
| 1960 | MCRegister Reg) { |
| 1961 | Reg = CCInfo.AllocateReg(Reg); |
| 1962 | assert(Reg != AMDGPU::NoRegister)((void)0); |
| 1963 | MachineFunction &MF = CCInfo.getMachineFunction(); |
| 1964 | MF.addLiveIn(Reg, RC); |
| 1965 | } |
| 1966 | |
| 1967 | static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) { |
| 1968 | if (Arg) { |
| 1969 | allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, |
| 1970 | Arg.getRegister()); |
| 1971 | } else |
| 1972 | Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32); |
| 1973 | } |
| 1974 | |
| 1975 | static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) { |
| 1976 | if (Arg) { |
| 1977 | allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, |
| 1978 | Arg.getRegister()); |
| 1979 | } else |
| 1980 | Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16); |
| 1981 | } |
| 1982 | |
| 1983 | /// Allocate implicit function VGPR arguments at the end of allocated user |
| 1984 | /// arguments. |
| 1985 | void SITargetLowering::allocateSpecialInputVGPRs( |
| 1986 | CCState &CCInfo, MachineFunction &MF, |
| 1987 | const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const { |
| 1988 | const unsigned Mask = 0x3ff; |
| 1989 | ArgDescriptor Arg; |
| 1990 | |
| 1991 | if (Info.hasWorkItemIDX()) { |
| 1992 | Arg = allocateVGPR32Input(CCInfo, Mask); |
| 1993 | Info.setWorkItemIDX(Arg); |
| 1994 | } |
| 1995 | |
| 1996 | if (Info.hasWorkItemIDY()) { |
| 1997 | Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg); |
| 1998 | Info.setWorkItemIDY(Arg); |
| 1999 | } |
| 2000 | |
| 2001 | if (Info.hasWorkItemIDZ()) |
| 2002 | Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg)); |
| 2003 | } |
| 2004 | |
| 2005 | /// Allocate implicit function VGPR arguments in fixed registers. |
| 2006 | void SITargetLowering::allocateSpecialInputVGPRsFixed( |
| 2007 | CCState &CCInfo, MachineFunction &MF, |
| 2008 | const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const { |
| 2009 | Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31); |
| 2010 | if (!Reg) |
| 2011 | report_fatal_error("failed to allocated VGPR for implicit arguments"); |
| 2012 | |
| 2013 | const unsigned Mask = 0x3ff; |
| 2014 | Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask)); |
| 2015 | Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask << 10)); |
| 2016 | Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask << 20)); |
| 2017 | } |
| 2018 | |
| 2019 | void SITargetLowering::allocateSpecialInputSGPRs( |
| 2020 | CCState &CCInfo, |
| 2021 | MachineFunction &MF, |
| 2022 | const SIRegisterInfo &TRI, |
| 2023 | SIMachineFunctionInfo &Info) const { |
| 2024 | auto &ArgInfo = Info.getArgInfo(); |
| 2025 | |
| 2026 | // TODO: Unify handling with private memory pointers. |
| 2027 | |
| 2028 | if (Info.hasDispatchPtr()) |
| 2029 | allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr); |
| 2030 | |
| 2031 | if (Info.hasQueuePtr()) |
| 2032 | allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr); |
| 2033 | |
| 2034 | // Implicit arg ptr takes the place of the kernarg segment pointer. This is a |
| 2035 | // constant offset from the kernarg segment. |
| 2036 | if (Info.hasImplicitArgPtr()) |
| 2037 | allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr); |
| 2038 | |
| 2039 | if (Info.hasDispatchID()) |
| 2040 | allocateSGPR64Input(CCInfo, ArgInfo.DispatchID); |
| 2041 | |
| 2042 | // flat_scratch_init is not applicable for non-kernel functions. |
| 2043 | |
| 2044 | if (Info.hasWorkGroupIDX()) |
| 2045 | allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX); |
| 2046 | |
| 2047 | if (Info.hasWorkGroupIDY()) |
| 2048 | allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY); |
| 2049 | |
| 2050 | if (Info.hasWorkGroupIDZ()) |
| 2051 | allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ); |
| 2052 | } |
| 2053 | |
| 2054 | // Allocate special inputs passed in user SGPRs. |
| 2055 | void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo, |
| 2056 | MachineFunction &MF, |
| 2057 | const SIRegisterInfo &TRI, |
| 2058 | SIMachineFunctionInfo &Info) const { |
| 2059 | if (Info.hasImplicitBufferPtr()) { |
| 2060 | Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI); |
| 2061 | MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass); |
| 2062 | CCInfo.AllocateReg(ImplicitBufferPtrReg); |
| 2063 | } |
| 2064 | |
| 2065 | // FIXME: How should these inputs interact with inreg / custom SGPR inputs? |
| 2066 | if (Info.hasPrivateSegmentBuffer()) { |
| 2067 | Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI); |
| 2068 | MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass); |
| 2069 | CCInfo.AllocateReg(PrivateSegmentBufferReg); |
| 2070 | } |
| 2071 | |
| 2072 | if (Info.hasDispatchPtr()) { |
| 2073 | Register DispatchPtrReg = Info.addDispatchPtr(TRI); |
| 2074 | MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass); |
| 2075 | CCInfo.AllocateReg(DispatchPtrReg); |
| 2076 | } |
| 2077 | |
| 2078 | if (Info.hasQueuePtr()) { |
| 2079 | Register QueuePtrReg = Info.addQueuePtr(TRI); |
| 2080 | MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass); |
| 2081 | CCInfo.AllocateReg(QueuePtrReg); |
| 2082 | } |
| 2083 | |
| 2084 | if (Info.hasKernargSegmentPtr()) { |
| 2085 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 2086 | Register InputPtrReg = Info.addKernargSegmentPtr(TRI); |
| 2087 | CCInfo.AllocateReg(InputPtrReg); |
| 2088 | |
| 2089 | Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass); |
| 2090 | MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)); |
| 2091 | } |
| 2092 | |
| 2093 | if (Info.hasDispatchID()) { |
| 2094 | Register DispatchIDReg = Info.addDispatchID(TRI); |
| 2095 | MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass); |
| 2096 | CCInfo.AllocateReg(DispatchIDReg); |
| 2097 | } |
| 2098 | |
| 2099 | if (Info.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) { |
| 2100 | Register FlatScratchInitReg = Info.addFlatScratchInit(TRI); |
| 2101 | MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass); |
| 2102 | CCInfo.AllocateReg(FlatScratchInitReg); |
| 2103 | } |
| 2104 | |
| 2105 | // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read |
| 2106 | // these from the dispatch pointer. |
| 2107 | } |
| 2108 | |
| 2109 | // Allocate special input registers that are initialized per-wave. |
| 2110 | void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, |
| 2111 | MachineFunction &MF, |
| 2112 | SIMachineFunctionInfo &Info, |
| 2113 | CallingConv::ID CallConv, |
| 2114 | bool IsShader) const { |
| 2115 | if (Info.hasWorkGroupIDX()) { |
| 2116 | Register Reg = Info.addWorkGroupIDX(); |
| 2117 | MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); |
| 2118 | CCInfo.AllocateReg(Reg); |
| 2119 | } |
| 2120 | |
| 2121 | if (Info.hasWorkGroupIDY()) { |
| 2122 | Register Reg = Info.addWorkGroupIDY(); |
| 2123 | MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); |
| 2124 | CCInfo.AllocateReg(Reg); |
| 2125 | } |
| 2126 | |
| 2127 | if (Info.hasWorkGroupIDZ()) { |
| 2128 | Register Reg = Info.addWorkGroupIDZ(); |
| 2129 | MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); |
| 2130 | CCInfo.AllocateReg(Reg); |
| 2131 | } |
| 2132 | |
| 2133 | if (Info.hasWorkGroupInfo()) { |
| 2134 | Register Reg = Info.addWorkGroupInfo(); |
| 2135 | MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass); |
| 2136 | CCInfo.AllocateReg(Reg); |
| 2137 | } |
| 2138 | |
| 2139 | if (Info.hasPrivateSegmentWaveByteOffset()) { |
| 2140 | // Scratch wave offset passed in system SGPR. |
| 2141 | unsigned PrivateSegmentWaveByteOffsetReg; |
| 2142 | |
| 2143 | if (IsShader) { |
| 2144 | PrivateSegmentWaveByteOffsetReg = |
| 2145 | Info.getPrivateSegmentWaveByteOffsetSystemSGPR(); |
| 2146 | |
| 2147 | // This is true if the scratch wave byte offset doesn't have a fixed |
| 2148 | // location. |
| 2149 | if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) { |
| 2150 | PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo); |
| 2151 | Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg); |
| 2152 | } |
| 2153 | } else |
| 2154 | PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset(); |
| 2155 | |
| 2156 | MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass); |
| 2157 | CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg); |
| 2158 | } |
| 2159 | } |
| 2160 | |
| 2161 | static void reservePrivateMemoryRegs(const TargetMachine &TM, |
| 2162 | MachineFunction &MF, |
| 2163 | const SIRegisterInfo &TRI, |
| 2164 | SIMachineFunctionInfo &Info) { |
| 2165 | // Now that we've figured out where the scratch register inputs are, see if |
| 2166 | // should reserve the arguments and use them directly. |
| 2167 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 2168 | bool HasStackObjects = MFI.hasStackObjects(); |
| 2169 | const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); |
| 2170 | |
| 2171 | // Record that we know we have non-spill stack objects so we don't need to |
| 2172 | // check all stack objects later. |
| 2173 | if (HasStackObjects) |
| 2174 | Info.setHasNonSpillStackObjects(true); |
| 2175 | |
| 2176 | // Everything live out of a block is spilled with fast regalloc, so it's |
| 2177 | // almost certain that spilling will be required. |
| 2178 | if (TM.getOptLevel() == CodeGenOpt::None) |
| 2179 | HasStackObjects = true; |
| 2180 | |
| 2181 | // For now assume stack access is needed in any callee functions, so we need |
| 2182 | // the scratch registers to pass in. |
| 2183 | bool RequiresStackAccess = HasStackObjects || MFI.hasCalls(); |
| 2184 | |
| 2185 | if (!ST.enableFlatScratch()) { |
| 2186 | if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) { |
| 2187 | // If we have stack objects, we unquestionably need the private buffer |
| 2188 | // resource. For the Code Object V2 ABI, this will be the first 4 user |
| 2189 | // SGPR inputs. We can reserve those and use them directly. |
| 2190 | |
| 2191 | Register PrivateSegmentBufferReg = |
| 2192 | Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER); |
| 2193 | Info.setScratchRSrcReg(PrivateSegmentBufferReg); |
| 2194 | } else { |
| 2195 | unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF); |
| 2196 | // We tentatively reserve the last registers (skipping the last registers |
| 2197 | // which may contain VCC, FLAT_SCR, and XNACK). After register allocation, |
| 2198 | // we'll replace these with the ones immediately after those which were |
| 2199 | // really allocated. In the prologue copies will be inserted from the |
| 2200 | // argument to these reserved registers. |
| 2201 | |
| 2202 | // Without HSA, relocations are used for the scratch pointer and the |
| 2203 | // buffer resource setup is always inserted in the prologue. Scratch wave |
| 2204 | // offset is still in an input SGPR. |
| 2205 | Info.setScratchRSrcReg(ReservedBufferReg); |
| 2206 | } |
| 2207 | } |
| 2208 | |
| 2209 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 2210 | |
| 2211 | // For entry functions we have to set up the stack pointer if we use it, |
| 2212 | // whereas non-entry functions get this "for free". This means there is no |
| 2213 | // intrinsic advantage to using S32 over S34 in cases where we do not have |
| 2214 | // calls but do need a frame pointer (i.e. if we are requested to have one |
| 2215 | // because frame pointer elimination is disabled). To keep things simple we |
| 2216 | // only ever use S32 as the call ABI stack pointer, and so using it does not |
| 2217 | // imply we need a separate frame pointer. |
| 2218 | // |
| 2219 | // Try to use s32 as the SP, but move it if it would interfere with input |
| 2220 | // arguments. This won't work with calls though. |
| 2221 | // |
| 2222 | // FIXME: Move SP to avoid any possible inputs, or find a way to spill input |
| 2223 | // registers. |
| 2224 | if (!MRI.isLiveIn(AMDGPU::SGPR32)) { |
| 2225 | Info.setStackPtrOffsetReg(AMDGPU::SGPR32); |
| 2226 | } else { |
| 2227 | assert(AMDGPU::isShader(MF.getFunction().getCallingConv()))((void)0); |
| 2228 | |
| 2229 | if (MFI.hasCalls()) |
| 2230 | report_fatal_error("call in graphics shader with too many input SGPRs"); |
| 2231 | |
| 2232 | for (unsigned Reg : AMDGPU::SGPR_32RegClass) { |
| 2233 | if (!MRI.isLiveIn(Reg)) { |
| 2234 | Info.setStackPtrOffsetReg(Reg); |
| 2235 | break; |
| 2236 | } |
| 2237 | } |
| 2238 | |
| 2239 | if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG) |
| 2240 | report_fatal_error("failed to find register for SP"); |
| 2241 | } |
| 2242 | |
| 2243 | // hasFP should be accurate for entry functions even before the frame is |
| 2244 | // finalized, because it does not rely on the known stack size, only |
| 2245 | // properties like whether variable sized objects are present. |
| 2246 | if (ST.getFrameLowering()->hasFP(MF)) { |
| 2247 | Info.setFrameOffsetReg(AMDGPU::SGPR33); |
| 2248 | } |
| 2249 | } |
| 2250 | |
| 2251 | bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const { |
| 2252 | const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); |
| 2253 | return !Info->isEntryFunction(); |
| 2254 | } |
| 2255 | |
| 2256 | void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { |
| 2257 | |
| 2258 | } |
| 2259 | |
| 2260 | void SITargetLowering::insertCopiesSplitCSR( |
| 2261 | MachineBasicBlock *Entry, |
| 2262 | const SmallVectorImpl<MachineBasicBlock *> &Exits) const { |
| 2263 | const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); |
| 2264 | |
| 2265 | const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); |
| 2266 | if (!IStart) |
| 2267 | return; |
| 2268 | |
| 2269 | const TargetInstrInfo *TII = Subtarget->getInstrInfo(); |
| 2270 | MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); |
| 2271 | MachineBasicBlock::iterator MBBI = Entry->begin(); |
| 2272 | for (const MCPhysReg *I = IStart; *I; ++I) { |
| 2273 | const TargetRegisterClass *RC = nullptr; |
| 2274 | if (AMDGPU::SReg_64RegClass.contains(*I)) |
| 2275 | RC = &AMDGPU::SGPR_64RegClass; |
| 2276 | else if (AMDGPU::SReg_32RegClass.contains(*I)) |
| 2277 | RC = &AMDGPU::SGPR_32RegClass; |
| 2278 | else |
| 2279 | llvm_unreachable("Unexpected register class in CSRsViaCopy!")__builtin_unreachable(); |
| 2280 | |
| 2281 | Register NewVR = MRI->createVirtualRegister(RC); |
| 2282 | // Create copy from CSR to a virtual register. |
| 2283 | Entry->addLiveIn(*I); |
| 2284 | BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) |
| 2285 | .addReg(*I); |
| 2286 | |
| 2287 | // Insert the copy-back instructions right before the terminator. |
| 2288 | for (auto *Exit : Exits) |
| 2289 | BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), |
| 2290 | TII->get(TargetOpcode::COPY), *I) |
| 2291 | .addReg(NewVR); |
| 2292 | } |
| 2293 | } |
| 2294 | |
| 2295 | SDValue SITargetLowering::LowerFormalArguments( |
| 2296 | SDValue Chain, CallingConv::ID CallConv, bool isVarArg, |
| 2297 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, |
| 2298 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { |
| 2299 | const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); |
| 2300 | |
| 2301 | MachineFunction &MF = DAG.getMachineFunction(); |
| 2302 | const Function &Fn = MF.getFunction(); |
| 2303 | FunctionType *FType = MF.getFunction().getFunctionType(); |
| 2304 | SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 2305 | |
| 2306 | if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) { |
| 2307 | DiagnosticInfoUnsupported NoGraphicsHSA( |
| 2308 | Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()); |
| 2309 | DAG.getContext()->diagnose(NoGraphicsHSA); |
| 2310 | return DAG.getEntryNode(); |
| 2311 | } |
| 2312 | |
| 2313 | Info->allocateModuleLDSGlobal(Fn.getParent()); |
| 2314 | |
| 2315 | SmallVector<ISD::InputArg, 16> Splits; |
| 2316 | SmallVector<CCValAssign, 16> ArgLocs; |
| 2317 | BitVector Skipped(Ins.size()); |
| 2318 | CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, |
| 2319 | *DAG.getContext()); |
| 2320 | |
| 2321 | bool IsGraphics = AMDGPU::isGraphics(CallConv); |
| 2322 | bool IsKernel = AMDGPU::isKernel(CallConv); |
| 2323 | bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv); |
| 2324 | |
| 2325 | if (IsGraphics) { |
| 2326 | assert(!Info->hasDispatchPtr() && !Info->hasKernargSegmentPtr() &&((void)0) |
| 2327 | (!Info->hasFlatScratchInit() || Subtarget->enableFlatScratch()) &&((void)0) |
| 2328 | !Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&((void)0) |
| 2329 | !Info->hasWorkGroupIDZ() && !Info->hasWorkGroupInfo() &&((void)0) |
| 2330 | !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() &&((void)0) |
| 2331 | !Info->hasWorkItemIDZ())((void)0); |
| 2332 | } |
| 2333 | |
| 2334 | if (CallConv == CallingConv::AMDGPU_PS) { |
| 2335 | processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info); |
| 2336 | |
| 2337 | // At least one interpolation mode must be enabled or else the GPU will |
| 2338 | // hang. |
| 2339 | // |
| 2340 | // Check PSInputAddr instead of PSInputEnable. The idea is that if the user |
| 2341 | // set PSInputAddr, the user wants to enable some bits after the compilation |
| 2342 | // based on run-time states. Since we can't know what the final PSInputEna |
| 2343 | // will look like, so we shouldn't do anything here and the user should take |
| 2344 | // responsibility for the correct programming. |
| 2345 | // |
| 2346 | // Otherwise, the following restrictions apply: |
| 2347 | // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled. |
| 2348 | // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be |
| 2349 | // enabled too. |
| 2350 | if ((Info->getPSInputAddr() & 0x7F) == 0 || |
| 2351 | ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) { |
| 2352 | CCInfo.AllocateReg(AMDGPU::VGPR0); |
| 2353 | CCInfo.AllocateReg(AMDGPU::VGPR1); |
| 2354 | Info->markPSInputAllocated(0); |
| 2355 | Info->markPSInputEnabled(0); |
| 2356 | } |
| 2357 | if (Subtarget->isAmdPalOS()) { |
| 2358 | // For isAmdPalOS, the user does not enable some bits after compilation |
| 2359 | // based on run-time states; the register values being generated here are |
| 2360 | // the final ones set in hardware. Therefore we need to apply the |
| 2361 | // workaround to PSInputAddr and PSInputEnable together. (The case where |
| 2362 | // a bit is set in PSInputAddr but not PSInputEnable is where the |
| 2363 | // frontend set up an input arg for a particular interpolation mode, but |
| 2364 | // nothing uses that input arg. Really we should have an earlier pass |
| 2365 | // that removes such an arg.) |
| 2366 | unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable(); |
| 2367 | if ((PsInputBits & 0x7F) == 0 || |
| 2368 | ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1))) |
| 2369 | Info->markPSInputEnabled( |
| 2370 | countTrailingZeros(Info->getPSInputAddr(), ZB_Undefined)); |
| 2371 | } |
| 2372 | } else if (IsKernel) { |
| 2373 | assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX())((void)0); |
| 2374 | } else { |
| 2375 | Splits.append(Ins.begin(), Ins.end()); |
| 2376 | } |
| 2377 | |
| 2378 | if (IsEntryFunc) { |
| 2379 | allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info); |
| 2380 | allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info); |
| 2381 | } else { |
| 2382 | // For the fixed ABI, pass workitem IDs in the last argument register. |
| 2383 | if (AMDGPUTargetMachine::EnableFixedFunctionABI) |
| 2384 | allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); |
| 2385 | } |
| 2386 | |
| 2387 | if (IsKernel) { |
| 2388 | analyzeFormalArgumentsCompute(CCInfo, Ins); |
| 2389 | } else { |
| 2390 | CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg); |
| 2391 | CCInfo.AnalyzeFormalArguments(Splits, AssignFn); |
| 2392 | } |
| 2393 | |
| 2394 | SmallVector<SDValue, 16> Chains; |
| 2395 | |
| 2396 | // FIXME: This is the minimum kernel argument alignment. We should improve |
| 2397 | // this to the maximum alignment of the arguments. |
| 2398 | // |
| 2399 | // FIXME: Alignment of explicit arguments totally broken with non-0 explicit |
| 2400 | // kern arg offset. |
| 2401 | const Align KernelArgBaseAlign = Align(16); |
| 2402 | |
| 2403 | for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) { |
| 2404 | const ISD::InputArg &Arg = Ins[i]; |
| 2405 | if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) { |
| 2406 | InVals.push_back(DAG.getUNDEF(Arg.VT)); |
| 2407 | continue; |
| 2408 | } |
| 2409 | |
| 2410 | CCValAssign &VA = ArgLocs[ArgIdx++]; |
| 2411 | MVT VT = VA.getLocVT(); |
| 2412 | |
| 2413 | if (IsEntryFunc && VA.isMemLoc()) { |
| 2414 | VT = Ins[i].VT; |
| 2415 | EVT MemVT = VA.getLocVT(); |
| 2416 | |
| 2417 | const uint64_t Offset = VA.getLocMemOffset(); |
| 2418 | Align Alignment = commonAlignment(KernelArgBaseAlign, Offset); |
| 2419 | |
| 2420 | if (Arg.Flags.isByRef()) { |
| 2421 | SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset); |
| 2422 | |
| 2423 | const GCNTargetMachine &TM = |
| 2424 | static_cast<const GCNTargetMachine &>(getTargetMachine()); |
| 2425 | if (!TM.isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS, |
| 2426 | Arg.Flags.getPointerAddrSpace())) { |
| 2427 | Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS, |
| 2428 | Arg.Flags.getPointerAddrSpace()); |
| 2429 | } |
| 2430 | |
| 2431 | InVals.push_back(Ptr); |
| 2432 | continue; |
| 2433 | } |
| 2434 | |
| 2435 | SDValue Arg = lowerKernargMemParameter( |
| 2436 | DAG, VT, MemVT, DL, Chain, Offset, Alignment, Ins[i].Flags.isSExt(), &Ins[i]); |
| 2437 | Chains.push_back(Arg.getValue(1)); |
| 2438 | |
| 2439 | auto *ParamTy = |
| 2440 | dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex())); |
| 2441 | if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS && |
| 2442 | ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || |
| 2443 | ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) { |
| 2444 | // On SI local pointers are just offsets into LDS, so they are always |
| 2445 | // less than 16-bits. On CI and newer they could potentially be |
| 2446 | // real pointers, so we can't guarantee their size. |
| 2447 | Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg, |
| 2448 | DAG.getValueType(MVT::i16)); |
| 2449 | } |
| 2450 | |
| 2451 | InVals.push_back(Arg); |
| 2452 | continue; |
| 2453 | } else if (!IsEntryFunc && VA.isMemLoc()) { |
| 2454 | SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg); |
| 2455 | InVals.push_back(Val); |
| 2456 | if (!Arg.Flags.isByVal()) |
| 2457 | Chains.push_back(Val.getValue(1)); |
| 2458 | continue; |
| 2459 | } |
| 2460 | |
| 2461 | assert(VA.isRegLoc() && "Parameter must be in a register!")((void)0); |
| 2462 | |
| 2463 | Register Reg = VA.getLocReg(); |
| 2464 | const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); |
| 2465 | EVT ValVT = VA.getValVT(); |
| 2466 | |
| 2467 | Reg = MF.addLiveIn(Reg, RC); |
| 2468 | SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT); |
| 2469 | |
| 2470 | if (Arg.Flags.isSRet()) { |
| 2471 | // The return object should be reasonably addressable. |
| 2472 | |
| 2473 | // FIXME: This helps when the return is a real sret. If it is a |
| 2474 | // automatically inserted sret (i.e. CanLowerReturn returns false), an |
| 2475 | // extra copy is inserted in SelectionDAGBuilder which obscures this. |
| 2476 | unsigned NumBits |
| 2477 | = 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex(); |
| 2478 | Val = DAG.getNode(ISD::AssertZext, DL, VT, Val, |
| 2479 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits))); |
| 2480 | } |
| 2481 | |
| 2482 | // If this is an 8 or 16-bit value, it is really passed promoted |
| 2483 | // to 32 bits. Insert an assert[sz]ext to capture this, then |
| 2484 | // truncate to the right size. |
| 2485 | switch (VA.getLocInfo()) { |
| 2486 | case CCValAssign::Full: |
| 2487 | break; |
| 2488 | case CCValAssign::BCvt: |
| 2489 | Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val); |
| 2490 | break; |
| 2491 | case CCValAssign::SExt: |
| 2492 | Val = DAG.getNode(ISD::AssertSext, DL, VT, Val, |
| 2493 | DAG.getValueType(ValVT)); |
| 2494 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val); |
| 2495 | break; |
| 2496 | case CCValAssign::ZExt: |
| 2497 | Val = DAG.getNode(ISD::AssertZext, DL, VT, Val, |
| 2498 | DAG.getValueType(ValVT)); |
| 2499 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val); |
| 2500 | break; |
| 2501 | case CCValAssign::AExt: |
| 2502 | Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val); |
| 2503 | break; |
| 2504 | default: |
| 2505 | llvm_unreachable("Unknown loc info!")__builtin_unreachable(); |
| 2506 | } |
| 2507 | |
| 2508 | InVals.push_back(Val); |
| 2509 | } |
| 2510 | |
| 2511 | if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) { |
| 2512 | // Special inputs come after user arguments. |
| 2513 | allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info); |
| 2514 | } |
| 2515 | |
| 2516 | // Start adding system SGPRs. |
| 2517 | if (IsEntryFunc) { |
| 2518 | allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics); |
| 2519 | } else { |
| 2520 | CCInfo.AllocateReg(Info->getScratchRSrcReg()); |
| 2521 | allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); |
| 2522 | } |
| 2523 | |
| 2524 | auto &ArgUsageInfo = |
| 2525 | DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>(); |
| 2526 | ArgUsageInfo.setFuncArgInfo(Fn, Info->getArgInfo()); |
| 2527 | |
| 2528 | unsigned StackArgSize = CCInfo.getNextStackOffset(); |
| 2529 | Info->setBytesInStackArgArea(StackArgSize); |
| 2530 | |
| 2531 | return Chains.empty() ? Chain : |
| 2532 | DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); |
| 2533 | } |
| 2534 | |
| 2535 | // TODO: If return values can't fit in registers, we should return as many as |
| 2536 | // possible in registers before passing on stack. |
| 2537 | bool SITargetLowering::CanLowerReturn( |
| 2538 | CallingConv::ID CallConv, |
| 2539 | MachineFunction &MF, bool IsVarArg, |
| 2540 | const SmallVectorImpl<ISD::OutputArg> &Outs, |
| 2541 | LLVMContext &Context) const { |
| 2542 | // Replacing returns with sret/stack usage doesn't make sense for shaders. |
| 2543 | // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn |
| 2544 | // for shaders. Vector types should be explicitly handled by CC. |
| 2545 | if (AMDGPU::isEntryFunctionCC(CallConv)) |
| 2546 | return true; |
| 2547 | |
| 2548 | SmallVector<CCValAssign, 16> RVLocs; |
| 2549 | CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); |
| 2550 | return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg)); |
| 2551 | } |
| 2552 | |
| 2553 | SDValue |
| 2554 | SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, |
| 2555 | bool isVarArg, |
| 2556 | const SmallVectorImpl<ISD::OutputArg> &Outs, |
| 2557 | const SmallVectorImpl<SDValue> &OutVals, |
| 2558 | const SDLoc &DL, SelectionDAG &DAG) const { |
| 2559 | MachineFunction &MF = DAG.getMachineFunction(); |
| 2560 | SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 2561 | |
| 2562 | if (AMDGPU::isKernel(CallConv)) { |
| 2563 | return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs, |
| 2564 | OutVals, DL, DAG); |
| 2565 | } |
| 2566 | |
| 2567 | bool IsShader = AMDGPU::isShader(CallConv); |
| 2568 | |
| 2569 | Info->setIfReturnsVoid(Outs.empty()); |
| 2570 | bool IsWaveEnd = Info->returnsVoid() && IsShader; |
| 2571 | |
| 2572 | // CCValAssign - represent the assignment of the return value to a location. |
| 2573 | SmallVector<CCValAssign, 48> RVLocs; |
| 2574 | SmallVector<ISD::OutputArg, 48> Splits; |
| 2575 | |
| 2576 | // CCState - Info about the registers and stack slots. |
| 2577 | CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, |
| 2578 | *DAG.getContext()); |
| 2579 | |
| 2580 | // Analyze outgoing return values. |
| 2581 | CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg)); |
| 2582 | |
| 2583 | SDValue Flag; |
| 2584 | SmallVector<SDValue, 48> RetOps; |
| 2585 | RetOps.push_back(Chain); // Operand #0 = Chain (updated below) |
| 2586 | |
| 2587 | // Add return address for callable functions. |
| 2588 | if (!Info->isEntryFunction()) { |
| 2589 | const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); |
| 2590 | SDValue ReturnAddrReg = CreateLiveInRegister( |
| 2591 | DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); |
| 2592 | |
| 2593 | SDValue ReturnAddrVirtualReg = DAG.getRegister( |
| 2594 | MF.getRegInfo().createVirtualRegister(&AMDGPU::CCR_SGPR_64RegClass), |
| 2595 | MVT::i64); |
| 2596 | Chain = |
| 2597 | DAG.getCopyToReg(Chain, DL, ReturnAddrVirtualReg, ReturnAddrReg, Flag); |
| 2598 | Flag = Chain.getValue(1); |
| 2599 | RetOps.push_back(ReturnAddrVirtualReg); |
| 2600 | } |
| 2601 | |
| 2602 | // Copy the result values into the output registers. |
| 2603 | for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E; |
| 2604 | ++I, ++RealRVLocIdx) { |
| 2605 | CCValAssign &VA = RVLocs[I]; |
| 2606 | assert(VA.isRegLoc() && "Can only return in registers!")((void)0); |
| 2607 | // TODO: Partially return in registers if return values don't fit. |
| 2608 | SDValue Arg = OutVals[RealRVLocIdx]; |
| 2609 | |
| 2610 | // Copied from other backends. |
| 2611 | switch (VA.getLocInfo()) { |
| 2612 | case CCValAssign::Full: |
| 2613 | break; |
| 2614 | case CCValAssign::BCvt: |
| 2615 | Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); |
| 2616 | break; |
| 2617 | case CCValAssign::SExt: |
| 2618 | Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); |
| 2619 | break; |
| 2620 | case CCValAssign::ZExt: |
| 2621 | Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); |
| 2622 | break; |
| 2623 | case CCValAssign::AExt: |
| 2624 | Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); |
| 2625 | break; |
| 2626 | default: |
| 2627 | llvm_unreachable("Unknown loc info!")__builtin_unreachable(); |
| 2628 | } |
| 2629 | |
| 2630 | Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag); |
| 2631 | Flag = Chain.getValue(1); |
| 2632 | RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); |
| 2633 | } |
| 2634 | |
| 2635 | // FIXME: Does sret work properly? |
| 2636 | if (!Info->isEntryFunction()) { |
| 2637 | const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); |
| 2638 | const MCPhysReg *I = |
| 2639 | TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); |
| 2640 | if (I) { |
| 2641 | for (; *I; ++I) { |
| 2642 | if (AMDGPU::SReg_64RegClass.contains(*I)) |
| 2643 | RetOps.push_back(DAG.getRegister(*I, MVT::i64)); |
| 2644 | else if (AMDGPU::SReg_32RegClass.contains(*I)) |
| 2645 | RetOps.push_back(DAG.getRegister(*I, MVT::i32)); |
| 2646 | else |
| 2647 | llvm_unreachable("Unexpected register class in CSRsViaCopy!")__builtin_unreachable(); |
| 2648 | } |
| 2649 | } |
| 2650 | } |
| 2651 | |
| 2652 | // Update chain and glue. |
| 2653 | RetOps[0] = Chain; |
| 2654 | if (Flag.getNode()) |
| 2655 | RetOps.push_back(Flag); |
| 2656 | |
| 2657 | unsigned Opc = AMDGPUISD::ENDPGM; |
| 2658 | if (!IsWaveEnd) |
| 2659 | Opc = IsShader ? AMDGPUISD::RETURN_TO_EPILOG : AMDGPUISD::RET_FLAG; |
| 2660 | return DAG.getNode(Opc, DL, MVT::Other, RetOps); |
| 2661 | } |
| 2662 | |
| 2663 | SDValue SITargetLowering::LowerCallResult( |
| 2664 | SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg, |
| 2665 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, |
| 2666 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn, |
| 2667 | SDValue ThisVal) const { |
| 2668 | CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg); |
| 2669 | |
| 2670 | // Assign locations to each value returned by this call. |
| 2671 | SmallVector<CCValAssign, 16> RVLocs; |
| 2672 | CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, |
| 2673 | *DAG.getContext()); |
| 2674 | CCInfo.AnalyzeCallResult(Ins, RetCC); |
| 2675 | |
| 2676 | // Copy all of the result registers out of their specified physreg. |
| 2677 | for (unsigned i = 0; i != RVLocs.size(); ++i) { |
| 2678 | CCValAssign VA = RVLocs[i]; |
| 2679 | SDValue Val; |
| 2680 | |
| 2681 | if (VA.isRegLoc()) { |
| 2682 | Val = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag); |
| 2683 | Chain = Val.getValue(1); |
| 2684 | InFlag = Val.getValue(2); |
| 2685 | } else if (VA.isMemLoc()) { |
| 2686 | report_fatal_error("TODO: return values in memory"); |
| 2687 | } else |
| 2688 | llvm_unreachable("unknown argument location type")__builtin_unreachable(); |
| 2689 | |
| 2690 | switch (VA.getLocInfo()) { |
| 2691 | case CCValAssign::Full: |
| 2692 | break; |
| 2693 | case CCValAssign::BCvt: |
| 2694 | Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); |
| 2695 | break; |
| 2696 | case CCValAssign::ZExt: |
| 2697 | Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val, |
| 2698 | DAG.getValueType(VA.getValVT())); |
| 2699 | Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val); |
| 2700 | break; |
| 2701 | case CCValAssign::SExt: |
| 2702 | Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val, |
| 2703 | DAG.getValueType(VA.getValVT())); |
| 2704 | Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val); |
| 2705 | break; |
| 2706 | case CCValAssign::AExt: |
| 2707 | Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val); |
| 2708 | break; |
| 2709 | default: |
| 2710 | llvm_unreachable("Unknown loc info!")__builtin_unreachable(); |
| 2711 | } |
| 2712 | |
| 2713 | InVals.push_back(Val); |
| 2714 | } |
| 2715 | |
| 2716 | return Chain; |
| 2717 | } |
| 2718 | |
| 2719 | // Add code to pass special inputs required depending on used features separate |
| 2720 | // from the explicit user arguments present in the IR. |
| 2721 | void SITargetLowering::passSpecialInputs( |
| 2722 | CallLoweringInfo &CLI, |
| 2723 | CCState &CCInfo, |
| 2724 | const SIMachineFunctionInfo &Info, |
| 2725 | SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, |
| 2726 | SmallVectorImpl<SDValue> &MemOpChains, |
| 2727 | SDValue Chain) const { |
| 2728 | // If we don't have a call site, this was a call inserted by |
| 2729 | // legalization. These can never use special inputs. |
| 2730 | if (!CLI.CB) |
| 2731 | return; |
| 2732 | |
| 2733 | SelectionDAG &DAG = CLI.DAG; |
| 2734 | const SDLoc &DL = CLI.DL; |
| 2735 | |
| 2736 | const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); |
| 2737 | const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo(); |
| 2738 | |
| 2739 | const AMDGPUFunctionArgInfo *CalleeArgInfo |
| 2740 | = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo; |
| 2741 | if (const Function *CalleeFunc = CLI.CB->getCalledFunction()) { |
| 2742 | auto &ArgUsageInfo = |
| 2743 | DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>(); |
| 2744 | CalleeArgInfo = &ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc); |
| 2745 | } |
| 2746 | |
| 2747 | // TODO: Unify with private memory register handling. This is complicated by |
| 2748 | // the fact that at least in kernels, the input argument is not necessarily |
| 2749 | // in the same location as the input. |
| 2750 | AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = { |
| 2751 | AMDGPUFunctionArgInfo::DISPATCH_PTR, |
| 2752 | AMDGPUFunctionArgInfo::QUEUE_PTR, |
| 2753 | AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, |
| 2754 | AMDGPUFunctionArgInfo::DISPATCH_ID, |
| 2755 | AMDGPUFunctionArgInfo::WORKGROUP_ID_X, |
| 2756 | AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, |
| 2757 | AMDGPUFunctionArgInfo::WORKGROUP_ID_Z |
| 2758 | }; |
| 2759 | |
| 2760 | for (auto InputID : InputRegs) { |
| 2761 | const ArgDescriptor *OutgoingArg; |
| 2762 | const TargetRegisterClass *ArgRC; |
| 2763 | LLT ArgTy; |
| 2764 | |
| 2765 | std::tie(OutgoingArg, ArgRC, ArgTy) = |
| 2766 | CalleeArgInfo->getPreloadedValue(InputID); |
| 2767 | if (!OutgoingArg) |
| 2768 | continue; |
| 2769 | |
| 2770 | const ArgDescriptor *IncomingArg; |
| 2771 | const TargetRegisterClass *IncomingArgRC; |
| 2772 | LLT Ty; |
| 2773 | std::tie(IncomingArg, IncomingArgRC, Ty) = |
| 2774 | CallerArgInfo.getPreloadedValue(InputID); |
| 2775 | assert(IncomingArgRC == ArgRC)((void)0); |
| 2776 | |
| 2777 | // All special arguments are ints for now. |
| 2778 | EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32; |
| 2779 | SDValue InputReg; |
| 2780 | |
| 2781 | if (IncomingArg) { |
| 2782 | InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg); |
| 2783 | } else { |
| 2784 | // The implicit arg ptr is special because it doesn't have a corresponding |
| 2785 | // input for kernels, and is computed from the kernarg segment pointer. |
| 2786 | assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR)((void)0); |
| 2787 | InputReg = getImplicitArgPtr(DAG, DL); |
| 2788 | } |
| 2789 | |
| 2790 | if (OutgoingArg->isRegister()) { |
| 2791 | RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg); |
| 2792 | if (!CCInfo.AllocateReg(OutgoingArg->getRegister())) |
| 2793 | report_fatal_error("failed to allocate implicit input argument"); |
| 2794 | } else { |
| 2795 | unsigned SpecialArgOffset = |
| 2796 | CCInfo.AllocateStack(ArgVT.getStoreSize(), Align(4)); |
| 2797 | SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg, |
| 2798 | SpecialArgOffset); |
| 2799 | MemOpChains.push_back(ArgStore); |
| 2800 | } |
| 2801 | } |
| 2802 | |
| 2803 | // Pack workitem IDs into a single register or pass it as is if already |
| 2804 | // packed. |
| 2805 | const ArgDescriptor *OutgoingArg; |
| 2806 | const TargetRegisterClass *ArgRC; |
| 2807 | LLT Ty; |
| 2808 | |
| 2809 | std::tie(OutgoingArg, ArgRC, Ty) = |
| 2810 | CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X); |
| 2811 | if (!OutgoingArg) |
| 2812 | std::tie(OutgoingArg, ArgRC, Ty) = |
| 2813 | CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y); |
| 2814 | if (!OutgoingArg) |
| 2815 | std::tie(OutgoingArg, ArgRC, Ty) = |
| 2816 | CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z); |
| 2817 | if (!OutgoingArg) |
| 2818 | return; |
| 2819 | |
| 2820 | const ArgDescriptor *IncomingArgX = std::get<0>( |
| 2821 | CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X)); |
| 2822 | const ArgDescriptor *IncomingArgY = std::get<0>( |
| 2823 | CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y)); |
| 2824 | const ArgDescriptor *IncomingArgZ = std::get<0>( |
| 2825 | CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z)); |
| 2826 | |
| 2827 | SDValue InputReg; |
| 2828 | SDLoc SL; |
| 2829 | |
| 2830 | // If incoming ids are not packed we need to pack them. |
| 2831 | if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX) |
| 2832 | InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX); |
| 2833 | |
| 2834 | if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY) { |
| 2835 | SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY); |
| 2836 | Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y, |
| 2837 | DAG.getShiftAmountConstant(10, MVT::i32, SL)); |
| 2838 | InputReg = InputReg.getNode() ? |
| 2839 | DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y; |
| 2840 | } |
| 2841 | |
| 2842 | if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ) { |
| 2843 | SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ); |
| 2844 | Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z, |
| 2845 | DAG.getShiftAmountConstant(20, MVT::i32, SL)); |
| 2846 | InputReg = InputReg.getNode() ? |
| 2847 | DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z; |
| 2848 | } |
| 2849 | |
| 2850 | if (!InputReg.getNode()) { |
| 2851 | // Workitem ids are already packed, any of present incoming arguments |
| 2852 | // will carry all required fields. |
| 2853 | ArgDescriptor IncomingArg = ArgDescriptor::createArg( |
| 2854 | IncomingArgX ? *IncomingArgX : |
| 2855 | IncomingArgY ? *IncomingArgY : |
| 2856 | *IncomingArgZ, ~0u); |
| 2857 | InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg); |
| 2858 | } |
| 2859 | |
| 2860 | if (OutgoingArg->isRegister()) { |
| 2861 | RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg); |
| 2862 | CCInfo.AllocateReg(OutgoingArg->getRegister()); |
| 2863 | } else { |
| 2864 | unsigned SpecialArgOffset = CCInfo.AllocateStack(4, Align(4)); |
| 2865 | SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg, |
| 2866 | SpecialArgOffset); |
| 2867 | MemOpChains.push_back(ArgStore); |
| 2868 | } |
| 2869 | } |
| 2870 | |
| 2871 | static bool canGuaranteeTCO(CallingConv::ID CC) { |
| 2872 | return CC == CallingConv::Fast; |
| 2873 | } |
| 2874 | |
| 2875 | /// Return true if we might ever do TCO for calls with this calling convention. |
| 2876 | static bool mayTailCallThisCC(CallingConv::ID CC) { |
| 2877 | switch (CC) { |
| 2878 | case CallingConv::C: |
| 2879 | case CallingConv::AMDGPU_Gfx: |
| 2880 | return true; |
| 2881 | default: |
| 2882 | return canGuaranteeTCO(CC); |
| 2883 | } |
| 2884 | } |
| 2885 | |
| 2886 | bool SITargetLowering::isEligibleForTailCallOptimization( |
| 2887 | SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg, |
| 2888 | const SmallVectorImpl<ISD::OutputArg> &Outs, |
| 2889 | const SmallVectorImpl<SDValue> &OutVals, |
| 2890 | const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const { |
| 2891 | if (!mayTailCallThisCC(CalleeCC)) |
| 2892 | return false; |
| 2893 | |
| 2894 | // For a divergent call target, we need to do a waterfall loop over the |
| 2895 | // possible callees which precludes us from using a simple jump. |
| 2896 | if (Callee->isDivergent()) |
| 2897 | return false; |
| 2898 | |
| 2899 | MachineFunction &MF = DAG.getMachineFunction(); |
| 2900 | const Function &CallerF = MF.getFunction(); |
| 2901 | CallingConv::ID CallerCC = CallerF.getCallingConv(); |
| 2902 | const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); |
| 2903 | const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); |
| 2904 | |
| 2905 | // Kernels aren't callable, and don't have a live in return address so it |
| 2906 | // doesn't make sense to do a tail call with entry functions. |
| 2907 | if (!CallerPreserved) |
| 2908 | return false; |
| 2909 | |
| 2910 | bool CCMatch = CallerCC == CalleeCC; |
| 2911 | |
| 2912 | if (DAG.getTarget().Options.GuaranteedTailCallOpt) { |
| 2913 | if (canGuaranteeTCO(CalleeCC) && CCMatch) |
| 2914 | return true; |
| 2915 | return false; |
| 2916 | } |
| 2917 | |
| 2918 | // TODO: Can we handle var args? |
| 2919 | if (IsVarArg) |
| 2920 | return false; |
| 2921 | |
| 2922 | for (const Argument &Arg : CallerF.args()) { |
| 2923 | if (Arg.hasByValAttr()) |
| 2924 | return false; |
| 2925 | } |
| 2926 | |
| 2927 | LLVMContext &Ctx = *DAG.getContext(); |
| 2928 | |
| 2929 | // Check that the call results are passed in the same way. |
| 2930 | if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins, |
| 2931 | CCAssignFnForCall(CalleeCC, IsVarArg), |
| 2932 | CCAssignFnForCall(CallerCC, IsVarArg))) |
| 2933 | return false; |
| 2934 | |
| 2935 | // The callee has to preserve all registers the caller needs to preserve. |
| 2936 | if (!CCMatch) { |
| 2937 | const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); |
| 2938 | if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) |
| 2939 | return false; |
| 2940 | } |
| 2941 | |
| 2942 | // Nothing more to check if the callee is taking no arguments. |
| 2943 | if (Outs.empty()) |
| 2944 | return true; |
| 2945 | |
| 2946 | SmallVector<CCValAssign, 16> ArgLocs; |
| 2947 | CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx); |
| 2948 | |
| 2949 | CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg)); |
| 2950 | |
| 2951 | const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); |
| 2952 | // If the stack arguments for this call do not fit into our own save area then |
| 2953 | // the call cannot be made tail. |
| 2954 | // TODO: Is this really necessary? |
| 2955 | if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) |
| 2956 | return false; |
| 2957 | |
| 2958 | const MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 2959 | return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals); |
| 2960 | } |
| 2961 | |
| 2962 | bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { |
| 2963 | if (!CI->isTailCall()) |
| 2964 | return false; |
| 2965 | |
| 2966 | const Function *ParentFn = CI->getParent()->getParent(); |
| 2967 | if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv())) |
| 2968 | return false; |
| 2969 | return true; |
| 2970 | } |
| 2971 | |
| 2972 | // The wave scratch offset register is used as the global base pointer. |
| 2973 | SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, |
| 2974 | SmallVectorImpl<SDValue> &InVals) const { |
| 2975 | SelectionDAG &DAG = CLI.DAG; |
| 2976 | const SDLoc &DL = CLI.DL; |
| 2977 | SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; |
| 2978 | SmallVector<SDValue, 32> &OutVals = CLI.OutVals; |
| 2979 | SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; |
| 2980 | SDValue Chain = CLI.Chain; |
| 2981 | SDValue Callee = CLI.Callee; |
| 2982 | bool &IsTailCall = CLI.IsTailCall; |
| 2983 | CallingConv::ID CallConv = CLI.CallConv; |
| 2984 | bool IsVarArg = CLI.IsVarArg; |
| 2985 | bool IsSibCall = false; |
| 2986 | bool IsThisReturn = false; |
| 2987 | MachineFunction &MF = DAG.getMachineFunction(); |
| 2988 | |
| 2989 | if (Callee.isUndef() || isNullConstant(Callee)) { |
| 2990 | if (!CLI.IsTailCall) { |
| 2991 | for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I) |
| 2992 | InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT)); |
| 2993 | } |
| 2994 | |
| 2995 | return Chain; |
| 2996 | } |
| 2997 | |
| 2998 | if (IsVarArg) { |
| 2999 | return lowerUnhandledCall(CLI, InVals, |
| 3000 | "unsupported call to variadic function "); |
| 3001 | } |
| 3002 | |
| 3003 | if (!CLI.CB) |
| 3004 | report_fatal_error("unsupported libcall legalization"); |
| 3005 | |
| 3006 | if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) { |
| 3007 | return lowerUnhandledCall(CLI, InVals, |
| 3008 | "unsupported required tail call to function "); |
| 3009 | } |
| 3010 | |
| 3011 | if (AMDGPU::isShader(CallConv)) { |
| 3012 | // Note the issue is with the CC of the called function, not of the call |
| 3013 | // itself. |
| 3014 | return lowerUnhandledCall(CLI, InVals, |
| 3015 | "unsupported call to a shader function "); |
| 3016 | } |
| 3017 | |
| 3018 | if (AMDGPU::isShader(MF.getFunction().getCallingConv()) && |
| 3019 | CallConv != CallingConv::AMDGPU_Gfx) { |
| 3020 | // Only allow calls with specific calling conventions. |
| 3021 | return lowerUnhandledCall(CLI, InVals, |
| 3022 | "unsupported calling convention for call from " |
| 3023 | "graphics shader of function "); |
| 3024 | } |
| 3025 | |
| 3026 | if (IsTailCall) { |
| 3027 | IsTailCall = isEligibleForTailCallOptimization( |
| 3028 | Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG); |
| 3029 | if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) { |
| 3030 | report_fatal_error("failed to perform tail call elimination on a call " |
| 3031 | "site marked musttail"); |
| 3032 | } |
| 3033 | |
| 3034 | bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt; |
| 3035 | |
| 3036 | // A sibling call is one where we're under the usual C ABI and not planning |
| 3037 | // to change that but can still do a tail call: |
| 3038 | if (!TailCallOpt && IsTailCall) |
| 3039 | IsSibCall = true; |
| 3040 | |
| 3041 | if (IsTailCall) |
| 3042 | ++NumTailCalls; |
| 3043 | } |
| 3044 | |
| 3045 | const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 3046 | SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; |
| 3047 | SmallVector<SDValue, 8> MemOpChains; |
| 3048 | |
| 3049 | // Analyze operands of the call, assigning locations to each operand. |
| 3050 | SmallVector<CCValAssign, 16> ArgLocs; |
| 3051 | CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); |
| 3052 | CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg); |
| 3053 | |
| 3054 | if (AMDGPUTargetMachine::EnableFixedFunctionABI && |
| 3055 | CallConv != CallingConv::AMDGPU_Gfx) { |
| 3056 | // With a fixed ABI, allocate fixed registers before user arguments. |
| 3057 | passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain); |
| 3058 | } |
| 3059 | |
| 3060 | CCInfo.AnalyzeCallOperands(Outs, AssignFn); |
| 3061 | |
| 3062 | // Get a count of how many bytes are to be pushed on the stack. |
| 3063 | unsigned NumBytes = CCInfo.getNextStackOffset(); |
| 3064 | |
| 3065 | if (IsSibCall) { |
| 3066 | // Since we're not changing the ABI to make this a tail call, the memory |
| 3067 | // operands are already available in the caller's incoming argument space. |
| 3068 | NumBytes = 0; |
| 3069 | } |
| 3070 | |
| 3071 | // FPDiff is the byte offset of the call's argument area from the callee's. |
| 3072 | // Stores to callee stack arguments will be placed in FixedStackSlots offset |
| 3073 | // by this amount for a tail call. In a sibling call it must be 0 because the |
| 3074 | // caller will deallocate the entire stack and the callee still expects its |
| 3075 | // arguments to begin at SP+0. Completely unused for non-tail calls. |
| 3076 | int32_t FPDiff = 0; |
| 3077 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 3078 | |
| 3079 | // Adjust the stack pointer for the new arguments... |
| 3080 | // These operations are automatically eliminated by the prolog/epilog pass |
| 3081 | if (!IsSibCall) { |
| 3082 | Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL); |
| 3083 | |
| 3084 | if (!Subtarget->enableFlatScratch()) { |
| 3085 | SmallVector<SDValue, 4> CopyFromChains; |
| 3086 | |
| 3087 | // In the HSA case, this should be an identity copy. |
| 3088 | SDValue ScratchRSrcReg |
| 3089 | = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32); |
| 3090 | RegsToPass.emplace_back(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg); |
| 3091 | CopyFromChains.push_back(ScratchRSrcReg.getValue(1)); |
| 3092 | Chain = DAG.getTokenFactor(DL, CopyFromChains); |
| 3093 | } |
| 3094 | } |
| 3095 | |
| 3096 | MVT PtrVT = MVT::i32; |
| 3097 | |
| 3098 | // Walk the register/memloc assignments, inserting copies/loads. |
| 3099 | for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { |
| 3100 | CCValAssign &VA = ArgLocs[i]; |
| 3101 | SDValue Arg = OutVals[i]; |
| 3102 | |
| 3103 | // Promote the value if needed. |
| 3104 | switch (VA.getLocInfo()) { |
| 3105 | case CCValAssign::Full: |
| 3106 | break; |
| 3107 | case CCValAssign::BCvt: |
| 3108 | Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); |
| 3109 | break; |
| 3110 | case CCValAssign::ZExt: |
| 3111 | Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); |
| 3112 | break; |
| 3113 | case CCValAssign::SExt: |
| 3114 | Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); |
| 3115 | break; |
| 3116 | case CCValAssign::AExt: |
| 3117 | Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); |
| 3118 | break; |
| 3119 | case CCValAssign::FPExt: |
| 3120 | Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg); |
| 3121 | break; |
| 3122 | default: |
| 3123 | llvm_unreachable("Unknown loc info!")__builtin_unreachable(); |
| 3124 | } |
| 3125 | |
| 3126 | if (VA.isRegLoc()) { |
| 3127 | RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); |
| 3128 | } else { |
| 3129 | assert(VA.isMemLoc())((void)0); |
| 3130 | |
| 3131 | SDValue DstAddr; |
| 3132 | MachinePointerInfo DstInfo; |
| 3133 | |
| 3134 | unsigned LocMemOffset = VA.getLocMemOffset(); |
| 3135 | int32_t Offset = LocMemOffset; |
| 3136 | |
| 3137 | SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT); |
| 3138 | MaybeAlign Alignment; |
| 3139 | |
| 3140 | if (IsTailCall) { |
| 3141 | ISD::ArgFlagsTy Flags = Outs[i].Flags; |
| 3142 | unsigned OpSize = Flags.isByVal() ? |
| 3143 | Flags.getByValSize() : VA.getValVT().getStoreSize(); |
| 3144 | |
| 3145 | // FIXME: We can have better than the minimum byval required alignment. |
| 3146 | Alignment = |
| 3147 | Flags.isByVal() |
| 3148 | ? Flags.getNonZeroByValAlign() |
| 3149 | : commonAlignment(Subtarget->getStackAlignment(), Offset); |
| 3150 | |
| 3151 | Offset = Offset + FPDiff; |
| 3152 | int FI = MFI.CreateFixedObject(OpSize, Offset, true); |
| 3153 | |
| 3154 | DstAddr = DAG.getFrameIndex(FI, PtrVT); |
| 3155 | DstInfo = MachinePointerInfo::getFixedStack(MF, FI); |
| 3156 | |
| 3157 | // Make sure any stack arguments overlapping with where we're storing |
| 3158 | // are loaded before this eventual operation. Otherwise they'll be |
| 3159 | // clobbered. |
| 3160 | |
| 3161 | // FIXME: Why is this really necessary? This seems to just result in a |
| 3162 | // lot of code to copy the stack and write them back to the same |
| 3163 | // locations, which are supposed to be immutable? |
| 3164 | Chain = addTokenForArgument(Chain, DAG, MFI, FI); |
| 3165 | } else { |
| 3166 | // Stores to the argument stack area are relative to the stack pointer. |
| 3167 | SDValue SP = DAG.getCopyFromReg(Chain, DL, Info->getStackPtrOffsetReg(), |
| 3168 | MVT::i32); |
| 3169 | DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff); |
| 3170 | DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset); |
| 3171 | Alignment = |
| 3172 | commonAlignment(Subtarget->getStackAlignment(), LocMemOffset); |
| 3173 | } |
| 3174 | |
| 3175 | if (Outs[i].Flags.isByVal()) { |
| 3176 | SDValue SizeNode = |
| 3177 | DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32); |
| 3178 | SDValue Cpy = |
| 3179 | DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode, |
| 3180 | Outs[i].Flags.getNonZeroByValAlign(), |
| 3181 | /*isVol = */ false, /*AlwaysInline = */ true, |
| 3182 | /*isTailCall = */ false, DstInfo, |
| 3183 | MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS)); |
| 3184 | |
| 3185 | MemOpChains.push_back(Cpy); |
| 3186 | } else { |
| 3187 | SDValue Store = |
| 3188 | DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Alignment); |
| 3189 | MemOpChains.push_back(Store); |
| 3190 | } |
| 3191 | } |
| 3192 | } |
| 3193 | |
| 3194 | if (!AMDGPUTargetMachine::EnableFixedFunctionABI && |
| 3195 | CallConv != CallingConv::AMDGPU_Gfx) { |
| 3196 | // Copy special input registers after user input arguments. |
| 3197 | passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain); |
| 3198 | } |
| 3199 | |
| 3200 | if (!MemOpChains.empty()) |
| 3201 | Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); |
| 3202 | |
| 3203 | // Build a sequence of copy-to-reg nodes chained together with token chain |
| 3204 | // and flag operands which copy the outgoing args into the appropriate regs. |
| 3205 | SDValue InFlag; |
| 3206 | for (auto &RegToPass : RegsToPass) { |
| 3207 | Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first, |
| 3208 | RegToPass.second, InFlag); |
| 3209 | InFlag = Chain.getValue(1); |
| 3210 | } |
| 3211 | |
| 3212 | |
| 3213 | SDValue PhysReturnAddrReg; |
| 3214 | if (IsTailCall) { |
| 3215 | // Since the return is being combined with the call, we need to pass on the |
| 3216 | // return address. |
| 3217 | |
| 3218 | const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); |
| 3219 | SDValue ReturnAddrReg = CreateLiveInRegister( |
| 3220 | DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); |
| 3221 | |
| 3222 | PhysReturnAddrReg = DAG.getRegister(TRI->getReturnAddressReg(MF), |
| 3223 | MVT::i64); |
| 3224 | Chain = DAG.getCopyToReg(Chain, DL, PhysReturnAddrReg, ReturnAddrReg, InFlag); |
| 3225 | InFlag = Chain.getValue(1); |
| 3226 | } |
| 3227 | |
| 3228 | // We don't usually want to end the call-sequence here because we would tidy |
| 3229 | // the frame up *after* the call, however in the ABI-changing tail-call case |
| 3230 | // we've carefully laid out the parameters so that when sp is reset they'll be |
| 3231 | // in the correct location. |
| 3232 | if (IsTailCall && !IsSibCall) { |
| 3233 | Chain = DAG.getCALLSEQ_END(Chain, |
| 3234 | DAG.getTargetConstant(NumBytes, DL, MVT::i32), |
| 3235 | DAG.getTargetConstant(0, DL, MVT::i32), |
| 3236 | InFlag, DL); |
| 3237 | InFlag = Chain.getValue(1); |
| 3238 | } |
| 3239 | |
| 3240 | std::vector<SDValue> Ops; |
| 3241 | Ops.push_back(Chain); |
| 3242 | Ops.push_back(Callee); |
| 3243 | // Add a redundant copy of the callee global which will not be legalized, as |
| 3244 | // we need direct access to the callee later. |
| 3245 | if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Callee)) { |
| 3246 | const GlobalValue *GV = GSD->getGlobal(); |
| 3247 | Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64)); |
| 3248 | } else { |
| 3249 | Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); |
| 3250 | } |
| 3251 | |
| 3252 | if (IsTailCall) { |
| 3253 | // Each tail call may have to adjust the stack by a different amount, so |
| 3254 | // this information must travel along with the operation for eventual |
| 3255 | // consumption by emitEpilogue. |
| 3256 | Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); |
| 3257 | |
| 3258 | Ops.push_back(PhysReturnAddrReg); |
| 3259 | } |
| 3260 | |
| 3261 | // Add argument registers to the end of the list so that they are known live |
| 3262 | // into the call. |
| 3263 | for (auto &RegToPass : RegsToPass) { |
| 3264 | Ops.push_back(DAG.getRegister(RegToPass.first, |
| 3265 | RegToPass.second.getValueType())); |
| 3266 | } |
| 3267 | |
| 3268 | // Add a register mask operand representing the call-preserved registers. |
| 3269 | |
| 3270 | auto *TRI = static_cast<const SIRegisterInfo*>(Subtarget->getRegisterInfo()); |
| 3271 | const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv); |
| 3272 | assert(Mask && "Missing call preserved mask for calling convention")((void)0); |
| 3273 | Ops.push_back(DAG.getRegisterMask(Mask)); |
| 3274 | |
| 3275 | if (InFlag.getNode()) |
| 3276 | Ops.push_back(InFlag); |
| 3277 | |
| 3278 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); |
| 3279 | |
| 3280 | // If we're doing a tall call, use a TC_RETURN here rather than an |
| 3281 | // actual call instruction. |
| 3282 | if (IsTailCall) { |
| 3283 | MFI.setHasTailCall(); |
| 3284 | return DAG.getNode(AMDGPUISD::TC_RETURN, DL, NodeTys, Ops); |
| 3285 | } |
| 3286 | |
| 3287 | // Returns a chain and a flag for retval copy to use. |
| 3288 | SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops); |
| 3289 | Chain = Call.getValue(0); |
| 3290 | InFlag = Call.getValue(1); |
| 3291 | |
| 3292 | uint64_t CalleePopBytes = NumBytes; |
| 3293 | Chain = DAG.getCALLSEQ_END(Chain, DAG.getTargetConstant(0, DL, MVT::i32), |
| 3294 | DAG.getTargetConstant(CalleePopBytes, DL, MVT::i32), |
| 3295 | InFlag, DL); |
| 3296 | if (!Ins.empty()) |
| 3297 | InFlag = Chain.getValue(1); |
| 3298 | |
| 3299 | // Handle result values, copying them out of physregs into vregs that we |
| 3300 | // return. |
| 3301 | return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG, |
| 3302 | InVals, IsThisReturn, |
| 3303 | IsThisReturn ? OutVals[0] : SDValue()); |
| 3304 | } |
| 3305 | |
| 3306 | // This is identical to the default implementation in ExpandDYNAMIC_STACKALLOC, |
| 3307 | // except for applying the wave size scale to the increment amount. |
| 3308 | SDValue SITargetLowering::lowerDYNAMIC_STACKALLOCImpl( |
| 3309 | SDValue Op, SelectionDAG &DAG) const { |
| 3310 | const MachineFunction &MF = DAG.getMachineFunction(); |
| 3311 | const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 3312 | |
| 3313 | SDLoc dl(Op); |
| 3314 | EVT VT = Op.getValueType(); |
| 3315 | SDValue Tmp1 = Op; |
| 3316 | SDValue Tmp2 = Op.getValue(1); |
| 3317 | SDValue Tmp3 = Op.getOperand(2); |
| 3318 | SDValue Chain = Tmp1.getOperand(0); |
| 3319 | |
| 3320 | Register SPReg = Info->getStackPtrOffsetReg(); |
| 3321 | |
| 3322 | // Chain the dynamic stack allocation so that it doesn't modify the stack |
| 3323 | // pointer when other instructions are using the stack. |
| 3324 | Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); |
| 3325 | |
| 3326 | SDValue Size = Tmp2.getOperand(1); |
| 3327 | SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); |
| 3328 | Chain = SP.getValue(1); |
| 3329 | MaybeAlign Alignment = cast<ConstantSDNode>(Tmp3)->getMaybeAlignValue(); |
| 3330 | const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); |
| 3331 | const TargetFrameLowering *TFL = ST.getFrameLowering(); |
| 3332 | unsigned Opc = |
| 3333 | TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ? |
| 3334 | ISD::ADD : ISD::SUB; |
| 3335 | |
| 3336 | SDValue ScaledSize = DAG.getNode( |
| 3337 | ISD::SHL, dl, VT, Size, |
| 3338 | DAG.getConstant(ST.getWavefrontSizeLog2(), dl, MVT::i32)); |
| 3339 | |
| 3340 | Align StackAlign = TFL->getStackAlign(); |
| 3341 | Tmp1 = DAG.getNode(Opc, dl, VT, SP, ScaledSize); // Value |
| 3342 | if (Alignment && *Alignment > StackAlign) { |
| 3343 | Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, |
| 3344 | DAG.getConstant(-(uint64_t)Alignment->value() |
| 3345 | << ST.getWavefrontSizeLog2(), |
| 3346 | dl, VT)); |
| 3347 | } |
| 3348 | |
| 3349 | Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain |
| 3350 | Tmp2 = DAG.getCALLSEQ_END( |
| 3351 | Chain, DAG.getIntPtrConstant(0, dl, true), |
| 3352 | DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); |
| 3353 | |
| 3354 | return DAG.getMergeValues({Tmp1, Tmp2}, dl); |
| 3355 | } |
| 3356 | |
| 3357 | SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, |
| 3358 | SelectionDAG &DAG) const { |
| 3359 | // We only handle constant sizes here to allow non-entry block, static sized |
| 3360 | // allocas. A truly dynamic value is more difficult to support because we |
| 3361 | // don't know if the size value is uniform or not. If the size isn't uniform, |
| 3362 | // we would need to do a wave reduction to get the maximum size to know how |
| 3363 | // much to increment the uniform stack pointer. |
| 3364 | SDValue Size = Op.getOperand(1); |
| 3365 | if (isa<ConstantSDNode>(Size)) |
| 3366 | return lowerDYNAMIC_STACKALLOCImpl(Op, DAG); // Use "generic" expansion. |
| 3367 | |
| 3368 | return AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(Op, DAG); |
| 3369 | } |
| 3370 | |
| 3371 | Register SITargetLowering::getRegisterByName(const char* RegName, LLT VT, |
| 3372 | const MachineFunction &MF) const { |
| 3373 | Register Reg = StringSwitch<Register>(RegName) |
| 3374 | .Case("m0", AMDGPU::M0) |
| 3375 | .Case("exec", AMDGPU::EXEC) |
| 3376 | .Case("exec_lo", AMDGPU::EXEC_LO) |
| 3377 | .Case("exec_hi", AMDGPU::EXEC_HI) |
| 3378 | .Case("flat_scratch", AMDGPU::FLAT_SCR) |
| 3379 | .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO) |
| 3380 | .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI) |
| 3381 | .Default(Register()); |
| 3382 | |
| 3383 | if (Reg == AMDGPU::NoRegister) { |
| 3384 | report_fatal_error(Twine("invalid register name \"" |
| 3385 | + StringRef(RegName) + "\".")); |
| 3386 | |
| 3387 | } |
| 3388 | |
| 3389 | if (!Subtarget->hasFlatScrRegister() && |
| 3390 | Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) { |
| 3391 | report_fatal_error(Twine("invalid register \"" |
| 3392 | + StringRef(RegName) + "\" for subtarget.")); |
| 3393 | } |
| 3394 | |
| 3395 | switch (Reg) { |
| 3396 | case AMDGPU::M0: |
| 3397 | case AMDGPU::EXEC_LO: |
| 3398 | case AMDGPU::EXEC_HI: |
| 3399 | case AMDGPU::FLAT_SCR_LO: |
| 3400 | case AMDGPU::FLAT_SCR_HI: |
| 3401 | if (VT.getSizeInBits() == 32) |
| 3402 | return Reg; |
| 3403 | break; |
| 3404 | case AMDGPU::EXEC: |
| 3405 | case AMDGPU::FLAT_SCR: |
| 3406 | if (VT.getSizeInBits() == 64) |
| 3407 | return Reg; |
| 3408 | break; |
| 3409 | default: |
| 3410 | llvm_unreachable("missing register type checking")__builtin_unreachable(); |
| 3411 | } |
| 3412 | |
| 3413 | report_fatal_error(Twine("invalid type for register \"" |
| 3414 | + StringRef(RegName) + "\".")); |
| 3415 | } |
| 3416 | |
| 3417 | // If kill is not the last instruction, split the block so kill is always a |
| 3418 | // proper terminator. |
| 3419 | MachineBasicBlock * |
| 3420 | SITargetLowering::splitKillBlock(MachineInstr &MI, |
| 3421 | MachineBasicBlock *BB) const { |
| 3422 | MachineBasicBlock *SplitBB = BB->splitAt(MI, false /*UpdateLiveIns*/); |
| 3423 | const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); |
| 3424 | MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode())); |
| 3425 | return SplitBB; |
| 3426 | } |
| 3427 | |
| 3428 | // Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true, |
| 3429 | // \p MI will be the only instruction in the loop body block. Otherwise, it will |
| 3430 | // be the first instruction in the remainder block. |
| 3431 | // |
| 3432 | /// \returns { LoopBody, Remainder } |
| 3433 | static std::pair<MachineBasicBlock *, MachineBasicBlock *> |
| 3434 | splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) { |
| 3435 | MachineFunction *MF = MBB.getParent(); |
| 3436 | MachineBasicBlock::iterator I(&MI); |
| 3437 | |
| 3438 | // To insert the loop we need to split the block. Move everything after this |
| 3439 | // point to a new block, and insert a new empty block between the two. |
| 3440 | MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock(); |
| 3441 | MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock(); |
| 3442 | MachineFunction::iterator MBBI(MBB); |
| 3443 | ++MBBI; |
| 3444 | |
| 3445 | MF->insert(MBBI, LoopBB); |
| 3446 | MF->insert(MBBI, RemainderBB); |
| 3447 | |
| 3448 | LoopBB->addSuccessor(LoopBB); |
| 3449 | LoopBB->addSuccessor(RemainderBB); |
| 3450 | |
| 3451 | // Move the rest of the block into a new block. |
| 3452 | RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); |
| 3453 | |
| 3454 | if (InstInLoop) { |
| 3455 | auto Next = std::next(I); |
| 3456 | |
| 3457 | // Move instruction to loop body. |
| 3458 | LoopBB->splice(LoopBB->begin(), &MBB, I, Next); |
| 3459 | |
| 3460 | // Move the rest of the block. |
| 3461 | RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end()); |
| 3462 | } else { |
| 3463 | RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end()); |
| 3464 | } |
| 3465 | |
| 3466 | MBB.addSuccessor(LoopBB); |
| 3467 | |
| 3468 | return std::make_pair(LoopBB, RemainderBB); |
| 3469 | } |
| 3470 | |
| 3471 | /// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it. |
| 3472 | void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const { |
| 3473 | MachineBasicBlock *MBB = MI.getParent(); |
| 3474 | const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); |
| 3475 | auto I = MI.getIterator(); |
| 3476 | auto E = std::next(I); |
| 3477 | |
| 3478 | BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT)) |
| 3479 | .addImm(0); |
| 3480 | |
| 3481 | MIBundleBuilder Bundler(*MBB, I, E); |
| 3482 | finalizeBundle(*MBB, Bundler.begin()); |
| 3483 | } |
| 3484 | |
| 3485 | MachineBasicBlock * |
| 3486 | SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI, |
| 3487 | MachineBasicBlock *BB) const { |
| 3488 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3489 | |
| 3490 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); |
| 3491 | |
| 3492 | MachineBasicBlock *LoopBB; |
| 3493 | MachineBasicBlock *RemainderBB; |
| 3494 | const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); |
| 3495 | |
| 3496 | // Apparently kill flags are only valid if the def is in the same block? |
| 3497 | if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0)) |
| 3498 | Src->setIsKill(false); |
| 3499 | |
| 3500 | std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true); |
| 3501 | |
| 3502 | MachineBasicBlock::iterator I = LoopBB->end(); |
| 3503 | |
| 3504 | const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg( |
| 3505 | AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1); |
| 3506 | |
| 3507 | // Clear TRAP_STS.MEM_VIOL |
| 3508 | BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32)) |
| 3509 | .addImm(0) |
| 3510 | .addImm(EncodedReg); |
| 3511 | |
| 3512 | bundleInstWithWaitcnt(MI); |
| 3513 | |
| 3514 | Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); |
| 3515 | |
| 3516 | // Load and check TRAP_STS.MEM_VIOL |
| 3517 | BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg) |
| 3518 | .addImm(EncodedReg); |
| 3519 | |
| 3520 | // FIXME: Do we need to use an isel pseudo that may clobber scc? |
| 3521 | BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32)) |
| 3522 | .addReg(Reg, RegState::Kill) |
| 3523 | .addImm(0); |
| 3524 | BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1)) |
| 3525 | .addMBB(LoopBB); |
| 3526 | |
| 3527 | return RemainderBB; |
| 3528 | } |
| 3529 | |
| 3530 | // Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the |
| 3531 | // wavefront. If the value is uniform and just happens to be in a VGPR, this |
| 3532 | // will only do one iteration. In the worst case, this will loop 64 times. |
| 3533 | // |
| 3534 | // TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value. |
| 3535 | static MachineBasicBlock::iterator |
| 3536 | emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI, |
| 3537 | MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, |
| 3538 | const DebugLoc &DL, const MachineOperand &Idx, |
| 3539 | unsigned InitReg, unsigned ResultReg, unsigned PhiReg, |
| 3540 | unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, |
| 3541 | Register &SGPRIdxReg) { |
| 3542 | |
| 3543 | MachineFunction *MF = OrigBB.getParent(); |
| 3544 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); |
| 3545 | const SIRegisterInfo *TRI = ST.getRegisterInfo(); |
| 3546 | MachineBasicBlock::iterator I = LoopBB.begin(); |
| 3547 | |
| 3548 | const TargetRegisterClass *BoolRC = TRI->getBoolRC(); |
| 3549 | Register PhiExec = MRI.createVirtualRegister(BoolRC); |
| 3550 | Register NewExec = MRI.createVirtualRegister(BoolRC); |
| 3551 | Register CurrentIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); |
| 3552 | Register CondReg = MRI.createVirtualRegister(BoolRC); |
| 3553 | |
| 3554 | BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg) |
| 3555 | .addReg(InitReg) |
| 3556 | .addMBB(&OrigBB) |
| 3557 | .addReg(ResultReg) |
| 3558 | .addMBB(&LoopBB); |
| 3559 | |
| 3560 | BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec) |
| 3561 | .addReg(InitSaveExecReg) |
| 3562 | .addMBB(&OrigBB) |
| 3563 | .addReg(NewExec) |
| 3564 | .addMBB(&LoopBB); |
| 3565 | |
| 3566 | // Read the next variant <- also loop target. |
| 3567 | BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg) |
| 3568 | .addReg(Idx.getReg(), getUndefRegState(Idx.isUndef())); |
| 3569 | |
| 3570 | // Compare the just read M0 value to all possible Idx values. |
| 3571 | BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg) |
| 3572 | .addReg(CurrentIdxReg) |
| 3573 | .addReg(Idx.getReg(), 0, Idx.getSubReg()); |
| 3574 | |
| 3575 | // Update EXEC, save the original EXEC value to VCC. |
| 3576 | BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 |
| 3577 | : AMDGPU::S_AND_SAVEEXEC_B64), |
| 3578 | NewExec) |
| 3579 | .addReg(CondReg, RegState::Kill); |
| 3580 | |
| 3581 | MRI.setSimpleHint(NewExec, CondReg); |
| 3582 | |
| 3583 | if (UseGPRIdxMode) { |
| 3584 | if (Offset == 0) { |
| 3585 | SGPRIdxReg = CurrentIdxReg; |
| 3586 | } else { |
| 3587 | SGPRIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); |
| 3588 | BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg) |
| 3589 | .addReg(CurrentIdxReg, RegState::Kill) |
| 3590 | .addImm(Offset); |
| 3591 | } |
| 3592 | } else { |
| 3593 | // Move index from VCC into M0 |
| 3594 | if (Offset == 0) { |
| 3595 | BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0) |
| 3596 | .addReg(CurrentIdxReg, RegState::Kill); |
| 3597 | } else { |
| 3598 | BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0) |
| 3599 | .addReg(CurrentIdxReg, RegState::Kill) |
| 3600 | .addImm(Offset); |
| 3601 | } |
| 3602 | } |
| 3603 | |
| 3604 | // Update EXEC, switch all done bits to 0 and all todo bits to 1. |
| 3605 | unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; |
| 3606 | MachineInstr *InsertPt = |
| 3607 | BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_XOR_B32_term |
| 3608 | : AMDGPU::S_XOR_B64_term), Exec) |
| 3609 | .addReg(Exec) |
| 3610 | .addReg(NewExec); |
| 3611 | |
| 3612 | // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use |
| 3613 | // s_cbranch_scc0? |
| 3614 | |
| 3615 | // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover. |
| 3616 | BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ)) |
| 3617 | .addMBB(&LoopBB); |
| 3618 | |
| 3619 | return InsertPt->getIterator(); |
| 3620 | } |
| 3621 | |
| 3622 | // This has slightly sub-optimal regalloc when the source vector is killed by |
| 3623 | // the read. The register allocator does not understand that the kill is |
| 3624 | // per-workitem, so is kept alive for the whole loop so we end up not re-using a |
| 3625 | // subregister from it, using 1 more VGPR than necessary. This was saved when |
| 3626 | // this was expanded after register allocation. |
| 3627 | static MachineBasicBlock::iterator |
| 3628 | loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, |
| 3629 | unsigned InitResultReg, unsigned PhiReg, int Offset, |
| 3630 | bool UseGPRIdxMode, Register &SGPRIdxReg) { |
| 3631 | MachineFunction *MF = MBB.getParent(); |
| 3632 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); |
| 3633 | const SIRegisterInfo *TRI = ST.getRegisterInfo(); |
| 3634 | MachineRegisterInfo &MRI = MF->getRegInfo(); |
| 3635 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3636 | MachineBasicBlock::iterator I(&MI); |
| 3637 | |
| 3638 | const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); |
| 3639 | Register DstReg = MI.getOperand(0).getReg(); |
| 3640 | Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); |
| 3641 | Register TmpExec = MRI.createVirtualRegister(BoolXExecRC); |
| 3642 | unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; |
| 3643 | unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; |
| 3644 | |
| 3645 | BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec); |
| 3646 | |
| 3647 | // Save the EXEC mask |
| 3648 | BuildMI(MBB, I, DL, TII->get(MovExecOpc), SaveExec) |
| 3649 | .addReg(Exec); |
| 3650 | |
| 3651 | MachineBasicBlock *LoopBB; |
| 3652 | MachineBasicBlock *RemainderBB; |
| 3653 | std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false); |
| 3654 | |
| 3655 | const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); |
| 3656 | |
| 3657 | auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx, |
| 3658 | InitResultReg, DstReg, PhiReg, TmpExec, |
| 3659 | Offset, UseGPRIdxMode, SGPRIdxReg); |
| 3660 | |
| 3661 | MachineBasicBlock* LandingPad = MF->CreateMachineBasicBlock(); |
| 3662 | MachineFunction::iterator MBBI(LoopBB); |
| 3663 | ++MBBI; |
| 3664 | MF->insert(MBBI, LandingPad); |
| 3665 | LoopBB->removeSuccessor(RemainderBB); |
| 3666 | LandingPad->addSuccessor(RemainderBB); |
| 3667 | LoopBB->addSuccessor(LandingPad); |
| 3668 | MachineBasicBlock::iterator First = LandingPad->begin(); |
| 3669 | BuildMI(*LandingPad, First, DL, TII->get(MovExecOpc), Exec) |
| 3670 | .addReg(SaveExec); |
| 3671 | |
| 3672 | return InsPt; |
| 3673 | } |
| 3674 | |
| 3675 | // Returns subreg index, offset |
| 3676 | static std::pair<unsigned, int> |
| 3677 | computeIndirectRegAndOffset(const SIRegisterInfo &TRI, |
| 3678 | const TargetRegisterClass *SuperRC, |
| 3679 | unsigned VecReg, |
| 3680 | int Offset) { |
| 3681 | int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32; |
| 3682 | |
| 3683 | // Skip out of bounds offsets, or else we would end up using an undefined |
| 3684 | // register. |
| 3685 | if (Offset >= NumElts || Offset < 0) |
| 3686 | return std::make_pair(AMDGPU::sub0, Offset); |
| 3687 | |
| 3688 | return std::make_pair(SIRegisterInfo::getSubRegFromChannel(Offset), 0); |
| 3689 | } |
| 3690 | |
| 3691 | static void setM0ToIndexFromSGPR(const SIInstrInfo *TII, |
| 3692 | MachineRegisterInfo &MRI, MachineInstr &MI, |
| 3693 | int Offset) { |
| 3694 | MachineBasicBlock *MBB = MI.getParent(); |
| 3695 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3696 | MachineBasicBlock::iterator I(&MI); |
| 3697 | |
| 3698 | const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); |
| 3699 | |
| 3700 | assert(Idx->getReg() != AMDGPU::NoRegister)((void)0); |
| 3701 | |
| 3702 | if (Offset == 0) { |
| 3703 | BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0).add(*Idx); |
| 3704 | } else { |
| 3705 | BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0) |
| 3706 | .add(*Idx) |
| 3707 | .addImm(Offset); |
| 3708 | } |
| 3709 | } |
| 3710 | |
| 3711 | static Register getIndirectSGPRIdx(const SIInstrInfo *TII, |
| 3712 | MachineRegisterInfo &MRI, MachineInstr &MI, |
| 3713 | int Offset) { |
| 3714 | MachineBasicBlock *MBB = MI.getParent(); |
| 3715 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3716 | MachineBasicBlock::iterator I(&MI); |
| 3717 | |
| 3718 | const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); |
| 3719 | |
| 3720 | if (Offset == 0) |
| 3721 | return Idx->getReg(); |
| 3722 | |
| 3723 | Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); |
| 3724 | BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp) |
| 3725 | .add(*Idx) |
| 3726 | .addImm(Offset); |
| 3727 | return Tmp; |
| 3728 | } |
| 3729 | |
| 3730 | static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI, |
| 3731 | MachineBasicBlock &MBB, |
| 3732 | const GCNSubtarget &ST) { |
| 3733 | const SIInstrInfo *TII = ST.getInstrInfo(); |
| 3734 | const SIRegisterInfo &TRI = TII->getRegisterInfo(); |
| 3735 | MachineFunction *MF = MBB.getParent(); |
| 3736 | MachineRegisterInfo &MRI = MF->getRegInfo(); |
| 3737 | |
| 3738 | Register Dst = MI.getOperand(0).getReg(); |
| 3739 | const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); |
| 3740 | Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg(); |
| 3741 | int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm(); |
| 3742 | |
| 3743 | const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg); |
| 3744 | const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg()); |
| 3745 | |
| 3746 | unsigned SubReg; |
| 3747 | std::tie(SubReg, Offset) |
| 3748 | = computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset); |
| 3749 | |
| 3750 | const bool UseGPRIdxMode = ST.useVGPRIndexMode(); |
| 3751 | |
| 3752 | // Check for a SGPR index. |
| 3753 | if (TII->getRegisterInfo().isSGPRClass(IdxRC)) { |
| 3754 | MachineBasicBlock::iterator I(&MI); |
| 3755 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3756 | |
| 3757 | if (UseGPRIdxMode) { |
| 3758 | // TODO: Look at the uses to avoid the copy. This may require rescheduling |
| 3759 | // to avoid interfering with other uses, so probably requires a new |
| 3760 | // optimization pass. |
| 3761 | Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset); |
| 3762 | |
| 3763 | const MCInstrDesc &GPRIDXDesc = |
| 3764 | TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true); |
| 3765 | BuildMI(MBB, I, DL, GPRIDXDesc, Dst) |
| 3766 | .addReg(SrcReg) |
| 3767 | .addReg(Idx) |
| 3768 | .addImm(SubReg); |
| 3769 | } else { |
| 3770 | setM0ToIndexFromSGPR(TII, MRI, MI, Offset); |
| 3771 | |
| 3772 | BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst) |
| 3773 | .addReg(SrcReg, 0, SubReg) |
| 3774 | .addReg(SrcReg, RegState::Implicit); |
| 3775 | } |
| 3776 | |
| 3777 | MI.eraseFromParent(); |
| 3778 | |
| 3779 | return &MBB; |
| 3780 | } |
| 3781 | |
| 3782 | // Control flow needs to be inserted if indexing with a VGPR. |
| 3783 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3784 | MachineBasicBlock::iterator I(&MI); |
| 3785 | |
| 3786 | Register PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); |
| 3787 | Register InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); |
| 3788 | |
| 3789 | BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg); |
| 3790 | |
| 3791 | Register SGPRIdxReg; |
| 3792 | auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg, Offset, |
| 3793 | UseGPRIdxMode, SGPRIdxReg); |
| 3794 | |
| 3795 | MachineBasicBlock *LoopBB = InsPt->getParent(); |
| 3796 | |
| 3797 | if (UseGPRIdxMode) { |
| 3798 | const MCInstrDesc &GPRIDXDesc = |
| 3799 | TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true); |
| 3800 | |
| 3801 | BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst) |
| 3802 | .addReg(SrcReg) |
| 3803 | .addReg(SGPRIdxReg) |
| 3804 | .addImm(SubReg); |
| 3805 | } else { |
| 3806 | BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst) |
| 3807 | .addReg(SrcReg, 0, SubReg) |
| 3808 | .addReg(SrcReg, RegState::Implicit); |
| 3809 | } |
| 3810 | |
| 3811 | MI.eraseFromParent(); |
| 3812 | |
| 3813 | return LoopBB; |
| 3814 | } |
| 3815 | |
| 3816 | static MachineBasicBlock *emitIndirectDst(MachineInstr &MI, |
| 3817 | MachineBasicBlock &MBB, |
| 3818 | const GCNSubtarget &ST) { |
| 3819 | const SIInstrInfo *TII = ST.getInstrInfo(); |
| 3820 | const SIRegisterInfo &TRI = TII->getRegisterInfo(); |
| 3821 | MachineFunction *MF = MBB.getParent(); |
| 3822 | MachineRegisterInfo &MRI = MF->getRegInfo(); |
| 3823 | |
| 3824 | Register Dst = MI.getOperand(0).getReg(); |
| 3825 | const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src); |
| 3826 | const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); |
| 3827 | const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val); |
| 3828 | int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm(); |
| 3829 | const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg()); |
| 3830 | const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg()); |
| 3831 | |
| 3832 | // This can be an immediate, but will be folded later. |
| 3833 | assert(Val->getReg())((void)0); |
| 3834 | |
| 3835 | unsigned SubReg; |
| 3836 | std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC, |
| 3837 | SrcVec->getReg(), |
| 3838 | Offset); |
| 3839 | const bool UseGPRIdxMode = ST.useVGPRIndexMode(); |
| 3840 | |
| 3841 | if (Idx->getReg() == AMDGPU::NoRegister) { |
| 3842 | MachineBasicBlock::iterator I(&MI); |
| 3843 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3844 | |
| 3845 | assert(Offset == 0)((void)0); |
| 3846 | |
| 3847 | BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst) |
| 3848 | .add(*SrcVec) |
| 3849 | .add(*Val) |
| 3850 | .addImm(SubReg); |
| 3851 | |
| 3852 | MI.eraseFromParent(); |
| 3853 | return &MBB; |
| 3854 | } |
| 3855 | |
| 3856 | // Check for a SGPR index. |
| 3857 | if (TII->getRegisterInfo().isSGPRClass(IdxRC)) { |
| 3858 | MachineBasicBlock::iterator I(&MI); |
| 3859 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3860 | |
| 3861 | if (UseGPRIdxMode) { |
| 3862 | Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset); |
| 3863 | |
| 3864 | const MCInstrDesc &GPRIDXDesc = |
| 3865 | TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false); |
| 3866 | BuildMI(MBB, I, DL, GPRIDXDesc, Dst) |
| 3867 | .addReg(SrcVec->getReg()) |
| 3868 | .add(*Val) |
| 3869 | .addReg(Idx) |
| 3870 | .addImm(SubReg); |
| 3871 | } else { |
| 3872 | setM0ToIndexFromSGPR(TII, MRI, MI, Offset); |
| 3873 | |
| 3874 | const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo( |
| 3875 | TRI.getRegSizeInBits(*VecRC), 32, false); |
| 3876 | BuildMI(MBB, I, DL, MovRelDesc, Dst) |
| 3877 | .addReg(SrcVec->getReg()) |
| 3878 | .add(*Val) |
| 3879 | .addImm(SubReg); |
| 3880 | } |
| 3881 | MI.eraseFromParent(); |
| 3882 | return &MBB; |
| 3883 | } |
| 3884 | |
| 3885 | // Control flow needs to be inserted if indexing with a VGPR. |
| 3886 | if (Val->isReg()) |
| 3887 | MRI.clearKillFlags(Val->getReg()); |
| 3888 | |
| 3889 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3890 | |
| 3891 | Register PhiReg = MRI.createVirtualRegister(VecRC); |
| 3892 | |
| 3893 | Register SGPRIdxReg; |
| 3894 | auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg, Offset, |
| 3895 | UseGPRIdxMode, SGPRIdxReg); |
| 3896 | MachineBasicBlock *LoopBB = InsPt->getParent(); |
| 3897 | |
| 3898 | if (UseGPRIdxMode) { |
| 3899 | const MCInstrDesc &GPRIDXDesc = |
| 3900 | TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false); |
| 3901 | |
| 3902 | BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst) |
| 3903 | .addReg(PhiReg) |
| 3904 | .add(*Val) |
| 3905 | .addReg(SGPRIdxReg) |
| 3906 | .addImm(AMDGPU::sub0); |
| 3907 | } else { |
| 3908 | const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo( |
| 3909 | TRI.getRegSizeInBits(*VecRC), 32, false); |
| 3910 | BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst) |
| 3911 | .addReg(PhiReg) |
| 3912 | .add(*Val) |
| 3913 | .addImm(AMDGPU::sub0); |
| 3914 | } |
| 3915 | |
| 3916 | MI.eraseFromParent(); |
| 3917 | return LoopBB; |
| 3918 | } |
| 3919 | |
| 3920 | MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( |
| 3921 | MachineInstr &MI, MachineBasicBlock *BB) const { |
| 3922 | |
| 3923 | const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); |
| 3924 | MachineFunction *MF = BB->getParent(); |
| 3925 | SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); |
| 3926 | |
| 3927 | switch (MI.getOpcode()) { |
| 3928 | case AMDGPU::S_UADDO_PSEUDO: |
| 3929 | case AMDGPU::S_USUBO_PSEUDO: { |
| 3930 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3931 | MachineOperand &Dest0 = MI.getOperand(0); |
| 3932 | MachineOperand &Dest1 = MI.getOperand(1); |
| 3933 | MachineOperand &Src0 = MI.getOperand(2); |
| 3934 | MachineOperand &Src1 = MI.getOperand(3); |
| 3935 | |
| 3936 | unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO) |
| 3937 | ? AMDGPU::S_ADD_I32 |
| 3938 | : AMDGPU::S_SUB_I32; |
| 3939 | BuildMI(*BB, MI, DL, TII->get(Opc), Dest0.getReg()).add(Src0).add(Src1); |
| 3940 | |
| 3941 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CSELECT_B64), Dest1.getReg()) |
| 3942 | .addImm(1) |
| 3943 | .addImm(0); |
| 3944 | |
| 3945 | MI.eraseFromParent(); |
| 3946 | return BB; |
| 3947 | } |
| 3948 | case AMDGPU::S_ADD_U64_PSEUDO: |
| 3949 | case AMDGPU::S_SUB_U64_PSEUDO: { |
| 3950 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); |
| 3951 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); |
| 3952 | const SIRegisterInfo *TRI = ST.getRegisterInfo(); |
| 3953 | const TargetRegisterClass *BoolRC = TRI->getBoolRC(); |
| 3954 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3955 | |
| 3956 | MachineOperand &Dest = MI.getOperand(0); |
| 3957 | MachineOperand &Src0 = MI.getOperand(1); |
| 3958 | MachineOperand &Src1 = MI.getOperand(2); |
| 3959 | |
| 3960 | Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); |
| 3961 | Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); |
| 3962 | |
| 3963 | MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm( |
| 3964 | MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); |
| 3965 | MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm( |
| 3966 | MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); |
| 3967 | |
| 3968 | MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm( |
| 3969 | MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); |
| 3970 | MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm( |
| 3971 | MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); |
| 3972 | |
| 3973 | bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); |
| 3974 | |
| 3975 | unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; |
| 3976 | unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; |
| 3977 | BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0).add(Src0Sub0).add(Src1Sub0); |
| 3978 | BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1).add(Src0Sub1).add(Src1Sub1); |
| 3979 | BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg()) |
| 3980 | .addReg(DestSub0) |
| 3981 | .addImm(AMDGPU::sub0) |
| 3982 | .addReg(DestSub1) |
| 3983 | .addImm(AMDGPU::sub1); |
| 3984 | MI.eraseFromParent(); |
| 3985 | return BB; |
| 3986 | } |
| 3987 | case AMDGPU::V_ADD_U64_PSEUDO: |
| 3988 | case AMDGPU::V_SUB_U64_PSEUDO: { |
| 3989 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); |
| 3990 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); |
| 3991 | const SIRegisterInfo *TRI = ST.getRegisterInfo(); |
| 3992 | const DebugLoc &DL = MI.getDebugLoc(); |
| 3993 | |
| 3994 | bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO); |
| 3995 | |
| 3996 | const auto *CarryRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); |
| 3997 | |
| 3998 | Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); |
| 3999 | Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); |
| 4000 | |
| 4001 | Register CarryReg = MRI.createVirtualRegister(CarryRC); |
| 4002 | Register DeadCarryReg = MRI.createVirtualRegister(CarryRC); |
| 4003 | |
| 4004 | MachineOperand &Dest = MI.getOperand(0); |
| 4005 | MachineOperand &Src0 = MI.getOperand(1); |
| 4006 | MachineOperand &Src1 = MI.getOperand(2); |
| 4007 | |
| 4008 | const TargetRegisterClass *Src0RC = Src0.isReg() |
| 4009 | ? MRI.getRegClass(Src0.getReg()) |
| 4010 | : &AMDGPU::VReg_64RegClass; |
| 4011 | const TargetRegisterClass *Src1RC = Src1.isReg() |
| 4012 | ? MRI.getRegClass(Src1.getReg()) |
| 4013 | : &AMDGPU::VReg_64RegClass; |
| 4014 | |
| 4015 | const TargetRegisterClass *Src0SubRC = |
| 4016 | TRI->getSubRegClass(Src0RC, AMDGPU::sub0); |
| 4017 | const TargetRegisterClass *Src1SubRC = |
| 4018 | TRI->getSubRegClass(Src1RC, AMDGPU::sub1); |
| 4019 | |
| 4020 | MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm( |
| 4021 | MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); |
| 4022 | MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm( |
| 4023 | MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); |
| 4024 | |
| 4025 | MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm( |
| 4026 | MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC); |
| 4027 | MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm( |
| 4028 | MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC); |
| 4029 | |
| 4030 | unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; |
| 4031 | MachineInstr *LoHalf = BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0) |
| 4032 | .addReg(CarryReg, RegState::Define) |
| 4033 | .add(SrcReg0Sub0) |
| 4034 | .add(SrcReg1Sub0) |
| 4035 | .addImm(0); // clamp bit |
| 4036 | |
| 4037 | unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; |
| 4038 | MachineInstr *HiHalf = |
| 4039 | BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1) |
| 4040 | .addReg(DeadCarryReg, RegState::Define | RegState::Dead) |
| 4041 | .add(SrcReg0Sub1) |
| 4042 | .add(SrcReg1Sub1) |
| 4043 | .addReg(CarryReg, RegState::Kill) |
| 4044 | .addImm(0); // clamp bit |
| 4045 | |
| 4046 | BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg()) |
| 4047 | .addReg(DestSub0) |
| 4048 | .addImm(AMDGPU::sub0) |
| 4049 | .addReg(DestSub1) |
| 4050 | .addImm(AMDGPU::sub1); |
| 4051 | TII->legalizeOperands(*LoHalf); |
| 4052 | TII->legalizeOperands(*HiHalf); |
| 4053 | MI.eraseFromParent(); |
| 4054 | return BB; |
| 4055 | } |
| 4056 | case AMDGPU::S_ADD_CO_PSEUDO: |
| 4057 | case AMDGPU::S_SUB_CO_PSEUDO: { |
| 4058 | // This pseudo has a chance to be selected |
| 4059 | // only from uniform add/subcarry node. All the VGPR operands |
| 4060 | // therefore assumed to be splat vectors. |
| 4061 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); |
| 4062 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); |
| 4063 | const SIRegisterInfo *TRI = ST.getRegisterInfo(); |
| 4064 | MachineBasicBlock::iterator MII = MI; |
| 4065 | const DebugLoc &DL = MI.getDebugLoc(); |
| 4066 | MachineOperand &Dest = MI.getOperand(0); |
| 4067 | MachineOperand &CarryDest = MI.getOperand(1); |
| 4068 | MachineOperand &Src0 = MI.getOperand(2); |
| 4069 | MachineOperand &Src1 = MI.getOperand(3); |
| 4070 | MachineOperand &Src2 = MI.getOperand(4); |
| 4071 | unsigned Opc = (MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) |
| 4072 | ? AMDGPU::S_ADDC_U32 |
| 4073 | : AMDGPU::S_SUBB_U32; |
| 4074 | if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) { |
| 4075 | Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); |
| 4076 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0) |
| 4077 | .addReg(Src0.getReg()); |
| 4078 | Src0.setReg(RegOp0); |
| 4079 | } |
| 4080 | if (Src1.isReg() && TRI->isVectorRegister(MRI, Src1.getReg())) { |
| 4081 | Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); |
| 4082 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1) |
| 4083 | .addReg(Src1.getReg()); |
| 4084 | Src1.setReg(RegOp1); |
| 4085 | } |
| 4086 | Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); |
| 4087 | if (TRI->isVectorRegister(MRI, Src2.getReg())) { |
| 4088 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2) |
| 4089 | .addReg(Src2.getReg()); |
| 4090 | Src2.setReg(RegOp2); |
| 4091 | } |
| 4092 | |
| 4093 | const TargetRegisterClass *Src2RC = MRI.getRegClass(Src2.getReg()); |
| 4094 | if (TRI->getRegSizeInBits(*Src2RC) == 64) { |
| 4095 | if (ST.hasScalarCompareEq64()) { |
| 4096 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64)) |
| 4097 | .addReg(Src2.getReg()) |
| 4098 | .addImm(0); |
| 4099 | } else { |
| 4100 | const TargetRegisterClass *SubRC = |
| 4101 | TRI->getSubRegClass(Src2RC, AMDGPU::sub0); |
| 4102 | MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm( |
| 4103 | MII, MRI, Src2, Src2RC, AMDGPU::sub0, SubRC); |
| 4104 | MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm( |
| 4105 | MII, MRI, Src2, Src2RC, AMDGPU::sub1, SubRC); |
| 4106 | Register Src2_32 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); |
| 4107 | |
| 4108 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_OR_B32), Src2_32) |
| 4109 | .add(Src2Sub0) |
| 4110 | .add(Src2Sub1); |
| 4111 | |
| 4112 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32)) |
| 4113 | .addReg(Src2_32, RegState::Kill) |
| 4114 | .addImm(0); |
| 4115 | } |
| 4116 | } else { |
| 4117 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMPK_LG_U32)) |
| 4118 | .addReg(Src2.getReg()) |
| 4119 | .addImm(0); |
| 4120 | } |
| 4121 | |
| 4122 | BuildMI(*BB, MII, DL, TII->get(Opc), Dest.getReg()).add(Src0).add(Src1); |
| 4123 | |
| 4124 | BuildMI(*BB, MII, DL, TII->get(AMDGPU::COPY), CarryDest.getReg()) |
| 4125 | .addReg(AMDGPU::SCC); |
| 4126 | MI.eraseFromParent(); |
| 4127 | return BB; |
| 4128 | } |
| 4129 | case AMDGPU::SI_INIT_M0: { |
| 4130 | BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(), |
| 4131 | TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0) |
| 4132 | .add(MI.getOperand(0)); |
| 4133 | MI.eraseFromParent(); |
| 4134 | return BB; |
| 4135 | } |
| 4136 | case AMDGPU::GET_GROUPSTATICSIZE: { |
| 4137 | assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||((void)0) |
| 4138 | getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL)((void)0); |
| 4139 | DebugLoc DL = MI.getDebugLoc(); |
| 4140 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_MOV_B32)) |
| 4141 | .add(MI.getOperand(0)) |
| 4142 | .addImm(MFI->getLDSSize()); |
| 4143 | MI.eraseFromParent(); |
| 4144 | return BB; |
| 4145 | } |
| 4146 | case AMDGPU::SI_INDIRECT_SRC_V1: |
| 4147 | case AMDGPU::SI_INDIRECT_SRC_V2: |
| 4148 | case AMDGPU::SI_INDIRECT_SRC_V4: |
| 4149 | case AMDGPU::SI_INDIRECT_SRC_V8: |
| 4150 | case AMDGPU::SI_INDIRECT_SRC_V16: |
| 4151 | case AMDGPU::SI_INDIRECT_SRC_V32: |
| 4152 | return emitIndirectSrc(MI, *BB, *getSubtarget()); |
| 4153 | case AMDGPU::SI_INDIRECT_DST_V1: |
| 4154 | case AMDGPU::SI_INDIRECT_DST_V2: |
| 4155 | case AMDGPU::SI_INDIRECT_DST_V4: |
| 4156 | case AMDGPU::SI_INDIRECT_DST_V8: |
| 4157 | case AMDGPU::SI_INDIRECT_DST_V16: |
| 4158 | case AMDGPU::SI_INDIRECT_DST_V32: |
| 4159 | return emitIndirectDst(MI, *BB, *getSubtarget()); |
| 4160 | case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: |
| 4161 | case AMDGPU::SI_KILL_I1_PSEUDO: |
| 4162 | return splitKillBlock(MI, BB); |
| 4163 | case AMDGPU::V_CNDMASK_B64_PSEUDO: { |
| 4164 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); |
| 4165 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); |
| 4166 | const SIRegisterInfo *TRI = ST.getRegisterInfo(); |
| 4167 | |
| 4168 | Register Dst = MI.getOperand(0).getReg(); |
| 4169 | Register Src0 = MI.getOperand(1).getReg(); |
| 4170 | Register Src1 = MI.getOperand(2).getReg(); |
| 4171 | const DebugLoc &DL = MI.getDebugLoc(); |
| 4172 | Register SrcCond = MI.getOperand(3).getReg(); |
| 4173 | |
| 4174 | Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); |
| 4175 | Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); |
| 4176 | const auto *CondRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); |
| 4177 | Register SrcCondCopy = MRI.createVirtualRegister(CondRC); |
| 4178 | |
| 4179 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy) |
| 4180 | .addReg(SrcCond); |
| 4181 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo) |
| 4182 | .addImm(0) |
| 4183 | .addReg(Src0, 0, AMDGPU::sub0) |
| 4184 | .addImm(0) |
| 4185 | .addReg(Src1, 0, AMDGPU::sub0) |
| 4186 | .addReg(SrcCondCopy); |
| 4187 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi) |
| 4188 | .addImm(0) |
| 4189 | .addReg(Src0, 0, AMDGPU::sub1) |
| 4190 | .addImm(0) |
| 4191 | .addReg(Src1, 0, AMDGPU::sub1) |
| 4192 | .addReg(SrcCondCopy); |
| 4193 | |
| 4194 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst) |
| 4195 | .addReg(DstLo) |
| 4196 | .addImm(AMDGPU::sub0) |
| 4197 | .addReg(DstHi) |
| 4198 | .addImm(AMDGPU::sub1); |
| 4199 | MI.eraseFromParent(); |
| 4200 | return BB; |
| 4201 | } |
| 4202 | case AMDGPU::SI_BR_UNDEF: { |
| 4203 | const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); |
| 4204 | const DebugLoc &DL = MI.getDebugLoc(); |
| 4205 | MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1)) |
| 4206 | .add(MI.getOperand(0)); |
| 4207 | Br->getOperand(1).setIsUndef(true); // read undef SCC |
| 4208 | MI.eraseFromParent(); |
| 4209 | return BB; |
| 4210 | } |
| 4211 | case AMDGPU::ADJCALLSTACKUP: |
| 4212 | case AMDGPU::ADJCALLSTACKDOWN: { |
| 4213 | const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); |
| 4214 | MachineInstrBuilder MIB(*MF, &MI); |
| 4215 | MIB.addReg(Info->getStackPtrOffsetReg(), RegState::ImplicitDefine) |
| 4216 | .addReg(Info->getStackPtrOffsetReg(), RegState::Implicit); |
| 4217 | return BB; |
| 4218 | } |
| 4219 | case AMDGPU::SI_CALL_ISEL: { |
| 4220 | const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); |
| 4221 | const DebugLoc &DL = MI.getDebugLoc(); |
| 4222 | |
| 4223 | unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF); |
| 4224 | |
| 4225 | MachineInstrBuilder MIB; |
| 4226 | MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg); |
| 4227 | |
| 4228 | for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) |
| 4229 | MIB.add(MI.getOperand(I)); |
| 4230 | |
| 4231 | MIB.cloneMemRefs(MI); |
| 4232 | MI.eraseFromParent(); |
| 4233 | return BB; |
| 4234 | } |
| 4235 | case AMDGPU::V_ADD_CO_U32_e32: |
| 4236 | case AMDGPU::V_SUB_CO_U32_e32: |
| 4237 | case AMDGPU::V_SUBREV_CO_U32_e32: { |
| 4238 | // TODO: Define distinct V_*_I32_Pseudo instructions instead. |
| 4239 | const DebugLoc &DL = MI.getDebugLoc(); |
| 4240 | unsigned Opc = MI.getOpcode(); |
| 4241 | |
| 4242 | bool NeedClampOperand = false; |
| 4243 | if (TII->pseudoToMCOpcode(Opc) == -1) { |
| 4244 | Opc = AMDGPU::getVOPe64(Opc); |
| 4245 | NeedClampOperand = true; |
| 4246 | } |
| 4247 | |
| 4248 | auto I = BuildMI(*BB, MI, DL, TII->get(Opc), MI.getOperand(0).getReg()); |
| 4249 | if (TII->isVOP3(*I)) { |
| 4250 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); |
| 4251 | const SIRegisterInfo *TRI = ST.getRegisterInfo(); |
| 4252 | I.addReg(TRI->getVCC(), RegState::Define); |
| 4253 | } |
| 4254 | I.add(MI.getOperand(1)) |
| 4255 | .add(MI.getOperand(2)); |
| 4256 | if (NeedClampOperand) |
| 4257 | I.addImm(0); // clamp bit for e64 encoding |
| 4258 | |
| 4259 | TII->legalizeOperands(*I); |
| 4260 | |
| 4261 | MI.eraseFromParent(); |
| 4262 | return BB; |
| 4263 | } |
| 4264 | case AMDGPU::DS_GWS_INIT: |
| 4265 | case AMDGPU::DS_GWS_SEMA_BR: |
| 4266 | case AMDGPU::DS_GWS_BARRIER: |
| 4267 | if (Subtarget->needsAlignedVGPRs()) { |
| 4268 | // Add implicit aligned super-reg to force alignment on the data operand. |
| 4269 | const DebugLoc &DL = MI.getDebugLoc(); |
| 4270 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); |
| 4271 | const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); |
| 4272 | MachineOperand *Op = TII->getNamedOperand(MI, AMDGPU::OpName::data0); |
| 4273 | Register DataReg = Op->getReg(); |
| 4274 | bool IsAGPR = TRI->isAGPR(MRI, DataReg); |
| 4275 | Register Undef = MRI.createVirtualRegister( |
| 4276 | IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass); |
| 4277 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), Undef); |
| 4278 | Register NewVR = |
| 4279 | MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass |
| 4280 | : &AMDGPU::VReg_64_Align2RegClass); |
| 4281 | BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), NewVR) |
| 4282 | .addReg(DataReg, 0, Op->getSubReg()) |
| 4283 | .addImm(AMDGPU::sub0) |
| 4284 | .addReg(Undef) |
| 4285 | .addImm(AMDGPU::sub1); |
| 4286 | Op->setReg(NewVR); |
| 4287 | Op->setSubReg(AMDGPU::sub0); |
| 4288 | MI.addOperand(MachineOperand::CreateReg(NewVR, false, true)); |
| 4289 | } |
| 4290 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; |
| 4291 | case AMDGPU::DS_GWS_SEMA_V: |
| 4292 | case AMDGPU::DS_GWS_SEMA_P: |
| 4293 | case AMDGPU::DS_GWS_SEMA_RELEASE_ALL: |
| 4294 | // A s_waitcnt 0 is required to be the instruction immediately following. |
| 4295 | if (getSubtarget()->hasGWSAutoReplay()) { |
| 4296 | bundleInstWithWaitcnt(MI); |
| 4297 | return BB; |
| 4298 | } |
| 4299 | |
| 4300 | return emitGWSMemViolTestLoop(MI, BB); |
| 4301 | case AMDGPU::S_SETREG_B32: { |
| 4302 | // Try to optimize cases that only set the denormal mode or rounding mode. |
| 4303 | // |
| 4304 | // If the s_setreg_b32 fully sets all of the bits in the rounding mode or |
| 4305 | // denormal mode to a constant, we can use s_round_mode or s_denorm_mode |
| 4306 | // instead. |
| 4307 | // |
| 4308 | // FIXME: This could be predicates on the immediate, but tablegen doesn't |
| 4309 | // allow you to have a no side effect instruction in the output of a |
| 4310 | // sideeffecting pattern. |
| 4311 | unsigned ID, Offset, Width; |
| 4312 | AMDGPU::Hwreg::decodeHwreg(MI.getOperand(1).getImm(), ID, Offset, Width); |
| 4313 | if (ID != AMDGPU::Hwreg::ID_MODE) |
| 4314 | return BB; |
| 4315 | |
| 4316 | const unsigned WidthMask = maskTrailingOnes<unsigned>(Width); |
| 4317 | const unsigned SetMask = WidthMask << Offset; |
| 4318 | |
| 4319 | if (getSubtarget()->hasDenormModeInst()) { |
| 4320 | unsigned SetDenormOp = 0; |
| 4321 | unsigned SetRoundOp = 0; |
| 4322 | |
| 4323 | // The dedicated instructions can only set the whole denorm or round mode |
| 4324 | // at once, not a subset of bits in either. |
| 4325 | if (SetMask == |
| 4326 | (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) { |
| 4327 | // If this fully sets both the round and denorm mode, emit the two |
| 4328 | // dedicated instructions for these. |
| 4329 | SetRoundOp = AMDGPU::S_ROUND_MODE; |
| 4330 | SetDenormOp = AMDGPU::S_DENORM_MODE; |
| 4331 | } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) { |
| 4332 | SetRoundOp = AMDGPU::S_ROUND_MODE; |
| 4333 | } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) { |
| 4334 | SetDenormOp = AMDGPU::S_DENORM_MODE; |
| 4335 | } |
| 4336 | |
| 4337 | if (SetRoundOp || SetDenormOp) { |
| 4338 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); |
| 4339 | MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg()); |
| 4340 | if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) { |
| 4341 | unsigned ImmVal = Def->getOperand(1).getImm(); |
| 4342 | if (SetRoundOp) { |
| 4343 | BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetRoundOp)) |
| 4344 | .addImm(ImmVal & 0xf); |
| 4345 | |
| 4346 | // If we also have the denorm mode, get just the denorm mode bits. |
| 4347 | ImmVal >>= 4; |
| 4348 | } |
| 4349 | |
| 4350 | if (SetDenormOp) { |
| 4351 | BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetDenormOp)) |
| 4352 | .addImm(ImmVal & 0xf); |
| 4353 | } |
| 4354 | |
| 4355 | MI.eraseFromParent(); |
| 4356 | return BB; |
| 4357 | } |
| 4358 | } |
| 4359 | } |
| 4360 | |
| 4361 | // If only FP bits are touched, used the no side effects pseudo. |
| 4362 | if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK | |
| 4363 | AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask) |
| 4364 | MI.setDesc(TII->get(AMDGPU::S_SETREG_B32_mode)); |
| 4365 | |
| 4366 | return BB; |
| 4367 | } |
| 4368 | default: |
| 4369 | return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); |
| 4370 | } |
| 4371 | } |
| 4372 | |
| 4373 | bool SITargetLowering::hasBitPreservingFPLogic(EVT VT) const { |
| 4374 | return isTypeLegal(VT.getScalarType()); |
| 4375 | } |
| 4376 | |
| 4377 | bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const { |
| 4378 | // This currently forces unfolding various combinations of fsub into fma with |
| 4379 | // free fneg'd operands. As long as we have fast FMA (controlled by |
| 4380 | // isFMAFasterThanFMulAndFAdd), we should perform these. |
| 4381 | |
| 4382 | // When fma is quarter rate, for f64 where add / sub are at best half rate, |
| 4383 | // most of these combines appear to be cycle neutral but save on instruction |
| 4384 | // count / code size. |
| 4385 | return true; |
| 4386 | } |
| 4387 | |
| 4388 | EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx, |
| 4389 | EVT VT) const { |
| 4390 | if (!VT.isVector()) { |
| 4391 | return MVT::i1; |
| 4392 | } |
| 4393 | return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements()); |
| 4394 | } |
| 4395 | |
| 4396 | MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const { |
| 4397 | // TODO: Should i16 be used always if legal? For now it would force VALU |
| 4398 | // shifts. |
| 4399 | return (VT == MVT::i16) ? MVT::i16 : MVT::i32; |
| 4400 | } |
| 4401 | |
| 4402 | LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const { |
| 4403 | return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts()) |
| 4404 | ? Ty.changeElementSize(16) |
| 4405 | : Ty.changeElementSize(32); |
| 4406 | } |
| 4407 | |
| 4408 | // Answering this is somewhat tricky and depends on the specific device which |
| 4409 | // have different rates for fma or all f64 operations. |
| 4410 | // |
| 4411 | // v_fma_f64 and v_mul_f64 always take the same number of cycles as each other |
| 4412 | // regardless of which device (although the number of cycles differs between |
| 4413 | // devices), so it is always profitable for f64. |
| 4414 | // |
| 4415 | // v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable |
| 4416 | // only on full rate devices. Normally, we should prefer selecting v_mad_f32 |
| 4417 | // which we can always do even without fused FP ops since it returns the same |
| 4418 | // result as the separate operations and since it is always full |
| 4419 | // rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32 |
| 4420 | // however does not support denormals, so we do report fma as faster if we have |
| 4421 | // a fast fma device and require denormals. |
| 4422 | // |
| 4423 | bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, |
| 4424 | EVT VT) const { |
| 4425 | VT = VT.getScalarType(); |
| 4426 | |
| 4427 | switch (VT.getSimpleVT().SimpleTy) { |
| 4428 | case MVT::f32: { |
| 4429 | // If mad is not available this depends only on if f32 fma is full rate. |
| 4430 | if (!Subtarget->hasMadMacF32Insts()) |
| 4431 | return Subtarget->hasFastFMAF32(); |
| 4432 | |
| 4433 | // Otherwise f32 mad is always full rate and returns the same result as |
| 4434 | // the separate operations so should be preferred over fma. |
| 4435 | // However does not support denomals. |
| 4436 | if (hasFP32Denormals(MF)) |
| 4437 | return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts(); |
| 4438 | |
| 4439 | // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32. |
| 4440 | return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts(); |
| 4441 | } |
| 4442 | case MVT::f64: |
| 4443 | return true; |
| 4444 | case MVT::f16: |
| 4445 | return Subtarget->has16BitInsts() && hasFP64FP16Denormals(MF); |
| 4446 | default: |
| 4447 | break; |
| 4448 | } |
| 4449 | |
| 4450 | return false; |
| 4451 | } |
| 4452 | |
| 4453 | bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG, |
| 4454 | const SDNode *N) const { |
| 4455 | // TODO: Check future ftz flag |
| 4456 | // v_mad_f32/v_mac_f32 do not support denormals. |
| 4457 | EVT VT = N->getValueType(0); |
| 4458 | if (VT == MVT::f32) |
| 4459 | return Subtarget->hasMadMacF32Insts() && |
| 4460 | !hasFP32Denormals(DAG.getMachineFunction()); |
| 4461 | if (VT == MVT::f16) { |
| 4462 | return Subtarget->hasMadF16() && |
| 4463 | !hasFP64FP16Denormals(DAG.getMachineFunction()); |
| 4464 | } |
| 4465 | |
| 4466 | return false; |
| 4467 | } |
| 4468 | |
| 4469 | //===----------------------------------------------------------------------===// |
| 4470 | // Custom DAG Lowering Operations |
| 4471 | //===----------------------------------------------------------------------===// |
| 4472 | |
| 4473 | // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the |
| 4474 | // wider vector type is legal. |
| 4475 | SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op, |
| 4476 | SelectionDAG &DAG) const { |
| 4477 | unsigned Opc = Op.getOpcode(); |
| 4478 | EVT VT = Op.getValueType(); |
| 4479 | assert(VT == MVT::v4f16 || VT == MVT::v4i16)((void)0); |
| 4480 | |
| 4481 | SDValue Lo, Hi; |
| 4482 | std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0); |
| 4483 | |
| 4484 | SDLoc SL(Op); |
| 4485 | SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo, |
| 4486 | Op->getFlags()); |
| 4487 | SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi, |
| 4488 | Op->getFlags()); |
| 4489 | |
| 4490 | return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi); |
| 4491 | } |
| 4492 | |
| 4493 | // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the |
| 4494 | // wider vector type is legal. |
| 4495 | SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op, |
| 4496 | SelectionDAG &DAG) const { |
| 4497 | unsigned Opc = Op.getOpcode(); |
| 4498 | EVT VT = Op.getValueType(); |
| 4499 | assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 ||((void)0) |
| 4500 | VT == MVT::v8f32 || VT == MVT::v16f32 || VT == MVT::v32f32)((void)0); |
| 4501 | |
| 4502 | SDValue Lo0, Hi0; |
| 4503 | std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0); |
| 4504 | SDValue Lo1, Hi1; |
| 4505 | std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1); |
| 4506 | |
| 4507 | SDLoc SL(Op); |
| 4508 | |
| 4509 | SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, |
| 4510 | Op->getFlags()); |
| 4511 | SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, |
| 4512 | Op->getFlags()); |
| 4513 | |
| 4514 | return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi); |
| 4515 | } |
| 4516 | |
| 4517 | SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op, |
| 4518 | SelectionDAG &DAG) const { |
| 4519 | unsigned Opc = Op.getOpcode(); |
| 4520 | EVT VT = Op.getValueType(); |
| 4521 | assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 ||((void)0) |
| 4522 | VT == MVT::v8f32 || VT == MVT::v16f32 || VT == MVT::v32f32)((void)0); |
| 4523 | |
| 4524 | SDValue Lo0, Hi0; |
| 4525 | std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0); |
| 4526 | SDValue Lo1, Hi1; |
| 4527 | std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1); |
| 4528 | SDValue Lo2, Hi2; |
| 4529 | std::tie(Lo2, Hi2) = DAG.SplitVectorOperand(Op.getNode(), 2); |
| 4530 | |
| 4531 | SDLoc SL(Op); |
| 4532 | |
| 4533 | SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Lo2, |
| 4534 | Op->getFlags()); |
| 4535 | SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Hi2, |
| 4536 | Op->getFlags()); |
| 4537 | |
| 4538 | return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi); |
| 4539 | } |
| 4540 | |
| 4541 | |
| 4542 | SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { |
| 4543 | switch (Op.getOpcode()) { |
| 4544 | default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); |
| 4545 | case ISD::BRCOND: return LowerBRCOND(Op, DAG); |
| 4546 | case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); |
| 4547 | case ISD::LOAD: { |
| 4548 | SDValue Result = LowerLOAD(Op, DAG); |
| 4549 | assert((!Result.getNode() ||((void)0) |
| 4550 | Result.getNode()->getNumValues() == 2) &&((void)0) |
| 4551 | "Load should return a value and a chain")((void)0); |
| 4552 | return Result; |
| 4553 | } |
| 4554 | |
| 4555 | case ISD::FSIN: |
| 4556 | case ISD::FCOS: |
| 4557 | return LowerTrig(Op, DAG); |
| 4558 | case ISD::SELECT: return LowerSELECT(Op, DAG); |
| 4559 | case ISD::FDIV: return LowerFDIV(Op, DAG); |
| 4560 | case ISD::ATOMIC_CMP_SWAP: return LowerATOMIC_CMP_SWAP(Op, DAG); |
| 4561 | case ISD::STORE: return LowerSTORE(Op, DAG); |
| 4562 | case ISD::GlobalAddress: { |
| 4563 | MachineFunction &MF = DAG.getMachineFunction(); |
| 4564 | SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| 4565 | return LowerGlobalAddress(MFI, Op, DAG); |
| 4566 | } |
| 4567 | case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); |
| 4568 | case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, DAG); |
| 4569 | case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG); |
| 4570 | case ISD::ADDRSPACECAST: return lowerADDRSPACECAST(Op, DAG); |
| 4571 | case ISD::INSERT_SUBVECTOR: |
| 4572 | return lowerINSERT_SUBVECTOR(Op, DAG); |
| 4573 | case ISD::INSERT_VECTOR_ELT: |
| 4574 | return lowerINSERT_VECTOR_ELT(Op, DAG); |
| 4575 | case ISD::EXTRACT_VECTOR_ELT: |
| 4576 | return lowerEXTRACT_VECTOR_ELT(Op, DAG); |
| 4577 | case ISD::VECTOR_SHUFFLE: |
| 4578 | return lowerVECTOR_SHUFFLE(Op, DAG); |
| 4579 | case ISD::BUILD_VECTOR: |
| 4580 | return lowerBUILD_VECTOR(Op, DAG); |
| 4581 | case ISD::FP_ROUND: |
| 4582 | return lowerFP_ROUND(Op, DAG); |
| 4583 | case ISD::TRAP: |
| 4584 | return lowerTRAP(Op, DAG); |
| 4585 | case ISD::DEBUGTRAP: |
| 4586 | return lowerDEBUGTRAP(Op, DAG); |
| 4587 | case ISD::FABS: |
| 4588 | case ISD::FNEG: |
| 4589 | case ISD::FCANONICALIZE: |
| 4590 | case ISD::BSWAP: |
| 4591 | return splitUnaryVectorOp(Op, DAG); |
| 4592 | case ISD::FMINNUM: |
| 4593 | case ISD::FMAXNUM: |
| 4594 | return lowerFMINNUM_FMAXNUM(Op, DAG); |
| 4595 | case ISD::FMA: |
| 4596 | return splitTernaryVectorOp(Op, DAG); |
| 4597 | case ISD::FP_TO_SINT: |
| 4598 | case ISD::FP_TO_UINT: |
| 4599 | return LowerFP_TO_INT(Op, DAG); |
| 4600 | case ISD::SHL: |
| 4601 | case ISD::SRA: |
| 4602 | case ISD::SRL: |
| 4603 | case ISD::ADD: |
| 4604 | case ISD::SUB: |
| 4605 | case ISD::MUL: |
| 4606 | case ISD::SMIN: |
| 4607 | case ISD::SMAX: |
| 4608 | case ISD::UMIN: |
| 4609 | case ISD::UMAX: |
| 4610 | case ISD::FADD: |
| 4611 | case ISD::FMUL: |
| 4612 | case ISD::FMINNUM_IEEE: |
| 4613 | case ISD::FMAXNUM_IEEE: |
| 4614 | case ISD::UADDSAT: |
| 4615 | case ISD::USUBSAT: |
| 4616 | case ISD::SADDSAT: |
| 4617 | case ISD::SSUBSAT: |
| 4618 | return splitBinaryVectorOp(Op, DAG); |
| 4619 | case ISD::SMULO: |
| 4620 | case ISD::UMULO: |
| 4621 | return lowerXMULO(Op, DAG); |
| 4622 | case ISD::DYNAMIC_STACKALLOC: |
| 4623 | return LowerDYNAMIC_STACKALLOC(Op, DAG); |
| 4624 | } |
| 4625 | return SDValue(); |
| 4626 | } |
| 4627 | |
| 4628 | // Used for D16: Casts the result of an instruction into the right vector, |
| 4629 | // packs values if loads return unpacked values. |
| 4630 | static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT, |
| 4631 | const SDLoc &DL, |
| 4632 | SelectionDAG &DAG, bool Unpacked) { |
| 4633 | if (!LoadVT.isVector()) |
| 4634 | return Result; |
| 4635 | |
| 4636 | // Cast back to the original packed type or to a larger type that is a |
| 4637 | // multiple of 32 bit for D16. Widening the return type is a required for |
| 4638 | // legalization. |
| 4639 | EVT FittingLoadVT = LoadVT; |
| 4640 | if ((LoadVT.getVectorNumElements() % 2) == 1) { |
| 4641 | FittingLoadVT = |
| 4642 | EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(), |
| 4643 | LoadVT.getVectorNumElements() + 1); |
| 4644 | } |
| 4645 | |
| 4646 | if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16. |
| 4647 | // Truncate to v2i16/v4i16. |
| 4648 | EVT IntLoadVT = FittingLoadVT.changeTypeToInteger(); |
| 4649 | |
| 4650 | // Workaround legalizer not scalarizing truncate after vector op |
| 4651 | // legalization but not creating intermediate vector trunc. |
| 4652 | SmallVector<SDValue, 4> Elts; |
| 4653 | DAG.ExtractVectorElements(Result, Elts); |
| 4654 | for (SDValue &Elt : Elts) |
| 4655 | Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt); |
| 4656 | |
| 4657 | // Pad illegal v1i16/v3fi6 to v4i16 |
| 4658 | if ((LoadVT.getVectorNumElements() % 2) == 1) |
| 4659 | Elts.push_back(DAG.getUNDEF(MVT::i16)); |
| 4660 | |
| 4661 | Result = DAG.getBuildVector(IntLoadVT, DL, Elts); |
| 4662 | |
| 4663 | // Bitcast to original type (v2f16/v4f16). |
| 4664 | return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result); |
| 4665 | } |
| 4666 | |
| 4667 | // Cast back to the original packed type. |
| 4668 | return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result); |
| 4669 | } |
| 4670 | |
| 4671 | SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, |
| 4672 | MemSDNode *M, |
| 4673 | SelectionDAG &DAG, |
| 4674 | ArrayRef<SDValue> Ops, |
| 4675 | bool IsIntrinsic) const { |
| 4676 | SDLoc DL(M); |
| 4677 | |
| 4678 | bool Unpacked = Subtarget->hasUnpackedD16VMem(); |
| 4679 | EVT LoadVT = M->getValueType(0); |
| 4680 | |
| 4681 | EVT EquivLoadVT = LoadVT; |
| 4682 | if (LoadVT.isVector()) { |
| 4683 | if (Unpacked) { |
| 4684 | EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, |
| 4685 | LoadVT.getVectorNumElements()); |
| 4686 | } else if ((LoadVT.getVectorNumElements() % 2) == 1) { |
| 4687 | // Widen v3f16 to legal type |
| 4688 | EquivLoadVT = |
| 4689 | EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(), |
| 4690 | LoadVT.getVectorNumElements() + 1); |
| 4691 | } |
| 4692 | } |
| 4693 | |
| 4694 | // Change from v4f16/v2f16 to EquivLoadVT. |
| 4695 | SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other); |
| 4696 | |
| 4697 | SDValue Load |
| 4698 | = DAG.getMemIntrinsicNode( |
| 4699 | IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, DL, |
| 4700 | VTList, Ops, M->getMemoryVT(), |
| 4701 | M->getMemOperand()); |
| 4702 | |
| 4703 | SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked); |
| 4704 | |
| 4705 | return DAG.getMergeValues({ Adjusted, Load.getValue(1) }, DL); |
| 4706 | } |
| 4707 | |
| 4708 | SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, |
| 4709 | SelectionDAG &DAG, |
| 4710 | ArrayRef<SDValue> Ops) const { |
| 4711 | SDLoc DL(M); |
| 4712 | EVT LoadVT = M->getValueType(0); |
| 4713 | EVT EltType = LoadVT.getScalarType(); |
| 4714 | EVT IntVT = LoadVT.changeTypeToInteger(); |
| 4715 | |
| 4716 | bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16); |
| 4717 | |
| 4718 | unsigned Opc = |
| 4719 | IsFormat ? AMDGPUISD::BUFFER_LOAD_FORMAT : AMDGPUISD::BUFFER_LOAD; |
| 4720 | |
| 4721 | if (IsD16) { |
| 4722 | return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops); |
| 4723 | } |
| 4724 | |
| 4725 | // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics |
| 4726 | if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32) |
| 4727 | return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, M); |
| 4728 | |
| 4729 | if (isTypeLegal(LoadVT)) { |
| 4730 | return getMemIntrinsicNode(Opc, DL, M->getVTList(), Ops, IntVT, |
| 4731 | M->getMemOperand(), DAG); |
| 4732 | } |
| 4733 | |
| 4734 | EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT); |
| 4735 | SDVTList VTList = DAG.getVTList(CastVT, MVT::Other); |
| 4736 | SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT, |
| 4737 | M->getMemOperand(), DAG); |
| 4738 | return DAG.getMergeValues( |
| 4739 | {DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode), MemNode.getValue(1)}, |
| 4740 | DL); |
| 4741 | } |
| 4742 | |
| 4743 | static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, |
| 4744 | SDNode *N, SelectionDAG &DAG) { |
| 4745 | EVT VT = N->getValueType(0); |
| 4746 | const auto *CD = cast<ConstantSDNode>(N->getOperand(3)); |
| 4747 | unsigned CondCode = CD->getZExtValue(); |
| 4748 | if (!ICmpInst::isIntPredicate(static_cast<ICmpInst::Predicate>(CondCode))) |
| 4749 | return DAG.getUNDEF(VT); |
| 4750 | |
| 4751 | ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode); |
| 4752 | |
| 4753 | SDValue LHS = N->getOperand(1); |
| 4754 | SDValue RHS = N->getOperand(2); |
| 4755 | |
| 4756 | SDLoc DL(N); |
| 4757 | |
| 4758 | EVT CmpVT = LHS.getValueType(); |
| 4759 | if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) { |
| 4760 | unsigned PromoteOp = ICmpInst::isSigned(IcInput) ? |
| 4761 | ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
| 4762 | LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS); |
| 4763 | RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS); |
| 4764 | } |
| 4765 | |
| 4766 | ISD::CondCode CCOpcode = getICmpCondCode(IcInput); |
| 4767 | |
| 4768 | unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize(); |
| 4769 | EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize); |
| 4770 | |
| 4771 | SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS, |
| 4772 | DAG.getCondCode(CCOpcode)); |
| 4773 | if (VT.bitsEq(CCVT)) |
| 4774 | return SetCC; |
| 4775 | return DAG.getZExtOrTrunc(SetCC, DL, VT); |
| 4776 | } |
| 4777 | |
| 4778 | static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI, |
| 4779 | SDNode *N, SelectionDAG &DAG) { |
| 4780 | EVT VT = N->getValueType(0); |
| 4781 | const auto *CD = cast<ConstantSDNode>(N->getOperand(3)); |
| 4782 | |
| 4783 | unsigned CondCode = CD->getZExtValue(); |
| 4784 | if (!FCmpInst::isFPPredicate(static_cast<FCmpInst::Predicate>(CondCode))) |
| 4785 | return DAG.getUNDEF(VT); |
| 4786 | |
| 4787 | SDValue Src0 = N->getOperand(1); |
| 4788 | SDValue Src1 = N->getOperand(2); |
| 4789 | EVT CmpVT = Src0.getValueType(); |
| 4790 | SDLoc SL(N); |
| 4791 | |
| 4792 | if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) { |
| 4793 | Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); |
| 4794 | Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); |
| 4795 | } |
| 4796 | |
| 4797 | FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode); |
| 4798 | ISD::CondCode CCOpcode = getFCmpCondCode(IcInput); |
| 4799 | unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize(); |
| 4800 | EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize); |
| 4801 | SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0, |
| 4802 | Src1, DAG.getCondCode(CCOpcode)); |
| 4803 | if (VT.bitsEq(CCVT)) |
| 4804 | return SetCC; |
| 4805 | return DAG.getZExtOrTrunc(SetCC, SL, VT); |
| 4806 | } |
| 4807 | |
| 4808 | static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N, |
| 4809 | SelectionDAG &DAG) { |
| 4810 | EVT VT = N->getValueType(0); |
| 4811 | SDValue Src = N->getOperand(1); |
| 4812 | SDLoc SL(N); |
| 4813 | |
| 4814 | if (Src.getOpcode() == ISD::SETCC) { |
| 4815 | // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...) |
| 4816 | return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0), |
| 4817 | Src.getOperand(1), Src.getOperand(2)); |
| 4818 | } |
| 4819 | if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) { |
| 4820 | // (ballot 0) -> 0 |
| 4821 | if (Arg->isNullValue()) |
| 4822 | return DAG.getConstant(0, SL, VT); |
| 4823 | |
| 4824 | // (ballot 1) -> EXEC/EXEC_LO |
| 4825 | if (Arg->isOne()) { |
| 4826 | Register Exec; |
| 4827 | if (VT.getScalarSizeInBits() == 32) |
| 4828 | Exec = AMDGPU::EXEC_LO; |
| 4829 | else if (VT.getScalarSizeInBits() == 64) |
| 4830 | Exec = AMDGPU::EXEC; |
| 4831 | else |
| 4832 | return SDValue(); |
| 4833 | |
| 4834 | return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT); |
| 4835 | } |
| 4836 | } |
| 4837 | |
| 4838 | // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0) |
| 4839 | // ISD::SETNE) |
| 4840 | return DAG.getNode( |
| 4841 | AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32), |
| 4842 | DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE)); |
| 4843 | } |
| 4844 | |
| 4845 | void SITargetLowering::ReplaceNodeResults(SDNode *N, |
| 4846 | SmallVectorImpl<SDValue> &Results, |
| 4847 | SelectionDAG &DAG) const { |
| 4848 | switch (N->getOpcode()) { |
| 4849 | case ISD::INSERT_VECTOR_ELT: { |
| 4850 | if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG)) |
| 4851 | Results.push_back(Res); |
| 4852 | return; |
| 4853 | } |
| 4854 | case ISD::EXTRACT_VECTOR_ELT: { |
| 4855 | if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG)) |
| 4856 | Results.push_back(Res); |
| 4857 | return; |
| 4858 | } |
| 4859 | case ISD::INTRINSIC_WO_CHAIN: { |
| 4860 | unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); |
| 4861 | switch (IID) { |
| 4862 | case Intrinsic::amdgcn_cvt_pkrtz: { |
| 4863 | SDValue Src0 = N->getOperand(1); |
| 4864 | SDValue Src1 = N->getOperand(2); |
| 4865 | SDLoc SL(N); |
| 4866 | SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, |
| 4867 | Src0, Src1); |
| 4868 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); |
| 4869 | return; |
| 4870 | } |
| 4871 | case Intrinsic::amdgcn_cvt_pknorm_i16: |
| 4872 | case Intrinsic::amdgcn_cvt_pknorm_u16: |
| 4873 | case Intrinsic::amdgcn_cvt_pk_i16: |
| 4874 | case Intrinsic::amdgcn_cvt_pk_u16: { |
| 4875 | SDValue Src0 = N->getOperand(1); |
| 4876 | SDValue Src1 = N->getOperand(2); |
| 4877 | SDLoc SL(N); |
| 4878 | unsigned Opcode; |
| 4879 | |
| 4880 | if (IID == Intrinsic::amdgcn_cvt_pknorm_i16) |
| 4881 | Opcode = AMDGPUISD::CVT_PKNORM_I16_F32; |
| 4882 | else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16) |
| 4883 | Opcode = AMDGPUISD::CVT_PKNORM_U16_F32; |
| 4884 | else if (IID == Intrinsic::amdgcn_cvt_pk_i16) |
| 4885 | Opcode = AMDGPUISD::CVT_PK_I16_I32; |
| 4886 | else |
| 4887 | Opcode = AMDGPUISD::CVT_PK_U16_U32; |
| 4888 | |
| 4889 | EVT VT = N->getValueType(0); |
| 4890 | if (isTypeLegal(VT)) |
| 4891 | Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1)); |
| 4892 | else { |
| 4893 | SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); |
| 4894 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); |
| 4895 | } |
| 4896 | return; |
| 4897 | } |
| 4898 | } |
| 4899 | break; |
| 4900 | } |
| 4901 | case ISD::INTRINSIC_W_CHAIN: { |
| 4902 | if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) { |
| 4903 | if (Res.getOpcode() == ISD::MERGE_VALUES) { |
| 4904 | // FIXME: Hacky |
| 4905 | for (unsigned I = 0; I < Res.getNumOperands(); I++) { |
| 4906 | Results.push_back(Res.getOperand(I)); |
| 4907 | } |
| 4908 | } else { |
| 4909 | Results.push_back(Res); |
| 4910 | Results.push_back(Res.getValue(1)); |
| 4911 | } |
| 4912 | return; |
| 4913 | } |
| 4914 | |
| 4915 | break; |
| 4916 | } |
| 4917 | case ISD::SELECT: { |
| 4918 | SDLoc SL(N); |
| 4919 | EVT VT = N->getValueType(0); |
| 4920 | EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT); |
| 4921 | SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1)); |
| 4922 | SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2)); |
| 4923 | |
| 4924 | EVT SelectVT = NewVT; |
| 4925 | if (NewVT.bitsLT(MVT::i32)) { |
| 4926 | LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); |
| 4927 | RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); |
| 4928 | SelectVT = MVT::i32; |
| 4929 | } |
| 4930 | |
| 4931 | SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT, |
| 4932 | N->getOperand(0), LHS, RHS); |
| 4933 | |
| 4934 | if (NewVT != SelectVT) |
| 4935 | NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect); |
| 4936 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect)); |
| 4937 | return; |
| 4938 | } |
| 4939 | case ISD::FNEG: { |
| 4940 | if (N->getValueType(0) != MVT::v2f16) |
| 4941 | break; |
| 4942 | |
| 4943 | SDLoc SL(N); |
| 4944 | SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); |
| 4945 | |
| 4946 | SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, |
| 4947 | BC, |
| 4948 | DAG.getConstant(0x80008000, SL, MVT::i32)); |
| 4949 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); |
| 4950 | return; |
| 4951 | } |
| 4952 | case ISD::FABS: { |
| 4953 | if (N->getValueType(0) != MVT::v2f16) |
| 4954 | break; |
| 4955 | |
| 4956 | SDLoc SL(N); |
| 4957 | SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); |
| 4958 | |
| 4959 | SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, |
| 4960 | BC, |
| 4961 | DAG.getConstant(0x7fff7fff, SL, MVT::i32)); |
| 4962 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); |
| 4963 | return; |
| 4964 | } |
| 4965 | default: |
| 4966 | break; |
| 4967 | } |
| 4968 | } |
| 4969 | |
| 4970 | /// Helper function for LowerBRCOND |
| 4971 | static SDNode *findUser(SDValue Value, unsigned Opcode) { |
| 4972 | |
| 4973 | SDNode *Parent = Value.getNode(); |
| 4974 | for (SDNode::use_iterator I = Parent->use_begin(), E = Parent->use_end(); |
| 4975 | I != E; ++I) { |
| 4976 | |
| 4977 | if (I.getUse().get() != Value) |
| 4978 | continue; |
| 4979 | |
| 4980 | if (I->getOpcode() == Opcode) |
| 4981 | return *I; |
| 4982 | } |
| 4983 | return nullptr; |
| 4984 | } |
| 4985 | |
| 4986 | unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const { |
| 4987 | if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) { |
| 4988 | switch (cast<ConstantSDNode>(Intr->getOperand(1))->getZExtValue()) { |
| 4989 | case Intrinsic::amdgcn_if: |
| 4990 | return AMDGPUISD::IF; |
| 4991 | case Intrinsic::amdgcn_else: |
| 4992 | return AMDGPUISD::ELSE; |
| 4993 | case Intrinsic::amdgcn_loop: |
| 4994 | return AMDGPUISD::LOOP; |
| 4995 | case Intrinsic::amdgcn_end_cf: |
| 4996 | llvm_unreachable("should not occur")__builtin_unreachable(); |
| 4997 | default: |
| 4998 | return 0; |
| 4999 | } |
| 5000 | } |
| 5001 | |
| 5002 | // break, if_break, else_break are all only used as inputs to loop, not |
| 5003 | // directly as branch conditions. |
| 5004 | return 0; |
| 5005 | } |
| 5006 | |
| 5007 | bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const { |
| 5008 | const Triple &TT = getTargetMachine().getTargetTriple(); |
| 5009 | return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || |
| 5010 | GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && |
| 5011 | AMDGPU::shouldEmitConstantsToTextSection(TT); |
| 5012 | } |
| 5013 | |
| 5014 | bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const { |
| 5015 | // FIXME: Either avoid relying on address space here or change the default |
| 5016 | // address space for functions to avoid the explicit check. |
| 5017 | return (GV->getValueType()->isFunctionTy() || |
| 5018 | !isNonGlobalAddrSpace(GV->getAddressSpace())) && |
| 5019 | !shouldEmitFixup(GV) && |
| 5020 | !getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV); |
| 5021 | } |
| 5022 | |
| 5023 | bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const { |
| 5024 | return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV); |
| 5025 | } |
| 5026 | |
| 5027 | bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const { |
| 5028 | if (!GV->hasExternalLinkage()) |
| 5029 | return true; |
| 5030 | |
| 5031 | const auto OS = getTargetMachine().getTargetTriple().getOS(); |
| 5032 | return OS == Triple::AMDHSA || OS == Triple::AMDPAL; |
| 5033 | } |
| 5034 | |
| 5035 | /// This transforms the control flow intrinsics to get the branch destination as |
| 5036 | /// last parameter, also switches branch target with BR if the need arise |
| 5037 | SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, |
| 5038 | SelectionDAG &DAG) const { |
| 5039 | SDLoc DL(BRCOND); |
| 5040 | |
| 5041 | SDNode *Intr = BRCOND.getOperand(1).getNode(); |
| 5042 | SDValue Target = BRCOND.getOperand(2); |
| 5043 | SDNode *BR = nullptr; |
| 5044 | SDNode *SetCC = nullptr; |
| 5045 | |
| 5046 | if (Intr->getOpcode() == ISD::SETCC) { |
| 5047 | // As long as we negate the condition everything is fine |
| 5048 | SetCC = Intr; |
| 5049 | Intr = SetCC->getOperand(0).getNode(); |
| 5050 | |
| 5051 | } else { |
| 5052 | // Get the target from BR if we don't negate the condition |
| 5053 | BR = findUser(BRCOND, ISD::BR); |
| 5054 | assert(BR && "brcond missing unconditional branch user")((void)0); |
| 5055 | Target = BR->getOperand(1); |
| 5056 | } |
| 5057 | |
| 5058 | unsigned CFNode = isCFIntrinsic(Intr); |
| 5059 | if (CFNode == 0) { |
| 5060 | // This is a uniform branch so we don't need to legalize. |
| 5061 | return BRCOND; |
| 5062 | } |
| 5063 | |
| 5064 | bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID || |
| 5065 | Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN; |
| 5066 | |
| 5067 | assert(!SetCC ||((void)0) |
| 5068 | (SetCC->getConstantOperandVal(1) == 1 &&((void)0) |
| 5069 | cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==((void)0) |
| 5070 | ISD::SETNE))((void)0); |
| 5071 | |
| 5072 | // operands of the new intrinsic call |
| 5073 | SmallVector<SDValue, 4> Ops; |
| 5074 | if (HaveChain) |
| 5075 | Ops.push_back(BRCOND.getOperand(0)); |
| 5076 | |
| 5077 | Ops.append(Intr->op_begin() + (HaveChain ? 2 : 1), Intr->op_end()); |
| 5078 | Ops.push_back(Target); |
| 5079 | |
| 5080 | ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end()); |
| 5081 | |
| 5082 | // build the new intrinsic call |
| 5083 | SDNode *Result = DAG.getNode(CFNode, DL, DAG.getVTList(Res), Ops).getNode(); |
| 5084 | |
| 5085 | if (!HaveChain) { |
| 5086 | SDValue Ops[] = { |
| 5087 | SDValue(Result, 0), |
| 5088 | BRCOND.getOperand(0) |
| 5089 | }; |
| 5090 | |
| 5091 | Result = DAG.getMergeValues(Ops, DL).getNode(); |
| 5092 | } |
| 5093 | |
| 5094 | if (BR) { |
| 5095 | // Give the branch instruction our target |
| 5096 | SDValue Ops[] = { |
| 5097 | BR->getOperand(0), |
| 5098 | BRCOND.getOperand(2) |
| 5099 | }; |
| 5100 | SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops); |
| 5101 | DAG.ReplaceAllUsesWith(BR, NewBR.getNode()); |
| 5102 | } |
| 5103 | |
| 5104 | SDValue Chain = SDValue(Result, Result->getNumValues() - 1); |
| 5105 | |
| 5106 | // Copy the intrinsic results to registers |
| 5107 | for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) { |
| 5108 | SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg); |
| 5109 | if (!CopyToReg) |
| 5110 | continue; |
| 5111 | |
| 5112 | Chain = DAG.getCopyToReg( |
| 5113 | Chain, DL, |
| 5114 | CopyToReg->getOperand(1), |
| 5115 | SDValue(Result, i - 1), |
| 5116 | SDValue()); |
| 5117 | |
| 5118 | DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0)); |
| 5119 | } |
| 5120 | |
| 5121 | // Remove the old intrinsic from the chain |
| 5122 | DAG.ReplaceAllUsesOfValueWith( |
| 5123 | SDValue(Intr, Intr->getNumValues() - 1), |
| 5124 | Intr->getOperand(0)); |
| 5125 | |
| 5126 | return Chain; |
| 5127 | } |
| 5128 | |
| 5129 | SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, |
| 5130 | SelectionDAG &DAG) const { |
| 5131 | MVT VT = Op.getSimpleValueType(); |
| 5132 | SDLoc DL(Op); |
| 5133 | // Checking the depth |
| 5134 | if (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() != 0) |
| 5135 | return DAG.getConstant(0, DL, VT); |
| 5136 | |
| 5137 | MachineFunction &MF = DAG.getMachineFunction(); |
| 5138 | const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 5139 | // Check for kernel and shader functions |
| 5140 | if (Info->isEntryFunction()) |
| 5141 | return DAG.getConstant(0, DL, VT); |
| 5142 | |
| 5143 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 5144 | // There is a call to @llvm.returnaddress in this function |
| 5145 | MFI.setReturnAddressIsTaken(true); |
| 5146 | |
| 5147 | const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); |
| 5148 | // Get the return address reg and mark it as an implicit live-in |
| 5149 | Register Reg = MF.addLiveIn(TRI->getReturnAddressReg(MF), getRegClassFor(VT, Op.getNode()->isDivergent())); |
| 5150 | |
| 5151 | return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT); |
| 5152 | } |
| 5153 | |
| 5154 | SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, |
| 5155 | SDValue Op, |
| 5156 | const SDLoc &DL, |
| 5157 | EVT VT) const { |
| 5158 | return Op.getValueType().bitsLE(VT) ? |
| 5159 | DAG.getNode(ISD::FP_EXTEND, DL, VT, Op) : |
| 5160 | DAG.getNode(ISD::FP_ROUND, DL, VT, Op, |
| 5161 | DAG.getTargetConstant(0, DL, MVT::i32)); |
| 5162 | } |
| 5163 | |
| 5164 | SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { |
| 5165 | assert(Op.getValueType() == MVT::f16 &&((void)0) |
| 5166 | "Do not know how to custom lower FP_ROUND for non-f16 type")((void)0); |
| 5167 | |
| 5168 | SDValue Src = Op.getOperand(0); |
| 5169 | EVT SrcVT = Src.getValueType(); |
| 5170 | if (SrcVT != MVT::f64) |
| 5171 | return Op; |
| 5172 | |
| 5173 | SDLoc DL(Op); |
| 5174 | |
| 5175 | SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); |
| 5176 | SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); |
| 5177 | return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); |
| 5178 | } |
| 5179 | |
| 5180 | SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op, |
| 5181 | SelectionDAG &DAG) const { |
| 5182 | EVT VT = Op.getValueType(); |
| 5183 | const MachineFunction &MF = DAG.getMachineFunction(); |
| 5184 | const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 5185 | bool IsIEEEMode = Info->getMode().IEEE; |
| 5186 | |
| 5187 | // FIXME: Assert during selection that this is only selected for |
| 5188 | // ieee_mode. Currently a combine can produce the ieee version for non-ieee |
| 5189 | // mode functions, but this happens to be OK since it's only done in cases |
| 5190 | // where there is known no sNaN. |
| 5191 | if (IsIEEEMode) |
| 5192 | return expandFMINNUM_FMAXNUM(Op.getNode(), DAG); |
| 5193 | |
| 5194 | if (VT == MVT::v4f16) |
| 5195 | return splitBinaryVectorOp(Op, DAG); |
| 5196 | return Op; |
| 5197 | } |
| 5198 | |
| 5199 | SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const { |
| 5200 | EVT VT = Op.getValueType(); |
| 5201 | SDLoc SL(Op); |
| 5202 | SDValue LHS = Op.getOperand(0); |
| 5203 | SDValue RHS = Op.getOperand(1); |
| 5204 | bool isSigned = Op.getOpcode() == ISD::SMULO; |
| 5205 | |
| 5206 | if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) { |
| 5207 | const APInt &C = RHSC->getAPIntValue(); |
| 5208 | // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X } |
| 5209 | if (C.isPowerOf2()) { |
| 5210 | // smulo(x, signed_min) is same as umulo(x, signed_min). |
| 5211 | bool UseArithShift = isSigned && !C.isMinSignedValue(); |
| 5212 | SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32); |
| 5213 | SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt); |
| 5214 | SDValue Overflow = DAG.getSetCC(SL, MVT::i1, |
| 5215 | DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, |
| 5216 | SL, VT, Result, ShiftAmt), |
| 5217 | LHS, ISD::SETNE); |
| 5218 | return DAG.getMergeValues({ Result, Overflow }, SL); |
| 5219 | } |
| 5220 | } |
| 5221 | |
| 5222 | SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS); |
| 5223 | SDValue Top = DAG.getNode(isSigned ? ISD::MULHS : ISD::MULHU, |
| 5224 | SL, VT, LHS, RHS); |
| 5225 | |
| 5226 | SDValue Sign = isSigned |
| 5227 | ? DAG.getNode(ISD::SRA, SL, VT, Result, |
| 5228 | DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32)) |
| 5229 | : DAG.getConstant(0, SL, VT); |
| 5230 | SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE); |
| 5231 | |
| 5232 | return DAG.getMergeValues({ Result, Overflow }, SL); |
| 5233 | } |
| 5234 | |
| 5235 | SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const { |
| 5236 | if (!Subtarget->isTrapHandlerEnabled() || |
| 5237 | Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) |
| 5238 | return lowerTrapEndpgm(Op, DAG); |
| 5239 | |
| 5240 | if (Optional<uint8_t> HsaAbiVer = AMDGPU::getHsaAbiVersion(Subtarget)) { |
| 5241 | switch (*HsaAbiVer) { |
| 5242 | case ELF::ELFABIVERSION_AMDGPU_HSA_V2: |
| 5243 | case ELF::ELFABIVERSION_AMDGPU_HSA_V3: |
| 5244 | return lowerTrapHsaQueuePtr(Op, DAG); |
| 5245 | case ELF::ELFABIVERSION_AMDGPU_HSA_V4: |
| 5246 | return Subtarget->supportsGetDoorbellID() ? |
| 5247 | lowerTrapHsa(Op, DAG) : lowerTrapHsaQueuePtr(Op, DAG); |
| 5248 | } |
| 5249 | } |
| 5250 | |
| 5251 | llvm_unreachable("Unknown trap handler")__builtin_unreachable(); |
| 5252 | } |
| 5253 | |
| 5254 | SDValue SITargetLowering::lowerTrapEndpgm( |
| 5255 | SDValue Op, SelectionDAG &DAG) const { |
| 5256 | SDLoc SL(Op); |
| 5257 | SDValue Chain = Op.getOperand(0); |
| 5258 | return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain); |
| 5259 | } |
| 5260 | |
| 5261 | SDValue SITargetLowering::lowerTrapHsaQueuePtr( |
| 5262 | SDValue Op, SelectionDAG &DAG) const { |
| 5263 | SDLoc SL(Op); |
| 5264 | SDValue Chain = Op.getOperand(0); |
| 5265 | |
| 5266 | MachineFunction &MF = DAG.getMachineFunction(); |
| 5267 | SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 5268 | Register UserSGPR = Info->getQueuePtrUserSGPR(); |
| 5269 | assert(UserSGPR != AMDGPU::NoRegister)((void)0); |
| 5270 | SDValue QueuePtr = CreateLiveInRegister( |
| 5271 | DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); |
| 5272 | SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64); |
| 5273 | SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, |
| 5274 | QueuePtr, SDValue()); |
| 5275 | |
| 5276 | uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap); |
| 5277 | SDValue Ops[] = { |
| 5278 | ToReg, |
| 5279 | DAG.getTargetConstant(TrapID, SL, MVT::i16), |
| 5280 | SGPR01, |
| 5281 | ToReg.getValue(1) |
| 5282 | }; |
| 5283 | return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); |
| 5284 | } |
| 5285 | |
| 5286 | SDValue SITargetLowering::lowerTrapHsa( |
| 5287 | SDValue Op, SelectionDAG &DAG) const { |
| 5288 | SDLoc SL(Op); |
| 5289 | SDValue Chain = Op.getOperand(0); |
| 5290 | |
| 5291 | uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap); |
| 5292 | SDValue Ops[] = { |
| 5293 | Chain, |
| 5294 | DAG.getTargetConstant(TrapID, SL, MVT::i16) |
| 5295 | }; |
| 5296 | return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); |
| 5297 | } |
| 5298 | |
| 5299 | SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const { |
| 5300 | SDLoc SL(Op); |
| 5301 | SDValue Chain = Op.getOperand(0); |
| 5302 | MachineFunction &MF = DAG.getMachineFunction(); |
| 5303 | |
| 5304 | if (!Subtarget->isTrapHandlerEnabled() || |
| 5305 | Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) { |
| 5306 | DiagnosticInfoUnsupported NoTrap(MF.getFunction(), |
| 5307 | "debugtrap handler not supported", |
| 5308 | Op.getDebugLoc(), |
| 5309 | DS_Warning); |
| 5310 | LLVMContext &Ctx = MF.getFunction().getContext(); |
| 5311 | Ctx.diagnose(NoTrap); |
| 5312 | return Chain; |
| 5313 | } |
| 5314 | |
| 5315 | uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap); |
| 5316 | SDValue Ops[] = { |
| 5317 | Chain, |
| 5318 | DAG.getTargetConstant(TrapID, SL, MVT::i16) |
| 5319 | }; |
| 5320 | return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); |
| 5321 | } |
| 5322 | |
| 5323 | SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL, |
| 5324 | SelectionDAG &DAG) const { |
| 5325 | // FIXME: Use inline constants (src_{shared, private}_base) instead. |
| 5326 | if (Subtarget->hasApertureRegs()) { |
| 5327 | unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ? |
| 5328 | AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE : |
| 5329 | AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE; |
| 5330 | unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ? |
| 5331 | AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE : |
| 5332 | AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE; |
| 5333 | unsigned Encoding = |
| 5334 | AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ | |
| 5335 | Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ | |
| 5336 | WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_; |
| 5337 | |
| 5338 | SDValue EncodingImm = DAG.getTargetConstant(Encoding, DL, MVT::i16); |
| 5339 | SDValue ApertureReg = SDValue( |
| 5340 | DAG.getMachineNode(AMDGPU::S_GETREG_B32, DL, MVT::i32, EncodingImm), 0); |
| 5341 | SDValue ShiftAmount = DAG.getTargetConstant(WidthM1 + 1, DL, MVT::i32); |
| 5342 | return DAG.getNode(ISD::SHL, DL, MVT::i32, ApertureReg, ShiftAmount); |
| 5343 | } |
| 5344 | |
| 5345 | MachineFunction &MF = DAG.getMachineFunction(); |
| 5346 | SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); |
| 5347 | Register UserSGPR = Info->getQueuePtrUserSGPR(); |
| 5348 | assert(UserSGPR != AMDGPU::NoRegister)((void)0); |
| 5349 | |
| 5350 | SDValue QueuePtr = CreateLiveInRegister( |
| 5351 | DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); |
| 5352 | |
| 5353 | // Offset into amd_queue_t for group_segment_aperture_base_hi / |
| 5354 | // private_segment_aperture_base_hi. |
| 5355 | uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44; |
| 5356 | |
| 5357 | SDValue Ptr = |
| 5358 | DAG.getObjectPtrOffset(DL, QueuePtr, TypeSize::Fixed(StructOffset)); |
| 5359 | |
| 5360 | // TODO: Use custom target PseudoSourceValue. |
| 5361 | // TODO: We should use the value from the IR intrinsic call, but it might not |
| 5362 | // be available and how do we get it? |
| 5363 | MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); |
| 5364 | return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo, |
| 5365 | commonAlignment(Align(64), StructOffset), |
| 5366 | MachineMemOperand::MODereferenceable | |
| 5367 | MachineMemOperand::MOInvariant); |
| 5368 | } |
| 5369 | |
| 5370 | SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op, |
| 5371 | SelectionDAG &DAG) const { |
| 5372 | SDLoc SL(Op); |
| 5373 | const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Op); |
| 5374 | |
| 5375 | SDValue Src = ASC->getOperand(0); |
| 5376 | SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); |
| 5377 | |
| 5378 | const AMDGPUTargetMachine &TM = |
| 5379 | static_cast<const AMDGPUTargetMachine &>(getTargetMachine()); |
| 5380 | |
| 5381 | // flat -> local/private |
| 5382 | if (ASC->getSrcAddressSpace() == AMDGPUAS::FLAT_ADDRESS) { |
| 5383 | unsigned DestAS = ASC->getDestAddressSpace(); |
| 5384 | |
| 5385 | if (DestAS == AMDGPUAS::LOCAL_ADDRESS || |
| 5386 | DestAS == AMDGPUAS::PRIVATE_ADDRESS) { |
| 5387 | unsigned NullVal = TM.getNullPointerValue(DestAS); |
| 5388 | SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); |
| 5389 | SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); |
| 5390 | SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); |
| 5391 | |
| 5392 | return DAG.getNode(ISD::SELECT, SL, MVT::i32, |
| 5393 | NonNull, Ptr, SegmentNullPtr); |
| 5394 | } |
| 5395 | } |
| 5396 | |
| 5397 | // local/private -> flat |
| 5398 | if (ASC->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS) { |
| 5399 | unsigned SrcAS = ASC->getSrcAddressSpace(); |
| 5400 | |
| 5401 | if (SrcAS == AMDGPUAS::LOCAL_ADDRESS || |
| 5402 | SrcAS == AMDGPUAS::PRIVATE_ADDRESS) { |
| 5403 | unsigned NullVal = TM.getNullPointerValue(SrcAS); |
| 5404 | SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); |
| 5405 | |
| 5406 | SDValue NonNull |
| 5407 | = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); |
| 5408 | |
| 5409 | SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG); |
| 5410 | SDValue CvtPtr |
| 5411 | = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); |
| 5412 | |
| 5413 | return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, |
| 5414 | DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr), |
| 5415 | FlatNullPtr); |
| 5416 | } |
| 5417 | } |
| 5418 | |
| 5419 | if (ASC->getDestAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT && |
| 5420 | Src.getValueType() == MVT::i64) |
| 5421 | return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); |
| 5422 | |
| 5423 | // global <-> flat are no-ops and never emitted. |
| 5424 | |
| 5425 | const MachineFunction &MF = DAG.getMachineFunction(); |
| 5426 | DiagnosticInfoUnsupported InvalidAddrSpaceCast( |
| 5427 | MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc()); |
| 5428 | DAG.getContext()->diagnose(InvalidAddrSpaceCast); |
| 5429 | |
| 5430 | return DAG.getUNDEF(ASC->getValueType(0)); |
| 5431 | } |
| 5432 | |
| 5433 | // This lowers an INSERT_SUBVECTOR by extracting the individual elements from |
| 5434 | // the small vector and inserting them into the big vector. That is better than |
| 5435 | // the default expansion of doing it via a stack slot. Even though the use of |
| 5436 | // the stack slot would be optimized away afterwards, the stack slot itself |
| 5437 | // remains. |
| 5438 | SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op, |
| 5439 | SelectionDAG &DAG) const { |
| 5440 | SDValue Vec = Op.getOperand(0); |
| 5441 | SDValue Ins = Op.getOperand(1); |
| 5442 | SDValue Idx = Op.getOperand(2); |
| 5443 | EVT VecVT = Vec.getValueType(); |
| 5444 | EVT InsVT = Ins.getValueType(); |
| 5445 | EVT EltVT = VecVT.getVectorElementType(); |
| 5446 | unsigned InsNumElts = InsVT.getVectorNumElements(); |
| 5447 | unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); |
| 5448 | SDLoc SL(Op); |
| 5449 | |
| 5450 | for (unsigned I = 0; I != InsNumElts; ++I) { |
| 5451 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins, |
| 5452 | DAG.getConstant(I, SL, MVT::i32)); |
| 5453 | Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt, |
| 5454 | DAG.getConstant(IdxVal + I, SL, MVT::i32)); |
| 5455 | } |
| 5456 | return Vec; |
| 5457 | } |
| 5458 | |
| 5459 | SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op, |
| 5460 | SelectionDAG &DAG) const { |
| 5461 | SDValue Vec = Op.getOperand(0); |
| 5462 | SDValue InsVal = Op.getOperand(1); |
| 5463 | SDValue Idx = Op.getOperand(2); |
| 5464 | EVT VecVT = Vec.getValueType(); |
| 5465 | EVT EltVT = VecVT.getVectorElementType(); |
| 5466 | unsigned VecSize = VecVT.getSizeInBits(); |
| 5467 | unsigned EltSize = EltVT.getSizeInBits(); |
| 5468 | |
| 5469 | |
| 5470 | assert(VecSize <= 64)((void)0); |
| 5471 | |
| 5472 | unsigned NumElts = VecVT.getVectorNumElements(); |
| 5473 | SDLoc SL(Op); |
| 5474 | auto KIdx = dyn_cast<ConstantSDNode>(Idx); |
| 5475 | |
| 5476 | if (NumElts == 4 && EltSize == 16 && KIdx) { |
| 5477 | SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); |
| 5478 | |
| 5479 | SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, |
| 5480 | DAG.getConstant(0, SL, MVT::i32)); |
| 5481 | SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, |
| 5482 | DAG.getConstant(1, SL, MVT::i32)); |
| 5483 | |
| 5484 | SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); |
| 5485 | SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); |
| 5486 | |
| 5487 | unsigned Idx = KIdx->getZExtValue(); |
| 5488 | bool InsertLo = Idx < 2; |
| 5489 | SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, |
| 5490 | InsertLo ? LoVec : HiVec, |
| 5491 | DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), |
| 5492 | DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); |
| 5493 | |
| 5494 | InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); |
| 5495 | |
| 5496 | SDValue Concat = InsertLo ? |
| 5497 | DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : |
| 5498 | DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); |
| 5499 | |
| 5500 | return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat); |
| 5501 | } |
| 5502 | |
| 5503 | if (isa<ConstantSDNode>(Idx)) |
| 5504 | return SDValue(); |
| 5505 | |
| 5506 | MVT IntVT = MVT::getIntegerVT(VecSize); |
| 5507 | |
| 5508 | // Avoid stack access for dynamic indexing. |
| 5509 | // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec |
| 5510 | |
| 5511 | // Create a congruent vector with the target value in each element so that |
| 5512 | // the required element can be masked and ORed into the target vector. |
| 5513 | SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT, |
| 5514 | DAG.getSplatBuildVector(VecVT, SL, InsVal)); |
| 5515 | |
| 5516 | assert(isPowerOf2_32(EltSize))((void)0); |
| 5517 | SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); |
| 5518 | |
| 5519 | // Convert vector index to bit-index. |
| 5520 | SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); |
| 5521 | |
| 5522 | SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); |
| 5523 | SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT, |
| 5524 | DAG.getConstant(0xffff, SL, IntVT), |
| 5525 | ScaledIdx); |
| 5526 | |
| 5527 | SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal); |
| 5528 | SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT, |
| 5529 | DAG.getNOT(SL, BFM, IntVT), BCVec); |
| 5530 | |
| 5531 | SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS); |
| 5532 | return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI); |
| 5533 | } |
| 5534 | |
| 5535 | SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op, |
| 5536 | SelectionDAG &DAG) const { |
| 5537 | SDLoc SL(Op); |
| 5538 | |
| 5539 | EVT ResultVT = Op.getValueType(); |
| 5540 | SDValue Vec = Op.getOperand(0); |
| 5541 | SDValue Idx = Op.getOperand(1); |
| 5542 | EVT VecVT = Vec.getValueType(); |
| 5543 | unsigned VecSize = VecVT.getSizeInBits(); |
| 5544 | EVT EltVT = VecVT.getVectorElementType(); |
| 5545 | assert(VecSize <= 64)((void)0); |
| 5546 | |
| 5547 | DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr); |
| 5548 | |
| 5549 | // Make sure we do any optimizations that will make it easier to fold |
| 5550 | // source modifiers before obscuring it with bit operations. |
| 5551 | |
| 5552 | // XXX - Why doesn't this get called when vector_shuffle is expanded? |
| 5553 | if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI)) |
| 5554 | return Combined; |
| 5555 | |
| 5556 | unsigned EltSize = EltVT.getSizeInBits(); |
| 5557 | assert(isPowerOf2_32(EltSize))((void)0); |
| 5558 | |
| 5559 | MVT IntVT = MVT::getIntegerVT(VecSize); |
| 5560 | SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); |
| 5561 | |
| 5562 | // Convert vector index to bit-index (* EltSize) |
| 5563 | SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); |
| 5564 | |
| 5565 | SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec); |
| 5566 | SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx); |
| 5567 | |
| 5568 | if (ResultVT == MVT::f16) { |
| 5569 | SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); |
| 5570 | return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result); |
| 5571 | } |
| 5572 | |
| 5573 | return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT); |
| 5574 | } |
| 5575 | |
| 5576 | static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) { |
| 5577 | assert(Elt % 2 == 0)((void)0); |
| 5578 | return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0); |
| 5579 | } |
| 5580 | |
| 5581 | SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op, |
| 5582 | SelectionDAG &DAG) const { |
| 5583 | SDLoc SL(Op); |
| 5584 | EVT ResultVT = Op.getValueType(); |
| 5585 | ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); |
| 5586 | |
| 5587 | EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16; |
| 5588 | EVT EltVT = PackVT.getVectorElementType(); |
| 5589 | int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements(); |
| 5590 | |
| 5591 | // vector_shuffle <0,1,6,7> lhs, rhs |
| 5592 | // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2) |
| 5593 | // |
| 5594 | // vector_shuffle <6,7,2,3> lhs, rhs |
| 5595 | // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2) |
| 5596 | // |
| 5597 | // vector_shuffle <6,7,0,1> lhs, rhs |
| 5598 | // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0) |
| 5599 | |
| 5600 | // Avoid scalarizing when both halves are reading from consecutive elements. |
| 5601 | SmallVector<SDValue, 4> Pieces; |
| 5602 | for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) { |
| 5603 | if (elementPairIsContiguous(SVN->getMask(), I)) { |
| 5604 | const int Idx = SVN->getMaskElt(I); |
| 5605 | int VecIdx = Idx < SrcNumElts ? 0 : 1; |
| 5606 | int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts; |
| 5607 | SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, |
| 5608 | PackVT, SVN->getOperand(VecIdx), |
| 5609 | DAG.getConstant(EltIdx, SL, MVT::i32)); |
| 5610 | Pieces.push_back(SubVec); |
| 5611 | } else { |
| 5612 | const int Idx0 = SVN->getMaskElt(I); |
| 5613 | const int Idx1 = SVN->getMaskElt(I + 1); |
| 5614 | int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1; |
| 5615 | int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1; |
| 5616 | int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts; |
| 5617 | int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts; |
| 5618 | |
| 5619 | SDValue Vec0 = SVN->getOperand(VecIdx0); |
| 5620 | SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, |
| 5621 | Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32)); |
| 5622 | |
| 5623 | SDValue Vec1 = SVN->getOperand(VecIdx1); |
| 5624 | SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, |
| 5625 | Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32)); |
| 5626 | Pieces.push_back(DAG.getBuildVector(PackVT, SL, { Elt0, Elt1 })); |
| 5627 | } |
| 5628 | } |
| 5629 | |
| 5630 | return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces); |
| 5631 | } |
| 5632 | |
| 5633 | SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op, |
| 5634 | SelectionDAG &DAG) const { |
| 5635 | SDLoc SL(Op); |
| 5636 | EVT VT = Op.getValueType(); |
| 5637 | |
| 5638 | if (VT == MVT::v4i16 || VT == MVT::v4f16) { |
| 5639 | EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), 2); |
| 5640 | |
| 5641 | // Turn into pair of packed build_vectors. |
| 5642 | // TODO: Special case for constants that can be materialized with s_mov_b64. |
| 5643 | SDValue Lo = DAG.getBuildVector(HalfVT, SL, |
| 5644 | { Op.getOperand(0), Op.getOperand(1) }); |
| 5645 | SDValue Hi = DAG.getBuildVector(HalfVT, SL, |
| 5646 | { Op.getOperand(2), Op.getOperand(3) }); |
| 5647 | |
| 5648 | SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo); |
| 5649 | SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi); |
| 5650 | |
| 5651 | SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi }); |
| 5652 | return DAG.getNode(ISD::BITCAST, SL, VT, Blend); |
| 5653 | } |
| 5654 | |
| 5655 | assert(VT == MVT::v2f16 || VT == MVT::v2i16)((void)0); |
| 5656 | assert(!Subtarget->hasVOP3PInsts() && "this should be legal")((void)0); |
| 5657 | |
| 5658 | SDValue Lo = Op.getOperand(0); |
| 5659 | SDValue Hi = Op.getOperand(1); |
| 5660 | |
| 5661 | // Avoid adding defined bits with the zero_extend. |
| 5662 | if (Hi.isUndef()) { |
| 5663 | Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); |
| 5664 | SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); |
| 5665 | return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo); |
| 5666 | } |
| 5667 | |
| 5668 | Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); |
| 5669 | Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); |
| 5670 | |
| 5671 | SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, |
| 5672 | DAG.getConstant(16, SL, MVT::i32)); |
| 5673 | if (Lo.isUndef()) |
| 5674 | return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi); |
| 5675 | |
| 5676 | Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); |
| 5677 | Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); |
| 5678 | |
| 5679 | SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); |
| 5680 | return DAG.getNode(ISD::BITCAST, SL, VT, Or); |
| 5681 | } |
| 5682 | |
| 5683 | bool |
| 5684 | SITargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { |
| 5685 | // We can fold offsets for anything that doesn't require a GOT relocation. |
| 5686 | return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS || |
| 5687 | GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || |
| 5688 | GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && |
| 5689 | !shouldEmitGOTReloc(GA->getGlobal()); |
| 5690 | } |
| 5691 | |
| 5692 | static SDValue |
| 5693 | buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV, |
| 5694 | const SDLoc &DL, int64_t Offset, EVT PtrVT, |
| 5695 | unsigned GAFlags = SIInstrInfo::MO_NONE) { |
| 5696 | assert(isInt<32>(Offset + 4) && "32-bit offset is expected!")((void)0); |
| 5697 | // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is |
| 5698 | // lowered to the following code sequence: |
| 5699 | // |
| 5700 | // For constant address space: |
| 5701 | // s_getpc_b64 s[0:1] |
| 5702 | // s_add_u32 s0, s0, $symbol |
| 5703 | // s_addc_u32 s1, s1, 0 |
| 5704 | // |
| 5705 | // s_getpc_b64 returns the address of the s_add_u32 instruction and then |
| 5706 | // a fixup or relocation is emitted to replace $symbol with a literal |
| 5707 | // constant, which is a pc-relative offset from the encoding of the $symbol |
| 5708 | // operand to the global variable. |
| 5709 | // |
| 5710 | // For global address space: |
| 5711 | // s_getpc_b64 s[0:1] |
| 5712 | // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo |
| 5713 | // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi |
| 5714 | // |
| 5715 | // s_getpc_b64 returns the address of the s_add_u32 instruction and then |
| 5716 | // fixups or relocations are emitted to replace $symbol@*@lo and |
| 5717 | // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant, |
| 5718 | // which is a 64-bit pc-relative offset from the encoding of the $symbol |
| 5719 | // operand to the global variable. |
| 5720 | // |
| 5721 | // What we want here is an offset from the value returned by s_getpc |
| 5722 | // (which is the address of the s_add_u32 instruction) to the global |
| 5723 | // variable, but since the encoding of $symbol starts 4 bytes after the start |
| 5724 | // of the s_add_u32 instruction, we end up with an offset that is 4 bytes too |
| 5725 | // small. This requires us to add 4 to the global variable offset in order to |
| 5726 | // compute the correct address. Similarly for the s_addc_u32 instruction, the |
| 5727 | // encoding of $symbol starts 12 bytes after the start of the s_add_u32 |
| 5728 | // instruction. |
| 5729 | SDValue PtrLo = |
| 5730 | DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, GAFlags); |
| 5731 | SDValue PtrHi; |
| 5732 | if (GAFlags == SIInstrInfo::MO_NONE) { |
| 5733 | PtrHi = DAG.getTargetConstant(0, DL, MVT::i32); |
| 5734 | } else { |
| 5735 | PtrHi = |
| 5736 | DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 12, GAFlags + 1); |
| 5737 | } |
| 5738 | return DAG.getNode(AMDGPUISD::PC_ADD_REL_OFFSET, DL, PtrVT, PtrLo, PtrHi); |
| 5739 | } |
| 5740 | |
| 5741 | SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI, |
| 5742 | SDValue Op, |
| 5743 | SelectionDAG &DAG) const { |
| 5744 | GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op); |
| 5745 | SDLoc DL(GSD); |
| 5746 | EVT PtrVT = Op.getValueType(); |
| 5747 | |
| 5748 | const GlobalValue *GV = GSD->getGlobal(); |
| 5749 | if ((GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && |
| 5750 | shouldUseLDSConstAddress(GV)) || |
| 5751 | GSD->getAddressSpace() == AMDGPUAS::REGION_ADDRESS || |
| 5752 | GSD->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) { |
| 5753 | if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && |
| 5754 | GV->hasExternalLinkage()) { |
| 5755 | Type *Ty = GV->getValueType(); |
| 5756 | // HIP uses an unsized array `extern __shared__ T s[]` or similar |
| 5757 | // zero-sized type in other languages to declare the dynamic shared |
| 5758 | // memory which size is not known at the compile time. They will be |
| 5759 | // allocated by the runtime and placed directly after the static |
| 5760 | // allocated ones. They all share the same offset. |
| 5761 | if (DAG.getDataLayout().getTypeAllocSize(Ty).isZero()) { |
| 5762 | assert(PtrVT == MVT::i32 && "32-bit pointer is expected.")((void)0); |
| 5763 | // Adjust alignment for that dynamic shared memory array. |
| 5764 | MFI->setDynLDSAlign(DAG.getDataLayout(), *cast<GlobalVariable>(GV)); |
| 5765 | return SDValue( |
| 5766 | DAG.getMachineNode(AMDGPU::GET_GROUPSTATICSIZE, DL, PtrVT), 0); |
| 5767 | } |
| 5768 | } |
| 5769 | return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG); |
| 5770 | } |
| 5771 | |
| 5772 | if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) { |
| 5773 | SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(), |
| 5774 | SIInstrInfo::MO_ABS32_LO); |
| 5775 | return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA); |
| 5776 | } |
| 5777 | |
| 5778 | if (shouldEmitFixup(GV)) |
| 5779 | return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT); |
| 5780 | else if (shouldEmitPCReloc(GV)) |
| 5781 | return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT, |
| 5782 | SIInstrInfo::MO_REL32); |
| 5783 | |
| 5784 | SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, 0, PtrVT, |
| 5785 | SIInstrInfo::MO_GOTPCREL32); |
| 5786 | |
| 5787 | Type *Ty = PtrVT.getTypeForEVT(*DAG.getContext()); |
| 5788 | PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS); |
| 5789 | const DataLayout &DataLayout = DAG.getDataLayout(); |
| 5790 | Align Alignment = DataLayout.getABITypeAlign(PtrTy); |
| 5791 | MachinePointerInfo PtrInfo |
| 5792 | = MachinePointerInfo::getGOT(DAG.getMachineFunction()); |
| 5793 | |
| 5794 | return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Alignment, |
| 5795 | MachineMemOperand::MODereferenceable | |
| 5796 | MachineMemOperand::MOInvariant); |
| 5797 | } |
| 5798 | |
| 5799 | SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain, |
| 5800 | const SDLoc &DL, SDValue V) const { |
| 5801 | // We can't use S_MOV_B32 directly, because there is no way to specify m0 as |
| 5802 | // the destination register. |
| 5803 | // |
| 5804 | // We can't use CopyToReg, because MachineCSE won't combine COPY instructions, |
| 5805 | // so we will end up with redundant moves to m0. |
| 5806 | // |
| 5807 | // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result. |
| 5808 | |
| 5809 | // A Null SDValue creates a glue result. |
| 5810 | SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue, |
| 5811 | V, Chain); |
| 5812 | return SDValue(M0, 0); |
| 5813 | } |
| 5814 | |
| 5815 | SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, |
| 5816 | SDValue Op, |
| 5817 | MVT VT, |
| 5818 | unsigned Offset) const { |
| 5819 | SDLoc SL(Op); |
| 5820 | SDValue Param = lowerKernargMemParameter( |
| 5821 | DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false); |
| 5822 | // The local size values will have the hi 16-bits as zero. |
| 5823 | return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, |
| 5824 | DAG.getValueType(VT)); |
| 5825 | } |
| 5826 | |
| 5827 | static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, |
| 5828 | EVT VT) { |
| 5829 | DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(), |
| 5830 | "non-hsa intrinsic with hsa target", |
| 5831 | DL.getDebugLoc()); |
| 5832 | DAG.getContext()->diagnose(BadIntrin); |
| 5833 | return DAG.getUNDEF(VT); |
| 5834 | } |
| 5835 | |
| 5836 | static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, |
| 5837 | EVT VT) { |
| 5838 | DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(), |
| 5839 | "intrinsic not supported on subtarget", |
| 5840 | DL.getDebugLoc()); |
| 5841 | DAG.getContext()->diagnose(BadIntrin); |
| 5842 | return DAG.getUNDEF(VT); |
| 5843 | } |
| 5844 | |
| 5845 | static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL, |
| 5846 | ArrayRef<SDValue> Elts) { |
| 5847 | assert(!Elts.empty())((void)0); |
| 5848 | MVT Type; |
| 5849 | unsigned NumElts = Elts.size(); |
| 5850 | |
| 5851 | if (NumElts <= 8) { |
| 5852 | Type = MVT::getVectorVT(MVT::f32, NumElts); |
| 5853 | } else { |
| 5854 | assert(Elts.size() <= 16)((void)0); |
| 5855 | Type = MVT::v16f32; |
| 5856 | NumElts = 16; |
| 5857 | } |
| 5858 | |
| 5859 | SmallVector<SDValue, 16> VecElts(NumElts); |
| 5860 | for (unsigned i = 0; i < Elts.size(); ++i) { |
| 5861 | SDValue Elt = Elts[i]; |
| 5862 | if (Elt.getValueType() != MVT::f32) |
| 5863 | Elt = DAG.getBitcast(MVT::f32, Elt); |
| 5864 | VecElts[i] = Elt; |
| 5865 | } |
| 5866 | for (unsigned i = Elts.size(); i < NumElts; ++i) |
| 5867 | VecElts[i] = DAG.getUNDEF(MVT::f32); |
| 5868 | |
| 5869 | if (NumElts == 1) |
| 5870 | return VecElts[0]; |
| 5871 | return DAG.getBuildVector(Type, DL, VecElts); |
| 5872 | } |
| 5873 | |
| 5874 | static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, |
| 5875 | SDValue Src, int ExtraElts) { |
| 5876 | EVT SrcVT = Src.getValueType(); |
| 5877 | |
| 5878 | SmallVector<SDValue, 8> Elts; |
| 5879 | |
| 5880 | if (SrcVT.isVector()) |
| 5881 | DAG.ExtractVectorElements(Src, Elts); |
| 5882 | else |
| 5883 | Elts.push_back(Src); |
| 5884 | |
| 5885 | SDValue Undef = DAG.getUNDEF(SrcVT.getScalarType()); |
| 5886 | while (ExtraElts--) |
| 5887 | Elts.push_back(Undef); |
| 5888 | |
| 5889 | return DAG.getBuildVector(CastVT, DL, Elts); |
| 5890 | } |
| 5891 | |
| 5892 | // Re-construct the required return value for a image load intrinsic. |
| 5893 | // This is more complicated due to the optional use TexFailCtrl which means the required |
| 5894 | // return type is an aggregate |
| 5895 | static SDValue constructRetValue(SelectionDAG &DAG, |
| 5896 | MachineSDNode *Result, |
| 5897 | ArrayRef<EVT> ResultTypes, |
| 5898 | bool IsTexFail, bool Unpacked, bool IsD16, |
| 5899 | int DMaskPop, int NumVDataDwords, |
| 5900 | const SDLoc &DL) { |
| 5901 | // Determine the required return type. This is the same regardless of IsTexFail flag |
| 5902 | EVT ReqRetVT = ResultTypes[0]; |
| 5903 | int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1; |
| 5904 | int NumDataDwords = (!IsD16 || (IsD16 && Unpacked)) ? |
| 5905 | ReqRetNumElts : (ReqRetNumElts + 1) / 2; |
| 5906 | |
| 5907 | int MaskPopDwords = (!IsD16 || (IsD16 && Unpacked)) ? |
| 5908 | DMaskPop : (DMaskPop + 1) / 2; |
| 5909 | |
| 5910 | MVT DataDwordVT = NumDataDwords == 1 ? |
| 5911 | MVT::i32 : MVT::getVectorVT(MVT::i32, NumDataDwords); |
| 5912 | |
| 5913 | MVT MaskPopVT = MaskPopDwords == 1 ? |
| 5914 | MVT::i32 : MVT::getVectorVT(MVT::i32, MaskPopDwords); |
| 5915 | |
| 5916 | SDValue Data(Result, 0); |
| 5917 | SDValue TexFail; |
| 5918 | |
| 5919 | if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) { |
| 5920 | SDValue ZeroIdx = DAG.getConstant(0, DL, MVT::i32); |
| 5921 | if (MaskPopVT.isVector()) { |
| 5922 | Data = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MaskPopVT, |
| 5923 | SDValue(Result, 0), ZeroIdx); |
| 5924 | } else { |
| 5925 | Data = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MaskPopVT, |
| 5926 | SDValue(Result, 0), ZeroIdx); |
| 5927 | } |
| 5928 | } |
| 5929 | |
| 5930 | if (DataDwordVT.isVector()) |
| 5931 | Data = padEltsToUndef(DAG, DL, DataDwordVT, Data, |
| 5932 | NumDataDwords - MaskPopDwords); |
| 5933 | |
| 5934 | if (IsD16) |
| 5935 | Data = adjustLoadValueTypeImpl(Data, ReqRetVT, DL, DAG, Unpacked); |
| 5936 | |
| 5937 | EVT LegalReqRetVT = ReqRetVT; |
| 5938 | if (!ReqRetVT.isVector()) { |
| 5939 | Data = DAG.getNode(ISD::TRUNCATE, DL, ReqRetVT.changeTypeToInteger(), Data); |
| 5940 | } else { |
| 5941 | // We need to widen the return vector to a legal type |
| 5942 | if ((ReqRetVT.getVectorNumElements() % 2) == 1 && |
| 5943 | ReqRetVT.getVectorElementType().getSizeInBits() == 16) { |
| 5944 | LegalReqRetVT = |
| 5945 | EVT::getVectorVT(*DAG.getContext(), ReqRetVT.getVectorElementType(), |
| 5946 | ReqRetVT.getVectorNumElements() + 1); |
| 5947 | } |
| 5948 | } |
| 5949 | Data = DAG.getNode(ISD::BITCAST, DL, LegalReqRetVT, Data); |
| 5950 | |
| 5951 | if (IsTexFail) { |
| 5952 | TexFail = |
| 5953 | DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, SDValue(Result, 0), |
| 5954 | DAG.getConstant(MaskPopDwords, DL, MVT::i32)); |
| 5955 | |
| 5956 | return DAG.getMergeValues({Data, TexFail, SDValue(Result, 1)}, DL); |
| 5957 | } |
| 5958 | |
| 5959 | if (Result->getNumValues() == 1) |
| 5960 | return Data; |
| 5961 | |
| 5962 | return DAG.getMergeValues({Data, SDValue(Result, 1)}, DL); |
| 5963 | } |
| 5964 | |
| 5965 | static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, |
| 5966 | SDValue *LWE, bool &IsTexFail) { |
| 5967 | auto TexFailCtrlConst = cast<ConstantSDNode>(TexFailCtrl.getNode()); |
| 5968 | |
| 5969 | uint64_t Value = TexFailCtrlConst->getZExtValue(); |
| 5970 | if (Value) { |
| 5971 | IsTexFail = true; |
| 5972 | } |
| 5973 | |
| 5974 | SDLoc DL(TexFailCtrlConst); |
| 5975 | *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); |
| 5976 | Value &= ~(uint64_t)0x1; |
| 5977 | *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); |
| 5978 | Value &= ~(uint64_t)0x2; |
| 5979 | |
| 5980 | return Value == 0; |
| 5981 | } |
| 5982 | |
| 5983 | static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op, |
| 5984 | MVT PackVectorVT, |
| 5985 | SmallVectorImpl<SDValue> &PackedAddrs, |
| 5986 | unsigned DimIdx, unsigned EndIdx, |
| 5987 | unsigned NumGradients) { |
| 5988 | SDLoc DL(Op); |
| 5989 | for (unsigned I = DimIdx; I < EndIdx; I++) { |
| 5990 | SDValue Addr = Op.getOperand(I); |
| 5991 | |
| 5992 | // Gradients are packed with undef for each coordinate. |
| 5993 | // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this: |
| 5994 | // 1D: undef,dx/dh; undef,dx/dv |
| 5995 | // 2D: dy/dh,dx/dh; dy/dv,dx/dv |
| 5996 | // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv |
| 5997 | if (((I + 1) >= EndIdx) || |
| 5998 | ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 || |
| 5999 | I == DimIdx + NumGradients - 1))) { |
| 6000 | if (Addr.getValueType() != MVT::i16) |
| 6001 | Addr = DAG.getBitcast(MVT::i16, Addr); |
| 6002 | Addr = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Addr); |
| 6003 | } else { |
| 6004 | Addr = DAG.getBuildVector(PackVectorVT, DL, {Addr, Op.getOperand(I + 1)}); |
| 6005 | I++; |
| 6006 | } |
| 6007 | Addr = DAG.getBitcast(MVT::f32, Addr); |
| 6008 | PackedAddrs.push_back(Addr); |
| 6009 | } |
| 6010 | } |
| 6011 | |
| 6012 | SDValue SITargetLowering::lowerImage(SDValue Op, |
| 6013 | const AMDGPU::ImageDimIntrinsicInfo *Intr, |
| 6014 | SelectionDAG &DAG, bool WithChain) const { |
| 6015 | SDLoc DL(Op); |
| 6016 | MachineFunction &MF = DAG.getMachineFunction(); |
| 6017 | const GCNSubtarget* ST = &MF.getSubtarget<GCNSubtarget>(); |
| 6018 | const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = |
| 6019 | AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); |
| 6020 | const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim); |
| 6021 | const AMDGPU::MIMGLZMappingInfo *LZMappingInfo = |
| 6022 | AMDGPU::getMIMGLZMappingInfo(Intr->BaseOpcode); |
| 6023 | const AMDGPU::MIMGMIPMappingInfo *MIPMappingInfo = |
| 6024 | AMDGPU::getMIMGMIPMappingInfo(Intr->BaseOpcode); |
| 6025 | unsigned IntrOpcode = Intr->BaseOpcode; |
| 6026 | bool IsGFX10Plus = AMDGPU::isGFX10Plus(*Subtarget); |
| 6027 | |
| 6028 | SmallVector<EVT, 3> ResultTypes(Op->values()); |
| 6029 | SmallVector<EVT, 3> OrigResultTypes(Op->values()); |
| 6030 | bool IsD16 = false; |
| 6031 | bool IsG16 = false; |
| 6032 | bool IsA16 = false; |
| 6033 | SDValue VData; |
| 6034 | int NumVDataDwords; |
| 6035 | bool AdjustRetType = false; |
| 6036 | |
| 6037 | // Offset of intrinsic arguments |
| 6038 | const unsigned ArgOffset = WithChain ? 2 : 1; |
| 6039 | |
| 6040 | unsigned DMask; |
| 6041 | unsigned DMaskLanes = 0; |
| 6042 | |
| 6043 | if (BaseOpcode->Atomic) { |
| 6044 | VData = Op.getOperand(2); |
| 6045 | |
| 6046 | bool Is64Bit = VData.getValueType() == MVT::i64; |
| 6047 | if (BaseOpcode->AtomicX2) { |
| 6048 | SDValue VData2 = Op.getOperand(3); |
| 6049 | VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL, |
| 6050 | {VData, VData2}); |
| 6051 | if (Is64Bit) |
| 6052 | VData = DAG.getBitcast(MVT::v4i32, VData); |
| 6053 | |
| 6054 | ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32; |
| 6055 | DMask = Is64Bit ? 0xf : 0x3; |
| 6056 | NumVDataDwords = Is64Bit ? 4 : 2; |
| 6057 | } else { |
| 6058 | DMask = Is64Bit ? 0x3 : 0x1; |
| 6059 | NumVDataDwords = Is64Bit ? 2 : 1; |
| 6060 | } |
| 6061 | } else { |
| 6062 | auto *DMaskConst = |
| 6063 | cast<ConstantSDNode>(Op.getOperand(ArgOffset + Intr->DMaskIndex)); |
| 6064 | DMask = DMaskConst->getZExtValue(); |
| 6065 | DMaskLanes = BaseOpcode->Gather4 ? 4 : countPopulation(DMask); |
| 6066 | |
| 6067 | if (BaseOpcode->Store) { |
| 6068 | VData = Op.getOperand(2); |
| 6069 | |
| 6070 | MVT StoreVT = VData.getSimpleValueType(); |
| 6071 | if (StoreVT.getScalarType() == MVT::f16) { |
| 6072 | if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16) |
| 6073 | return Op; // D16 is unsupported for this instruction |
| 6074 | |
| 6075 | IsD16 = true; |
| 6076 | VData = handleD16VData(VData, DAG, true); |
| 6077 | } |
| 6078 | |
| 6079 | NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32; |
| 6080 | } else { |
| 6081 | // Work out the num dwords based on the dmask popcount and underlying type |
| 6082 | // and whether packing is supported. |
| 6083 | MVT LoadVT = ResultTypes[0].getSimpleVT(); |
| 6084 | if (LoadVT.getScalarType() == MVT::f16) { |
| 6085 | if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16) |
| 6086 | return Op; // D16 is unsupported for this instruction |
| 6087 | |
| 6088 | IsD16 = true; |
| 6089 | } |
| 6090 | |
| 6091 | // Confirm that the return type is large enough for the dmask specified |
| 6092 | if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) || |
| 6093 | (!LoadVT.isVector() && DMaskLanes > 1)) |
| 6094 | return Op; |
| 6095 | |
| 6096 | // The sq block of gfx8 and gfx9 do not estimate register use correctly |
| 6097 | // for d16 image_gather4, image_gather4_l, and image_gather4_lz |
| 6098 | // instructions. |
| 6099 | if (IsD16 && !Subtarget->hasUnpackedD16VMem() && |
| 6100 | !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug())) |
| 6101 | NumVDataDwords = (DMaskLanes + 1) / 2; |
| 6102 | else |
| 6103 | NumVDataDwords = DMaskLanes; |
| 6104 | |
| 6105 | AdjustRetType = true; |
| 6106 | } |
| 6107 | } |
| 6108 | |
| 6109 | unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd; |
| 6110 | SmallVector<SDValue, 4> VAddrs; |
| 6111 | |
| 6112 | // Optimize _L to _LZ when _L is zero |
| 6113 | if (LZMappingInfo) { |
| 6114 | if (auto *ConstantLod = dyn_cast<ConstantFPSDNode>( |
| 6115 | Op.getOperand(ArgOffset + Intr->LodIndex))) { |
| 6116 | if (ConstantLod->isZero() || ConstantLod->isNegative()) { |
| 6117 | IntrOpcode = LZMappingInfo->LZ; // set new opcode to _lz variant of _l |
| 6118 | VAddrEnd--; // remove 'lod' |
| 6119 | } |
| 6120 | } |
| 6121 | } |
| 6122 | |
| 6123 | // Optimize _mip away, when 'lod' is zero |
| 6124 | if (MIPMappingInfo) { |
| 6125 | if (auto *ConstantLod = dyn_cast<ConstantSDNode>( |
| 6126 | Op.getOperand(ArgOffset + Intr->MipIndex))) { |
| 6127 | if (ConstantLod->isNullValue()) { |
| 6128 | IntrOpcode = MIPMappingInfo->NONMIP; // set new opcode to variant without _mip |
| 6129 | VAddrEnd--; // remove 'mip' |
| 6130 | } |
| 6131 | } |
| 6132 | } |
| 6133 | |
| 6134 | // Push back extra arguments. |
| 6135 | for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) |
| 6136 | VAddrs.push_back(Op.getOperand(ArgOffset + I)); |
| 6137 | |
| 6138 | // Check for 16 bit addresses or derivatives and pack if true. |
| 6139 | MVT VAddrVT = |
| 6140 | Op.getOperand(ArgOffset + Intr->GradientStart).getSimpleValueType(); |
| 6141 | MVT VAddrScalarVT = VAddrVT.getScalarType(); |
| 6142 | MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; |
| 6143 | IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; |
| 6144 | |
| 6145 | VAddrVT = Op.getOperand(ArgOffset + Intr->CoordStart).getSimpleValueType(); |
| 6146 | VAddrScalarVT = VAddrVT.getScalarType(); |
| 6147 | MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; |
| 6148 | IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; |
| 6149 | |
| 6150 | if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) { |
| 6151 | // 16 bit gradients are supported, but are tied to the A16 control |
| 6152 | // so both gradients and addresses must be 16 bit |
| 6153 | LLVM_DEBUG(do { } while (false) |
| 6154 | dbgs() << "Failed to lower image intrinsic: 16 bit addresses "do { } while (false) |
| 6155 | "require 16 bit args for both gradients and addresses")do { } while (false); |
| 6156 | return Op; |
| 6157 | } |
| 6158 | |
| 6159 | if (IsA16) { |
| 6160 | if (!ST->hasA16()) { |
| 6161 | LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "do { } while (false) |
| 6162 | "support 16 bit addresses\n")do { } while (false); |
| 6163 | return Op; |
| 6164 | } |
| 6165 | } |
| 6166 | |
| 6167 | // We've dealt with incorrect input so we know that if IsA16, IsG16 |
| 6168 | // are set then we have to compress/pack operands (either address, |
| 6169 | // gradient or both) |
| 6170 | // In the case where a16 and gradients are tied (no G16 support) then we |
| 6171 | // have already verified that both IsA16 and IsG16 are true |
| 6172 | if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) { |
| 6173 | // Activate g16 |
| 6174 | const AMDGPU::MIMGG16MappingInfo *G16MappingInfo = |
| 6175 | AMDGPU::getMIMGG16MappingInfo(Intr->BaseOpcode); |
| 6176 | IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16 |
| 6177 | } |
| 6178 | |
| 6179 | // Add gradients (packed or unpacked) |
| 6180 | if (IsG16) { |
| 6181 | // Pack the gradients |
| 6182 | // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart); |
| 6183 | packImage16bitOpsToDwords(DAG, Op, GradPackVectorVT, VAddrs, |
| 6184 | ArgOffset + Intr->GradientStart, |
| 6185 | ArgOffset + Intr->CoordStart, Intr->NumGradients); |
| 6186 | } else { |
| 6187 | for (unsigned I = ArgOffset + Intr->GradientStart; |
| 6188 | I < ArgOffset + Intr->CoordStart; I++) |
| 6189 | VAddrs.push_back(Op.getOperand(I)); |
| 6190 | } |
| 6191 | |
| 6192 | // Add addresses (packed or unpacked) |
| 6193 | if (IsA16) { |
| 6194 | packImage16bitOpsToDwords(DAG, Op, AddrPackVectorVT, VAddrs, |
| 6195 | ArgOffset + Intr->CoordStart, VAddrEnd, |
| 6196 | 0 /* No gradients */); |
| 6197 | } else { |
| 6198 | // Add uncompressed address |
| 6199 | for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++) |
| 6200 | VAddrs.push_back(Op.getOperand(I)); |
| 6201 | } |
| 6202 | |
| 6203 | // If the register allocator cannot place the address registers contiguously |
| 6204 | // without introducing moves, then using the non-sequential address encoding |
| 6205 | // is always preferable, since it saves VALU instructions and is usually a |
| 6206 | // wash in terms of code size or even better. |
| 6207 | // |
| 6208 | // However, we currently have no way of hinting to the register allocator that |
| 6209 | // MIMG addresses should be placed contiguously when it is possible to do so, |
| 6210 | // so force non-NSA for the common 2-address case as a heuristic. |
| 6211 | // |
| 6212 | // SIShrinkInstructions will convert NSA encodings to non-NSA after register |
| 6213 | // allocation when possible. |
| 6214 | bool UseNSA = ST->hasFeature(AMDGPU::FeatureNSAEncoding) && |
| 6215 | VAddrs.size() >= 3 && |
| 6216 | VAddrs.size() <= (unsigned)ST->getNSAMaxSize(); |
| 6217 | SDValue VAddr; |
| 6218 | if (!UseNSA) |
| 6219 | VAddr = getBuildDwordsVector(DAG, DL, VAddrs); |
| 6220 | |
| 6221 | SDValue True = DAG.getTargetConstant(1, DL, MVT::i1); |
| 6222 | SDValue False = DAG.getTargetConstant(0, DL, MVT::i1); |
| 6223 | SDValue Unorm; |
| 6224 | if (!BaseOpcode->Sampler) { |
| 6225 | Unorm = True; |
| 6226 | } else { |
| 6227 | auto UnormConst = |
| 6228 | cast<ConstantSDNode>(Op.getOperand(ArgOffset + Intr->UnormIndex)); |
| 6229 | |
| 6230 | Unorm = UnormConst->getZExtValue() ? True : False; |
| 6231 | } |
| 6232 | |
| 6233 | SDValue TFE; |
| 6234 | SDValue LWE; |
| 6235 | SDValue TexFail = Op.getOperand(ArgOffset + Intr->TexFailCtrlIndex); |
| 6236 | bool IsTexFail = false; |
| 6237 | if (!parseTexFail(TexFail, DAG, &TFE, &LWE, IsTexFail)) |
| 6238 | return Op; |
| 6239 | |
| 6240 | if (IsTexFail) { |
| 6241 | if (!DMaskLanes) { |
| 6242 | // Expecting to get an error flag since TFC is on - and dmask is 0 |
| 6243 | // Force dmask to be at least 1 otherwise the instruction will fail |
| 6244 | DMask = 0x1; |
| 6245 | DMaskLanes = 1; |
| 6246 | NumVDataDwords = 1; |
| 6247 | } |
| 6248 | NumVDataDwords += 1; |
| 6249 | AdjustRetType = true; |
| 6250 | } |
| 6251 | |
| 6252 | // Has something earlier tagged that the return type needs adjusting |
| 6253 | // This happens if the instruction is a load or has set TexFailCtrl flags |
| 6254 | if (AdjustRetType) { |
| 6255 | // NumVDataDwords reflects the true number of dwords required in the return type |
| 6256 | if (DMaskLanes == 0 && !BaseOpcode->Store) { |
| 6257 | // This is a no-op load. This can be eliminated |
| 6258 | SDValue Undef = DAG.getUNDEF(Op.getValueType()); |
| 6259 | if (isa<MemSDNode>(Op)) |
| 6260 | return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL); |
| 6261 | return Undef; |
| 6262 | } |
| 6263 | |
| 6264 | EVT NewVT = NumVDataDwords > 1 ? |
| 6265 | EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumVDataDwords) |
| 6266 | : MVT::i32; |
| 6267 | |
| 6268 | ResultTypes[0] = NewVT; |
| 6269 | if (ResultTypes.size() == 3) { |
| 6270 | // Original result was aggregate type used for TexFailCtrl results |
| 6271 | // The actual instruction returns as a vector type which has now been |
| 6272 | // created. Remove the aggregate result. |
| 6273 | ResultTypes.erase(&ResultTypes[1]); |
| 6274 | } |
| 6275 | } |
| 6276 | |
| 6277 | unsigned CPol = cast<ConstantSDNode>( |
| 6278 | Op.getOperand(ArgOffset + Intr->CachePolicyIndex))->getZExtValue(); |
| 6279 | if (BaseOpcode->Atomic) |
| 6280 | CPol |= AMDGPU::CPol::GLC; // TODO no-return optimization |
| 6281 | if (CPol & ~AMDGPU::CPol::ALL) |
| 6282 | return Op; |
| 6283 | |
| 6284 | SmallVector<SDValue, 26> Ops; |
| 6285 | if (BaseOpcode->Store || BaseOpcode->Atomic) |
| 6286 | Ops.push_back(VData); // vdata |
| 6287 | if (UseNSA) |
| 6288 | append_range(Ops, VAddrs); |
| 6289 | else |
| 6290 | Ops.push_back(VAddr); |
| 6291 | Ops.push_back(Op.getOperand(ArgOffset + Intr->RsrcIndex)); |
| 6292 | if (BaseOpcode->Sampler) |
| 6293 | Ops.push_back(Op.getOperand(ArgOffset + Intr->SampIndex)); |
| 6294 | Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32)); |
| 6295 | if (IsGFX10Plus) |
| 6296 | Ops.push_back(DAG.getTargetConstant(DimInfo->Encoding, DL, MVT::i32)); |
| 6297 | Ops.push_back(Unorm); |
| 6298 | Ops.push_back(DAG.getTargetConstant(CPol, DL, MVT::i32)); |
| 6299 | Ops.push_back(IsA16 && // r128, a16 for gfx9 |
| 6300 | ST->hasFeature(AMDGPU::FeatureR128A16) ? True : False); |
| 6301 | if (IsGFX10Plus) |
| 6302 | Ops.push_back(IsA16 ? True : False); |
| 6303 | if (!Subtarget->hasGFX90AInsts()) { |
| 6304 | Ops.push_back(TFE); //tfe |
| 6305 | } else if (cast<ConstantSDNode>(TFE)->getZExtValue()) { |
| 6306 | report_fatal_error("TFE is not supported on this GPU"); |
| 6307 | } |
| 6308 | Ops.push_back(LWE); // lwe |
| 6309 | if (!IsGFX10Plus) |
| 6310 | Ops.push_back(DimInfo->DA ? True : False); |
| 6311 | if (BaseOpcode->HasD16) |
| 6312 | Ops.push_back(IsD16 ? True : False); |
| 6313 | if (isa<MemSDNode>(Op)) |
| 6314 | Ops.push_back(Op.getOperand(0)); // chain |
| 6315 | |
| 6316 | int NumVAddrDwords = |
| 6317 | UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32; |
| 6318 | int Opcode = -1; |
| 6319 | |
| 6320 | if (IsGFX10Plus) { |
| 6321 | Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, |
| 6322 | UseNSA ? AMDGPU::MIMGEncGfx10NSA |
| 6323 | : AMDGPU::MIMGEncGfx10Default, |
| 6324 | NumVDataDwords, NumVAddrDwords); |
| 6325 | } else { |
| 6326 | if (Subtarget->hasGFX90AInsts()) { |
| 6327 | Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a, |
| 6328 | NumVDataDwords, NumVAddrDwords); |
| 6329 | if (Opcode == -1) |
| 6330 | report_fatal_error( |
| 6331 | "requested image instruction is not supported on this GPU"); |
| 6332 | } |
| 6333 | if (Opcode == -1 && |
| 6334 | Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) |
| 6335 | Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8, |
| 6336 | NumVDataDwords, NumVAddrDwords); |
| 6337 | if (Opcode == -1) |
| 6338 | Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6, |
| 6339 | NumVDataDwords, NumVAddrDwords); |
| 6340 | } |
| 6341 | assert(Opcode != -1)((void)0); |
| 6342 | |
| 6343 | MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops); |
| 6344 | if (auto MemOp = dyn_cast<MemSDNode>(Op)) { |
| 6345 | MachineMemOperand *MemRef = MemOp->getMemOperand(); |
| 6346 | DAG.setNodeMemRefs(NewNode, {MemRef}); |
| 6347 | } |
| 6348 | |
| 6349 | if (BaseOpcode->AtomicX2) { |
| 6350 | SmallVector<SDValue, 1> Elt; |
| 6351 | DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1); |
| 6352 | return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL); |
| 6353 | } |
| 6354 | if (BaseOpcode->Store) |
| 6355 | return SDValue(NewNode, 0); |
| 6356 | return constructRetValue(DAG, NewNode, |
| 6357 | OrigResultTypes, IsTexFail, |
| 6358 | Subtarget->hasUnpackedD16VMem(), IsD16, |
| 6359 | DMaskLanes, NumVDataDwords, DL); |
| 6360 | } |
| 6361 | |
| 6362 | SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, |
| 6363 | SDValue Offset, SDValue CachePolicy, |
| 6364 | SelectionDAG &DAG) const { |
| 6365 | MachineFunction &MF = DAG.getMachineFunction(); |
| 6366 | |
| 6367 | const DataLayout &DataLayout = DAG.getDataLayout(); |
| 6368 | Align Alignment = |
| 6369 | DataLayout.getABITypeAlign(VT.getTypeForEVT(*DAG.getContext())); |
| 6370 | |
| 6371 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
| 6372 | MachinePointerInfo(), |
| 6373 | MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | |
| 6374 | MachineMemOperand::MOInvariant, |
| 6375 | VT.getStoreSize(), Alignment); |
| 6376 | |
| 6377 | if (!Offset->isDivergent()) { |
| 6378 | SDValue Ops[] = { |
| 6379 | Rsrc, |
| 6380 | Offset, // Offset |
| 6381 | CachePolicy |
| 6382 | }; |
| 6383 | |
| 6384 | // Widen vec3 load to vec4. |
| 6385 | if (VT.isVector() && VT.getVectorNumElements() == 3) { |
| 6386 | EVT WidenedVT = |
| 6387 | EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4); |
| 6388 | auto WidenedOp = DAG.getMemIntrinsicNode( |
| 6389 | AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT), Ops, WidenedVT, |
| 6390 | MF.getMachineMemOperand(MMO, 0, WidenedVT.getStoreSize())); |
| 6391 | auto Subvector = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WidenedOp, |
| 6392 | DAG.getVectorIdxConstant(0, DL)); |
| 6393 | return Subvector; |
| 6394 | } |
| 6395 | |
| 6396 | return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL, |
| 6397 | DAG.getVTList(VT), Ops, VT, MMO); |
| 6398 | } |
| 6399 | |
| 6400 | // We have a divergent offset. Emit a MUBUF buffer load instead. We can |
| 6401 | // assume that the buffer is unswizzled. |
| 6402 | SmallVector<SDValue, 4> Loads; |
| 6403 | unsigned NumLoads = 1; |
| 6404 | MVT LoadVT = VT.getSimpleVT(); |
| 6405 | unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1; |
| 6406 | assert((LoadVT.getScalarType() == MVT::i32 ||((void)0) |
| 6407 | LoadVT.getScalarType() == MVT::f32))((void)0); |
| 6408 | |
| 6409 | if (NumElts == 8 || NumElts == 16) { |
| 6410 | NumLoads = NumElts / 4; |
| 6411 | LoadVT = MVT::getVectorVT(LoadVT.getScalarType(), 4); |
| 6412 | } |
| 6413 | |
| 6414 | SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue}); |
| 6415 | SDValue Ops[] = { |